reformat as tabs
This commit is contained in:
Elan Ruusamäe
parent
184a596dc4
commit
1e6ddcea4b
1 changed files with 186 additions and 186 deletions
372
moin2doku.py
372
moin2doku.py
|
|
@ -22,215 +22,215 @@ from moinformat import moin2doku
|
||||||
USEC = 1000000
|
USEC = 1000000
|
||||||
|
|
||||||
def init_dirs(output_dir):
|
def init_dirs(output_dir):
|
||||||
if not isdir(output_dir):
|
if not isdir(output_dir):
|
||||||
print >> sys.stderr, "Output directory doesn't exist!"
|
print >> sys.stderr, "Output directory doesn't exist!"
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
pagedir = os.path.join(output_dir, 'pages')
|
pagedir = os.path.join(output_dir, 'pages')
|
||||||
if not isdir(pagedir):
|
if not isdir(pagedir):
|
||||||
mkdir(pagedir)
|
mkdir(pagedir)
|
||||||
|
|
||||||
mediadir = os.path.join(output_dir, 'media')
|
mediadir = os.path.join(output_dir, 'media')
|
||||||
if not isdir(mediadir):
|
if not isdir(mediadir):
|
||||||
mkdir(mediadir)
|
mkdir(mediadir)
|
||||||
|
|
||||||
metadir = os.path.join(output_dir, 'meta')
|
metadir = os.path.join(output_dir, 'meta')
|
||||||
if not isdir(metadir):
|
if not isdir(metadir):
|
||||||
mkdir(metadir)
|
mkdir(metadir)
|
||||||
|
|
||||||
def readfile(filename):
|
def readfile(filename):
|
||||||
with open(filename, 'r') as f:
|
with open(filename, 'r') as f:
|
||||||
text = f.read()
|
text = f.read()
|
||||||
return unicode(text.decode('utf-8'))
|
return unicode(text.decode('utf-8'))
|
||||||
|
|
||||||
def writefile(filename, content, overwrite=False):
|
def writefile(filename, content, overwrite=False):
|
||||||
dir = os.path.dirname(os.path.abspath(filename))
|
dir = os.path.dirname(os.path.abspath(filename))
|
||||||
if not isdir(dir):
|
if not isdir(dir):
|
||||||
os.makedirs(dir);
|
os.makedirs(dir);
|
||||||
|
|
||||||
if os.path.exists(filename) and overwrite == False:
|
if os.path.exists(filename) and overwrite == False:
|
||||||
raise OSError, 'File already exists: %s' % filename
|
raise OSError, 'File already exists: %s' % filename
|
||||||
|
|
||||||
f = codecs.open(filename, 'w', 'utf-8')
|
f = codecs.open(filename, 'w', 'utf-8')
|
||||||
f.write(content)
|
f.write(content)
|
||||||
f.close()
|
f.close()
|
||||||
|
|
||||||
# pagedir = MoinMoin page dir
|
# pagedir = MoinMoin page dir
|
||||||
# ns = DokuWiki namespace where attachments to copy
|
# ns = DokuWiki namespace where attachments to copy
|
||||||
def copy_attachments(pagedir, ns):
|
def copy_attachments(pagedir, ns):
|
||||||
srcdir = os.path.join(pagedir, 'attachments')
|
srcdir = os.path.join(pagedir, 'attachments')
|
||||||
if not isdir(srcdir):
|
if not isdir(srcdir):
|
||||||
return
|
return
|
||||||
|
|
||||||
attachment_dir = os.path.join(output_dir, 'media', dw.mediaFN(ns))
|
attachment_dir = os.path.join(output_dir, 'media', dw.mediaFN(ns))
|
||||||
if not isdir(attachment_dir):
|
if not isdir(attachment_dir):
|
||||||
os.makedirs(attachment_dir);
|
os.makedirs(attachment_dir);
|
||||||
|
|
||||||
attachments = listdir(srcdir)
|
attachments = listdir(srcdir)
|
||||||
for attachment in attachments:
|
for attachment in attachments:
|
||||||
src = os.path.join(srcdir, attachment)
|
src = os.path.join(srcdir, attachment)
|
||||||
dst = os.path.join(output_dir, 'media', dw.mediaFN(dw.cleanID("%s/%s" % (ns, attachment))))
|
dst = os.path.join(output_dir, 'media', dw.mediaFN(dw.cleanID("%s/%s" % (ns, attachment))))
|
||||||
copyfile(src, dst)
|
copyfile(src, dst)
|
||||||
copystat(src, dst)
|
copystat(src, dst)
|
||||||
|
|
||||||
def print_help():
|
def print_help():
|
||||||
program = sys.argv[0]
|
program = sys.argv[0]
|
||||||
print "Usage: %s OPTIONS" % program
|
print "Usage: %s OPTIONS" % program
|
||||||
print "Convert MoinMoin pages to DokuWiki."
|
print "Convert MoinMoin pages to DokuWiki."
|
||||||
print "Options:"
|
print "Options:"
|
||||||
print "-d DIR - output directory"
|
print "-d DIR - output directory"
|
||||||
print "-a - Convert Attic pages (history)"
|
print "-a - Convert Attic pages (history)"
|
||||||
print "-f - overwrite output files"
|
print "-f - overwrite output files"
|
||||||
print "-F FILE - convert single file"
|
print "-F FILE - convert single file"
|
||||||
print "-r FILE - write config for redirect plugin"
|
print "-r FILE - write config for redirect plugin"
|
||||||
print ""
|
print ""
|
||||||
print "%s -a -d /var/lib/dokuwiki" % program
|
print "%s -a -d /var/lib/dokuwiki" % program
|
||||||
print "%s -F moinmoin/data/pages/frontpage -d out" % program
|
print "%s -F moinmoin/data/pages/frontpage -d out" % program
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
# return unicode encoded wikiname
|
# return unicode encoded wikiname
|
||||||
# input is a dir from moinmoin pages/ dir
|
# input is a dir from moinmoin pages/ dir
|
||||||
def wikiname(filename):
|
def wikiname(filename):
|
||||||
return wikiutil.unquoteWikiname(basename(filename))
|
return wikiutil.unquoteWikiname(basename(filename))
|
||||||
|
|
||||||
def convert_editlog(pagedir, output = None, overwrite = False):
|
def convert_editlog(pagedir, output = None, overwrite = False):
|
||||||
pagedir = os.path.abspath(pagedir)
|
pagedir = os.path.abspath(pagedir)
|
||||||
print "pagedir: %s" % pagedir
|
print "pagedir: %s" % pagedir
|
||||||
pagename = wikiname(pagedir)
|
pagename = wikiname(pagedir)
|
||||||
if not output:
|
if not output:
|
||||||
output = pagename
|
output = pagename
|
||||||
pagelog = os.path.join(pagedir, 'edit-log')
|
pagelog = os.path.join(pagedir, 'edit-log')
|
||||||
edit_log = editlog.EditLog(request, filename = pagelog)
|
edit_log = editlog.EditLog(request, filename = pagelog)
|
||||||
|
|
||||||
changes = {}
|
changes = {}
|
||||||
for log in edit_log:
|
for log in edit_log:
|
||||||
# not supported. perhaps add anyway?
|
# not supported. perhaps add anyway?
|
||||||
if log.action in ('ATTNEW', 'ATTDEL', 'ATTDRW'):
|
if log.action in ('ATTNEW', 'ATTDEL', 'ATTDRW'):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# 1201095949 192.168.2.23 E start glen@delfi.ee
|
# 1201095949 192.168.2.23 E start glen@delfi.ee
|
||||||
author = log.hostname
|
author = log.hostname
|
||||||
if log.userid:
|
if log.userid:
|
||||||
userdata = user.User(request, log.userid)
|
userdata = user.User(request, log.userid)
|
||||||
if userdata.name:
|
if userdata.name:
|
||||||
author = userdata.name
|
author = userdata.name
|
||||||
|
|
||||||
try:
|
try:
|
||||||
action = {
|
action = {
|
||||||
'SAVE' : 'E',
|
'SAVE' : 'E',
|
||||||
'SAVENEW' : 'C',
|
'SAVENEW' : 'C',
|
||||||
'SAVE/REVERT' : 'R',
|
'SAVE/REVERT' : 'R',
|
||||||
}[log.action]
|
}[log.action]
|
||||||
except KeyError:
|
except KeyError:
|
||||||
action = log.action
|
action = log.action
|
||||||
|
|
||||||
mtime = str(log.ed_time_usecs / USEC)
|
mtime = str(log.ed_time_usecs / USEC)
|
||||||
changes[mtime] = "\t".join([mtime, log.addr, action, dw.cleanID(log.pagename), author, log.comment])
|
changes[mtime] = "\t".join([mtime, log.addr, action, dw.cleanID(log.pagename), author, log.comment])
|
||||||
|
|
||||||
# see if we have missing entries, try to recover
|
# see if we have missing entries, try to recover
|
||||||
page = Page(request, pagename)
|
page = Page(request, pagename)
|
||||||
if len(page.getRevList()) != len(changes):
|
if len(page.getRevList()) != len(changes):
|
||||||
print "RECOVERING edit-log, missing %d entries" % (len(page.getRevList()) - len(changes))
|
print "RECOVERING edit-log, missing %d entries" % (len(page.getRevList()) - len(changes))
|
||||||
for rev in page.getRevList():
|
for rev in page.getRevList():
|
||||||
page = Page(request, pagename, rev = rev)
|
page = Page(request, pagename, rev = rev)
|
||||||
mtime = page.mtime_usecs() / USEC
|
mtime = page.mtime_usecs() / USEC
|
||||||
|
|
||||||
if not mtime:
|
if not mtime:
|
||||||
pagefile, realrev, exists = page.get_rev(rev = rev);
|
pagefile, realrev, exists = page.get_rev(rev = rev);
|
||||||
if os.path.exists(pagefile):
|
if os.path.exists(pagefile):
|
||||||
mtime = int(os.path.getmtime(pagefile))
|
mtime = int(os.path.getmtime(pagefile))
|
||||||
print "Recovered %s: %s" % (rev, mtime)
|
print "Recovered %s: %s" % (rev, mtime)
|
||||||
|
|
||||||
mtime = str(mtime)
|
mtime = str(mtime)
|
||||||
if not changes.has_key(mtime):
|
if not changes.has_key(mtime):
|
||||||
changes[mtime] = "\t".join([mtime, '127.0.0.1', '?', dw.cleanID(pagename), 'root', 'recovered entry'])
|
changes[mtime] = "\t".join([mtime, '127.0.0.1', '?', dw.cleanID(pagename), 'root', 'recovered entry'])
|
||||||
print "ADDING %s" % mtime
|
print "ADDING %s" % mtime
|
||||||
|
|
||||||
changes = sorted(changes.values())
|
changes = sorted(changes.values())
|
||||||
out_file = os.path.join(output_dir, 'meta', dw.metaFN(output, '.changes'))
|
out_file = os.path.join(output_dir, 'meta', dw.metaFN(output, '.changes'))
|
||||||
writefile(out_file, "\n".join(changes), overwrite = overwrite)
|
writefile(out_file, "\n".join(changes), overwrite = overwrite)
|
||||||
|
|
||||||
def convertfile(pagedir, output = None, overwrite = False):
|
def convertfile(pagedir, output = None, overwrite = False):
|
||||||
pagedir = os.path.abspath(pagedir)
|
pagedir = os.path.abspath(pagedir)
|
||||||
pagename = wikiname(pagedir)
|
pagename = wikiname(pagedir)
|
||||||
if not output:
|
if not output:
|
||||||
output = pagename
|
output = pagename
|
||||||
|
|
||||||
page = Page(request, pagename)
|
page = Page(request, pagename)
|
||||||
if page.isUnderlayPage():
|
if page.isUnderlayPage():
|
||||||
print "SKIP UNDERLAY"
|
print "SKIP UNDERLAY"
|
||||||
return
|
return
|
||||||
|
|
||||||
current_exists = page.exists()
|
current_exists = page.exists()
|
||||||
current_rev = page.current_rev()
|
current_rev = page.current_rev()
|
||||||
|
|
||||||
if convert_attic:
|
if convert_attic:
|
||||||
revs = page.getRevList()
|
revs = page.getRevList()
|
||||||
else:
|
else:
|
||||||
revs = [current_rev]
|
revs = [current_rev]
|
||||||
|
|
||||||
for rev in revs:
|
for rev in revs:
|
||||||
page = Page(request, pagename, rev = rev)
|
page = Page(request, pagename, rev = rev)
|
||||||
pagefile, realrev, exists = page.get_rev(rev = rev);
|
pagefile, realrev, exists = page.get_rev(rev = rev);
|
||||||
print "EXISTS loop: %s " % exists
|
print "EXISTS loop: %s " % exists
|
||||||
|
|
||||||
mtime = page.mtime_usecs() / USEC
|
mtime = page.mtime_usecs() / USEC
|
||||||
|
|
||||||
if not mtime:
|
if not mtime:
|
||||||
if os.path.exists(pagefile) != exists:
|
if os.path.exists(pagefile) != exists:
|
||||||
raise Exception, "IT SHOULD NOT HAPPEN"
|
raise Exception, "IT SHOULD NOT HAPPEN"
|
||||||
|
|
||||||
if os.path.exists(pagefile):
|
if os.path.exists(pagefile):
|
||||||
mtime = int(os.path.getmtime(pagefile))
|
mtime = int(os.path.getmtime(pagefile))
|
||||||
print "recovered %s: %s" % (rev, mtime)
|
print "recovered %s: %s" % (rev, mtime)
|
||||||
|
|
||||||
if not mtime:
|
if not mtime:
|
||||||
print "NO REVISION: for %s" % pagefile
|
print "NO REVISION: for %s" % pagefile
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if rev == current_rev:
|
if rev == current_rev:
|
||||||
out_file = os.path.join(output_dir, 'pages', dw.wikiFN(output))
|
out_file = os.path.join(output_dir, 'pages', dw.wikiFN(output))
|
||||||
if not convert_attic and not exists:
|
if not convert_attic and not exists:
|
||||||
# if not converting attic, allow current version may not exist anymore
|
# if not converting attic, allow current version may not exist anymore
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
out_file = os.path.join(output_dir, 'attic', dw.wikiFN(output, str(mtime)))
|
out_file = os.path.join(output_dir, 'attic', dw.wikiFN(output, str(mtime)))
|
||||||
|
|
||||||
content = moin2doku(pagename, page.get_raw_body())
|
content = moin2doku(pagename, page.get_raw_body())
|
||||||
if len(content) == 0:
|
if len(content) == 0:
|
||||||
# raise Exception, "No content"
|
# raise Exception, "No content"
|
||||||
print "NO CONTENT: exists: %s,%s" % (exists, os.path.exists(pagefile))
|
print "NO CONTENT: exists: %s,%s" % (exists, os.path.exists(pagefile))
|
||||||
|
|
||||||
writefile(out_file, content, overwrite = overwrite)
|
writefile(out_file, content, overwrite = overwrite)
|
||||||
copystat(pagefile, out_file)
|
copystat(pagefile, out_file)
|
||||||
|
|
||||||
ID = dw.cleanID(output)
|
ID = dw.cleanID(output)
|
||||||
ns = dw.getNS(ID)
|
ns = dw.getNS(ID)
|
||||||
copy_attachments(pagedir, ns)
|
copy_attachments(pagedir, ns)
|
||||||
|
|
||||||
# convert edit-log, it's always present even if current page is not
|
# convert edit-log, it's always present even if current page is not
|
||||||
convert_editlog(pagedir, output = output, overwrite = overwrite)
|
convert_editlog(pagedir, output = output, overwrite = overwrite)
|
||||||
|
|
||||||
# add to redirect.conf if filenames differ
|
# add to redirect.conf if filenames differ
|
||||||
# and page must exist (no redirect for deleted pages)
|
# and page must exist (no redirect for deleted pages)
|
||||||
if redirect_conf and current_exists:
|
if redirect_conf and current_exists:
|
||||||
# redirect dokuwiki plugin is quite picky
|
# redirect dokuwiki plugin is quite picky
|
||||||
# - it doesn't understand if entries are not lowercase
|
# - it doesn't understand if entries are not lowercase
|
||||||
# - it doesn't understand if paths are separated by forward slash
|
# - it doesn't understand if paths are separated by forward slash
|
||||||
old_page = pagename.lower().replace('/', ':').replace(' ', '_')
|
old_page = pagename.lower().replace('/', ':').replace(' ', '_')
|
||||||
if old_page != ID:
|
if old_page != ID:
|
||||||
redirect_map[old_page] = ID
|
redirect_map[old_page] = ID
|
||||||
|
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
#
|
#
|
||||||
# "main" starts here
|
# "main" starts here
|
||||||
#
|
#
|
||||||
try:
|
try:
|
||||||
opts, args = getopt.getopt(sys.argv[1:], 'hfad:F:r:', [ "help" ])
|
opts, args = getopt.getopt(sys.argv[1:], 'hfad:F:r:', [ "help" ])
|
||||||
except getopt.GetoptError, e:
|
except getopt.GetoptError, e:
|
||||||
print >> sys.stderr, 'Incorrect parameters! Use --help switch to learn more.: %s' % e
|
print >> sys.stderr, 'Incorrect parameters! Use --help switch to learn more.: %s' % e
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
overwrite = False
|
overwrite = False
|
||||||
input_file = None
|
input_file = None
|
||||||
|
|
@ -239,22 +239,22 @@ convert_attic = False
|
||||||
redirect_conf = False
|
redirect_conf = False
|
||||||
redirect_map = {}
|
redirect_map = {}
|
||||||
for o, a in opts:
|
for o, a in opts:
|
||||||
if o == "--help" or o == "-h":
|
if o == "--help" or o == "-h":
|
||||||
print_help()
|
print_help()
|
||||||
if o == "-f":
|
if o == "-f":
|
||||||
overwrite = True
|
overwrite = True
|
||||||
if o == "-a":
|
if o == "-a":
|
||||||
convert_attic = True
|
convert_attic = True
|
||||||
if o == "-r":
|
if o == "-r":
|
||||||
redirect_conf = a
|
redirect_conf = a
|
||||||
if o == "-d":
|
if o == "-d":
|
||||||
output_dir = a
|
output_dir = a
|
||||||
if o == "-F":
|
if o == "-F":
|
||||||
input_file = a
|
input_file = a
|
||||||
|
|
||||||
if not output_dir:
|
if not output_dir:
|
||||||
print_help()
|
print_help()
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
print "Output dir is: '%s'" % output_dir
|
print "Output dir is: '%s'" % output_dir
|
||||||
init_dirs(output_dir)
|
init_dirs(output_dir)
|
||||||
|
|
@ -263,25 +263,25 @@ dw = DokuWiki()
|
||||||
request = RequestCLI()
|
request = RequestCLI()
|
||||||
|
|
||||||
if input_file != None:
|
if input_file != None:
|
||||||
res = convertfile(input_file, overwrite = overwrite)
|
res = convertfile(input_file, overwrite = overwrite)
|
||||||
else:
|
else:
|
||||||
converted = 0
|
converted = 0
|
||||||
|
|
||||||
# special: process frontpage so that MoinMoin frontpage gets saved as DokuWiki frontpage
|
# special: process frontpage so that MoinMoin frontpage gets saved as DokuWiki frontpage
|
||||||
page = wikiutil.getFrontPage(request)
|
page = wikiutil.getFrontPage(request)
|
||||||
res = convertfile(page.getPagePath(), output = dw.getId(), overwrite = overwrite)
|
res = convertfile(page.getPagePath(), output = dw.getId(), overwrite = overwrite)
|
||||||
if res != None:
|
if res != None:
|
||||||
converted += 1
|
converted += 1
|
||||||
|
|
||||||
# get list of all pages in wiki
|
# get list of all pages in wiki
|
||||||
pages = request.rootpage.getPageDict(user = '', exists = not convert_attic)
|
pages = request.rootpage.getPageDict(user = '', exists = not convert_attic)
|
||||||
for page in pages.values():
|
for page in pages.values():
|
||||||
res = convertfile(page.getPagePath(), overwrite = overwrite)
|
res = convertfile(page.getPagePath(), overwrite = overwrite)
|
||||||
if res != None:
|
if res != None:
|
||||||
converted += 1
|
converted += 1
|
||||||
print "Processed %d files, converted %d" % (len(pages) + 1, converted)
|
print "Processed %d files, converted %d" % (len(pages) + 1, converted)
|
||||||
|
|
||||||
if redirect_conf:
|
if redirect_conf:
|
||||||
print "Writing %s: %d items" % (redirect_conf, len(redirect_map))
|
print "Writing %s: %d items" % (redirect_conf, len(redirect_map))
|
||||||
content = '\n'.join('\t'.join(pair) for pair in redirect_map.items())
|
content = '\n'.join('\t'.join(pair) for pair in redirect_map.items())
|
||||||
writefile(redirect_conf, content, overwrite = overwrite)
|
writefile(redirect_conf, content, overwrite = overwrite)
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue