fix: handle non-ascii characters in page names and attachments

* Added Windows hints and v1.2 infos
* added some windows batch files
* switched from 'cliopts' to 'cli'
* "named args"
* All arguments passed to php are UTF-8 encoded
* removed local path from repository. Use doku.local.php
* added/changed support for
  - small
  - big
  - anchordef
  - definition_list
  - comment
  - include and search macros
* added #pragma directives as comments to dokuwiki
* recreate "_dokuwiki.changes"
* Support for non-ascii characters in media files

.gitignore

@@ -0,0 +1,4 @@
+settings.local.cmd
+*.log
+*.pyc
+out/

README

@@ -1,39 +0,0 @@
-Complete MoinMoin to DokuWiki converter
-
-Uses native MoinMoin modules to handle converting and translating paths.
-Converts also page history and edit-log.
-
-http://www.dokuwiki.org/tips:moinmoin2doku
-
-Tested with MoinMoin 1.5 and DokuWiki 2012-09-10 releases
-
-You need to run this on host where both MoinMoin and DokuWiki are configured,
-it uses current configuration from both wikis.
-
-Edit doku.php if your DokuWiki installation is other than /usr/share/dokuwiki
-
-To convert moinmoin all pages with history, invoke:
-$ ./moin2doku.py -a -d /var/lib/dokuwiki
-
-To convert single page (FrontPage):
-$ ./moin2doku.py -F moinmoin/data/pages/FrontPage -d out
-
-after conversion be sure to fix ownership
-(www-data:www-data being your uid/gid webserver runs):
-# chown -R www-data:www-data /var/lib/dokuwiki/pages/*
-# chown -R www-data:www-data /var/lib/dokuwiki/media/*
-
-also, depending on your configuration, you may need to gzip the attic pages.
-
-History:
-version 0.1  2010-02
-  Slim Gaillard, based on the "extended python" convert.py script here:
-  https://www.dokuwiki.org/tips:moinmoin2doku?rev=1297006559#extended_python
-
-version 0.2  2011
-  Elan Ruusamäe, moved to github, track history there
-  https://github.com/glensc/moin2doku
-
-version 1.0  2012
-  Complete moinmoin to dokuwiki converter, uses native moinmoin code to handle
-  converting and translating paths. Converts also page history and edit-log.

README.md

@@ -0,0 +1,93 @@
+Complete MoinMoin to DokuWiki converter
+=======================================
+
+Uses native MoinMoin modules to handle converting and translating paths.
+Converts also page history and edit-log.
+
+http://www.dokuwiki.org/tips:moinmoin2doku
+
+Tested with MoinMoin 1.9.9 and DokuWiki 2018-04-22 releases under Windows 7
+
+You need to run this on host where both MoinMoin and DokuWiki are configured,
+it uses current configuration from both wikis.
+
+Edit `doku.php` if your DokuWiki installation is other than `/usr/share/dokuwiki`
+
+To convert moinmoin all pages with history, invoke:
+```
+$ ./moin2doku.py -a -d /var/lib/dokuwiki
+```
+
+To convert single page (FrontPage):
+```
+$ ./moin2doku.py -F moinmoin/data/pages/FrontPage -d out
+```
+
+You should invoke `bin/indexer.php` after conversion to make all pages are indexed.
+
+and ensure ownership of files is correct:
+(`www-data:www-data` being your uid/gid webserver runs):
+```
+# chown -R www-data:www-data /var/lib/dokuwiki/pages/*
+# chown -R www-data:www-data /var/lib/dokuwiki/media/*
+```
+additionally, depending on your configuration, you may need to gzip the attic pages.
+
+
+Hints for Windows Users
+-----------------------
+
+The Batchfiles (`*.cmd`) should help to do the conversation under Windows. You should
+create a copy of the `settings.cmd` and call it `settings.local.cmd` to set your
+own local paths.
+
+Call `moin2doku.cmd` to convert the full MoinMoin Wiki. All DokuWiki pages will be
+written to an `out` folder in the current directory.
+
+This will convert a single page:
+```
+D:\moin2doku\> moin2doku.cmd MyMoinPage
+```
+
+Set `%OUTDIR%` to an alternativ output folder. This should not be the dokuwiki `data`
+folder if you want to do a full conversation.
+
+The `reindex.cmd` will call the `bin/indexer.php`-Skript.
+
+
+History
+=======
+
+version 0.1 (2010-02)
+-------------------
+
+Slim Gaillard, based on the "extended python" convert.py script here:
+https://www.dokuwiki.org/tips:moinmoin2doku?rev=1297006559#extended_python
+
+version 0.2 (2011)
+----------------
+
+Elan Ruusamäe, moved to github, track history there
+https://github.com/glensc/moin2doku
+
+version 1.0 (2012)
+----------------
+
+Complete moinmoin to dokuwiki converter, uses native moinmoin code to handle
+converting and translating paths. Converts also page history and edit-log.
+
+This marks the project "done", I will no longer develop it or support it, as I got my conversion done. However, I do accept patches (pull requests) to sane amount.
+
+I put repo online so others have better starting point than I did.
+
+
+version 1.1 (2015)
+----------------
+Modifed the script to work with newer Moin versions and API changes.
+
+
+version 1.2 (2019-01)
+----------------
+Some modifications to work with current DokuWiki and added more formattings.
+
+Search GitHub Forks for newer versions of this project.

doku.php

@@ -13,10 +13,14 @@
 
 if ('cli' != php_sapi_name()) die();
 
-define('DOKU_INC', '/usr/share/dokuwiki/');
+//add to following define of 'DOKU_INC' to your "doku.local.php" file and adjust the path:
+//define('DOKU_INC', '/home/caddy/wikifarm/dokuwiki/dokuwiki/');
+//define('DOKU_INC', "d:/website/wwwroot/dokuwiki/" );
+require_once './doku.local.php';
+
 require_once DOKU_INC.'inc/init.php';
 require_once DOKU_INC.'inc/common.php';
-require_once DOKU_INC.'inc/cliopts.php';
+require_once DOKU_INC.'inc/cli.php';
 
 # disable gzip regardless of config, then we don't have to compress when converting
 $conf['compression'] = 0; //compress old revisions: (0: off) ('gz': gnuzip) ('bz2': bzip)
@@ -29,29 +33,33 @@ function strip_dir($dir, $fn) {
 	return end(explode($dir.'/', $fn, 2));
 }
 
-switch ($argv[1]) {
+$action = $argv[1];
+$argPage = $argv[2];
+//filext = $argv[3];
+
+switch ($action) {
 case 'cleanID':
-	echo cleanID($argv[2]);
+	echo cleanID($argPage);
 	break;
 case 'wikiFN':
 	if ($argc > 3 && $argv[3]) {
-		echo strip_dir($conf['olddir'], wikiFN($argv[2], $argv[3]));
+		echo strip_dir($conf['olddir'], wikiFN($argPage, $argv[3]));
 	} else {
-		echo strip_dir($conf['datadir'], wikiFN($argv[2]));
+		echo strip_dir($conf['datadir'], wikiFN($argPage));
 	}
 	break;
 case 'mediaFN':
-	echo strip_dir($conf['mediadir'], mediaFN($argv[2]));
+	echo strip_dir($conf['mediadir'], mediaFN($argPage));
 	break;
 case 'metaFN':
-	echo strip_dir($conf['metadir'], metaFN($argv[2], $argv[3]));
+	echo strip_dir($conf['metadir'], metaFN($argPage, $argv[3]));
 	break;
 case 'getNS':
-	echo getNS($argv[2]);
+	echo getNS($argPage);
 	break;
 case 'getId':
 	echo getId();
 	break;
 default:
-	die("Unknown knob: {$argv[1]}");
+	die("Unknown knob: {$action}");
 }

doku.py

@@ -11,6 +11,9 @@
 import sys
 import subprocess
 
+from MoinMoin import log
+logging = log.getLogger(__name__)
+
 class DokuWiki:
 	def __init__(self):
 		self.callcache = {}
@@ -24,10 +27,17 @@ class DokuWiki:
 
 	def __call(self, method, *args):
 		args = list(args)
-		key = "%s:%s" % (method, ",".join(args))
+		uargs = []
+		for arg in args:
+			try:
+				arg.decode('utf-8')
+				#already UTF-8 ready
+				uargs.append(arg)
+			except UnicodeError:
+				uargs.append(arg.encode('utf-8'))
+		key = "%s:%s" % (method, ",".join(uargs))
 		if not self.callcache.has_key(key):
-			cmd = ['./doku.php', method ] + args
-			res = subprocess.Popen(cmd, stdin = None, stdout = subprocess.PIPE, stderr = sys.stderr, close_fds = True).communicate()
+			cmd = ['php', './doku.php', method] + [arg.encode("utf-8") for arg in uargs]
+			res = subprocess.Popen(cmd, stdin = None, stdout = subprocess.PIPE, stderr = sys.stderr, close_fds = False).communicate()
 			self.callcache[key] = unicode(res[0].decode('utf-8'))
-			print "%s->%s" % (cmd, self.callcache[key])
 		return self.callcache[key]

moin2doku.cmd

@@ -0,0 +1,36 @@
+@echo off
+setlocal
+
+call settings.cmd
+
+if not "%1"=="" goto :singlePage %1
+
+if "%OUTDIR%"=="" (
+  call :deldir out\attic || (pause & goto :eof)
+  call :deldir out\media || (pause & goto :eof)
+  call :deldir out\meta || (pause & goto :eof)
+  call :deldir out\pages || (pause & goto :eof)
+  if exist %~n0.pages.log del %~n0.pages.log
+  if not exist out md out
+  set OUTDIR=%CD%\out
+)
+
+call python moin2doku.py %DOKU_FULL_HISTORY% -d "%OUTDIR:\=/%" >"%~n0.log" 2>"%~n0.err.log"
+
+goto :eof
+:deldir
+if exist %1 rd /s/q %1
+if exist %1 exit /B 1
+
+goto :eof
+:singlePage
+if "%OUTDIR%"=="" set OUTDIR=%CD%\out
+call python moin2doku.py %DOKU_FULL_HISTORY% -p "%MOIN_DATA_HOME%\pages\%~1" -f -d "%OUTDIR:\=/%" >>"%~n0.log" 2>>"%~n0.err.log" || type "%~n0.err.log"
+if %ERRORLEVEL% == 0 if exist "%DOKU_ANIMALS_HOME%\%ANIMAL%\conf\local.php" (
+  rem touching "%DOKU_ANIMALS_HOME%\%ANIMAL%\conf\local.php" to invalidate cache
+  pushd "%DOKU_ANIMALS_HOME%\%ANIMAL%\conf"
+  copy /y/b local.php +,, >nul
+  popd
+)
+
+goto :eof

moin2doku.py

@@ -11,7 +11,7 @@
 import sys, os, os.path, re, codecs
 import getopt
 from MoinMoin import user, wikiutil
-from MoinMoin.request import RequestCLI
+from MoinMoin.web.contexts import ScriptContext as RequestCLI
 from MoinMoin.logfile import editlog
 from MoinMoin.Page import Page
 from shutil import copyfile, copystat
@@ -19,6 +19,11 @@ from os import listdir, mkdir
 from os.path import isdir, basename
 from doku import DokuWiki
 from moinformat import moin2doku
+import random
+
+# sys.setdefaultencoding() does not exist, here!
+reload(sys)  # Reload does the trick!
+sys.setdefaultencoding('cp1252')
 
 USEC = 1000000
 
@@ -40,7 +45,7 @@ def init_dirs(output_dir):
 		mkdir(metadir)
 
 def readfile(filename):
-	with open(filename, 'r') as f:
+	f = open(filename, 'r')
 	text = f.read()
 	return unicode(text.decode('utf-8'))
 
@@ -50,7 +55,7 @@ def writefile(filename, content, overwrite=False):
 		os.makedirs(dir);
 
 	if os.path.exists(filename) and overwrite == False:
-		raise OSError, 'File already exists: %s' % filename
+		raise (OSError, 'File already exists: %s' % filename)
 
 	# ensure it's a list
 	if not isinstance(content, (list, tuple)):
@@ -60,9 +65,14 @@ def writefile(filename, content, overwrite=False):
 	f.writelines([line + u'\n' for line in content])
 	f.close()
 
+
+def encode_relaxed(text):
+	return text.encode("ascii", errors="ignore")
+
+
 # page = MoinMoin Page oject
 # ns = DokuWiki namespace where attachments to copy
-def copy_attachments(page, ns):
+def copy_attachments(page, ns,randomID):
 	srcdir = page.getPagePath('attachments', check_create = 0)
 	if not isdir(srcdir):
 		return
@@ -73,10 +83,13 @@ def copy_attachments(page, ns):
 
 	attachments = listdir(srcdir)
 	for attachment in attachments:
+		try:
 			src = os.path.join(srcdir, attachment)
-		dst = os.path.join(output_dir, 'media', dw.mediaFN(dw.cleanID("%s/%s" % (ns, attachment))))
+			dst = os.path.join(output_dir, 'media', dw.mediaFN(dw.cleanID(u"%s/%s" % (ns, str(randomID)+attachment))))
 			copyfile(src, dst)
 			copystat(src, dst)
+		except UnicodeDecodeError:
+			print 'ERROR: unable to convert attachment "%s"' % encode_relaxed(attachment)
 
 def print_help():
 	program = sys.argv[0]
@@ -156,13 +169,16 @@ def convert_editlog(page, output = None, overwrite = False):
 def convertfile(page, output = None, overwrite = False):
 	pagedir = page.getPagePath()
 	pagename = wikiname(pagedir)
+
 	if not output:
 		output = pagename
 
+	print "Converting %s" % encode_relaxed(pagename)
+
 	if page.isUnderlayPage():
 		print "underlay: %s" % page.request.cfg.data_underlay_dir
 		print "underlay: %s" % request.cfg.data_underlay_dir
-		print "SKIP UNDERLAY: %s" % pagename
+		print "SKIP UNDERLAY: %s" % encode_relaxed(pagename)
 		return False
 
 	current_exists = page.exists()
@@ -173,6 +189,9 @@ def convertfile(page, output = None, overwrite = False):
 	else:
 		revs = [current_rev]
 
+	# Generate random ID Number for collision avoidance when attachments in Namespace have the same name
+	randomID = random.randint(101,999)
+
 	for rev in revs:
 		page = Page(request, pagename, rev = rev)
 		pagefile, realrev, exists = page.get_rev(rev = rev);
@@ -188,7 +207,7 @@ def convertfile(page, output = None, overwrite = False):
 				print "recovered %s: %s" % (rev, mtime)
 
 			if not mtime:
-				print "NO REVISION: for %s" % pagefile
+				print "NO REVISION: for %s" % encode_relaxed(pagefile)
 				continue
 
 		if rev == current_rev:
@@ -199,7 +218,7 @@ def convertfile(page, output = None, overwrite = False):
 		else:
 			out_file = os.path.join(output_dir, 'attic', dw.wikiFN(output, str(mtime)))
 
-		content = moin2doku(pagename, page.get_raw_body())
+		content = moin2doku(pagename, page.get_raw_body(),randomID)
 		if len(content) == 0:
 #			raise Exception, "No content"
 			print "NO CONTENT: exists: %s,%s" % (exists, os.path.exists(pagefile))
@@ -208,7 +227,7 @@ def convertfile(page, output = None, overwrite = False):
 		copystat(pagefile, out_file)
 
 	ID = dw.cleanID(output)
-	copy_attachments(page, dw.getNS(ID))
+	copy_attachments(page, dw.getNS(ID),randomID)
 
 	# convert edit-log, it's always present even if current page is not
 	convert_editlog(page, output = output, overwrite = overwrite)
@@ -223,7 +242,7 @@ def convertfile(page, output = None, overwrite = False):
 		if old_page != ID:
 			redirect_map[old_page] = ID
 
-	print "Converted %s as %s" % (pagename, dw.wikiFN(output))
+	print "Converted %s as %s" % (encode_relaxed(pagename), dw.wikiFN(output))
 
 	return True
 
@@ -294,6 +313,7 @@ else:
 	# get list of all pages in wiki
 	# hide underlay dir temporarily
 	underlay_dir = request.rootpage.cfg.data_underlay_dir
+	print(underlay_dir)
 	request.rootpage.cfg.data_underlay_dir = None
 	pages = request.rootpage.getPageList(user = '', exists = not convert_attic, filter = filter)
 	pages = dict(zip(pages, pages))
@@ -307,6 +327,11 @@ else:
 		del pages[frontpage.page_name]
 	pages[dw.getId()] = frontpage.page_name
 
+print "--------------------------------------------------"
+for output, pagename in pages.items():
+	print " - %s" % encode_relaxed(pagename)
+print "--------------------------------------------------"
+
 converted = 0
 for output, pagename in pages.items():
 	page = Page(request, pagename)

moinformat.py

@@ -7,20 +7,22 @@
 #
 # Author: Elan Ruusamäe <glen@pld-linux.org>
 # Version: 1.0
-
-from MoinMoin import wikimacro, wikiutil
+#from MoinMoin.web.request import Request as RequestCLI
+from MoinMoin.web.contexts import ScriptContext
+from MoinMoin import  wikiutil
 from MoinMoin.Page import Page
-from MoinMoin.parser.wiki import Parser
+from MoinMoin.parser.text_moin_wiki import Parser
 
 from text_dokuwiki import Formatter
-from MoinMoin.request import RequestCLI
 
 import sys
 import StringIO
 
-def moin2doku(pagename, text):
+def moin2doku(pagename, text, randomID=None):
 	parser = Parser(text, request)
 
+	formatter.setRandomID(randomID)
+	
 	# this needed for macros
 	request.formatter = formatter
 
@@ -37,7 +39,7 @@ def moin2doku(pagename, text):
 
 	return unicode(output.getvalue().decode('utf-8'))
 
-request = RequestCLI()
+request = ScriptContext()
 formatter = Formatter(request)
 
 if __name__ == "__main__":
@@ -47,6 +49,6 @@ if __name__ == "__main__":
 	else:
 		inputfile = 'syntaxreference.txt'
 
-	with open(inputfile, 'r') as f:
+	f = open(inputfile, 'r')
 	text = f.read()
 	print moin2doku('test', text)

reindex.cmd

@@ -0,0 +1,69 @@
+@echo off
+setlocal
+
+call settings.cmd
+
+pushd "%DOKU_ANIMALS_HOME%\%ANIMAL%"
+
+if not exist data\pages\ (
+  echo WARNUNG:
+  echo Es wurden kein "pages" Verzeichnis gefunden. Wurde die neue Version bereits
+  echo ins Zielverzeichnis kopiert?
+  pause
+  goto :eof
+)
+
+if not exist data\media\logo.png (
+  echo working in %CD%
+  rem call :cleanup data\attic || goto :ende
+  call :cleanup data\cache || goto :ende
+  call :cleanup data\index || goto :ende
+  call :cleanup data\locks || goto :ende
+  call :cleanup data\media_attic || goto :ende
+  call :cleanup data\media_meta || goto :ende
+  call :cleanup data\tmp || goto :ende
+  xcopy /S/I/Y/Q common\*.* data
+)
+
+REM if exist data\pages\startseiteneu.txt (
+  REM ren data\pages\startseiteneu.txt startseite.txt
+  REM ren data\meta\startseiteneu.changes startseite.changes
+REM )
+
+popd 
+pushd "%DOKU_HOME%"
+
+php bin\indexer.php || goto :ende
+
+popd
+pushd "%DOKU_ANIMALS_HOME%\%ANIMAL%"
+
+if exist data\meta\_dokuwiki.changes del data\meta\_dokuwiki.changes
+if exist data\meta\_dokuwiki.changes del data\meta\_dokuwiki.changes
+(
+  for /F "delims=*" %%D in ('dir /b/S/A:D data\meta\*.*') do (
+    if exist "%%D\*.changes" type "%%D\*.changes" 2>nul
+  )
+) > _dokuwiki_unsorted.changes
+sort _dokuwiki_unsorted.changes /O data\meta\_dokuwiki.changes
+del _dokuwiki_unsorted.changes
+
+echo --- compressing old files
+for /F "delims=*" %%T in ('dir data\attic\*.txt /s/b') do (
+  "c:\Program Files\7-Zip\7z.exe" a -bso0 "%%T.gz" "%%T" && del "%%T" || goto :ende
+)
+echo --- done
+
+:ende
+popd
+
+pause
+
+goto :eof
+:cleanup
+if not exist %1 goto :eof
+echo cleaning up %1
+rd /s/q %1
+if exist %1 exit /b 1
+md %1
+echo >nul 2>"%~1\_dummy"

settings.cmd

@@ -0,0 +1,45 @@
+@echo off
+REM -- no setlocal in this script!
+REM make a copy of this file and adjust the paths
+
+set PHP_HOME=c:\Program Files\php
+set PYTHON_HOME=c:\Python27
+REM -- MoinMoin settings 
+set MOIN_HOME=c:\wwwroot\wiki\moin
+set MOIN_CONFIG=%MOIN_HOME%\wiki\config
+REM set MOIN_CONFIG=c:\wwwroot\moinfarmdata\config
+set MOIN_DATA_HOME=%MOIN_HOME%\wiki\data
+REM set MOIN_CONFIG=c:\wwwroot\moinfarmdata\<farmwiki>
+REM -- DokuWiki settings
+set DOKU_HOME=c:\wwwroot\wiki\dokuwiki
+set DOKU_ANIMALS_HOME=%DOKU_HOME%
+REM set DOKU_ANIMALS_HOME=c:\wwwroot\dokufarmdata
+REM set animal=<yourFarmAnimalName>
+REM comment this in to do a full converstion
+REM set DOKU_FULL_HISTORY=-a
+
+REM -- path to your php.ini used by your webserver
+REM set PHP_INI_SCAN_DIR=c:\Program Files\ApacheHttpd\conf\
+
+REM -- set this to your "production" dokuwiki if you want to update only.
+REM set OUTDIR=%DOKU_ANIMALS_HOME%\%animal%\data
+
+REM ----8<--------8<--------8<--------8<--------8<--------8<--------8<----
+REM -- remove everything beyond the line from your copy
+chcp 1252
+
+if exist "%~dpn0.local.cmd" call "%~dpn0.local.cmd"
+
+REM -- %animal% must be lowercase!
+(
+set animal=
+set animal=%animal%
+)
+
+set PATH=%PATH%;%PYTHON_HOME%;%PHP_HOME%
+
+set PYTHONPATH=.
+set PYTHONPATH=%PYTHONPATH%;%MOIN_HOME%
+set PYTHONPATH=%PYTHONPATH%;%MOIN_HOME%\MoinMoin\support
+set PYTHONPATH=%PYTHONPATH%;%MOIN_CONFIG%
+set PYTHONPATH=%PYTHONPATH%;

text_dokuwiki.py

@@ -9,10 +9,13 @@
 """
 
 from xml.sax import saxutils
-from MoinMoin.formatter.base import FormatterBase
+from MoinMoin.formatter import FormatterBase
 from MoinMoin import config
 from MoinMoin.Page import Page
 from types import *
+from MoinMoin import log
+
+logging = log.getLogger(__name__)
 
 # TODO: let base class MoinMoin/formatter/base.py handle not implemented methods
 
@@ -31,10 +34,13 @@ class Formatter(FormatterBase):
         self.in_pre = 0
         self.in_table = 0
         self._text = None # XXX does not work with links in headings!!!!!
-
+        self.randomID= None
         self.list_depth = 0
         self.list_type = ' '
 
+    def setRandomID(self,ID):
+        self.randomID = str(ID)
+
     def _escape(self, text, extra_mapping={"'": "'", '"': """}):
         return saxutils.escape(text, extra_mapping)
 
@@ -57,7 +63,8 @@ class Formatter(FormatterBase):
         return ('<sysmsg>', '</sysmsg>')[not on]
 
     def rawHTML(self, markup):
-		return '<html>' + markup + '</html>'
+        #return '<html>' + markup + '</html>'
+        return ''
 
     def pagelink(self, on, pagename='', page=None, **kw):
         if on:
@@ -69,6 +76,14 @@ class Formatter(FormatterBase):
         if on:
             if interwiki == 'Self':
                 return self.pagelink(on, pagename, **kw)
+            interwikis = {
+                'WikiPedia':'wp',
+                'FrWikiPedia':'wpfr',
+                'DeWikiPedia':'wpde',
+                'MetaWikiPedia':'wpmeta'
+                }
+            if interwiki in interwikis:
+                return '[[%s>%s|' % (interwikis.get(interwiki), pagename)
             return '[[%s>%s|' % (interwiki, pagename)
         else:
             return ']]'
@@ -76,14 +91,17 @@ class Formatter(FormatterBase):
     def url(self, on, url='', css=None, **kw):
         return ('[[%s|' % (self._escape(url)), ']]')[not on]
 
-	def attachment_link(self, url, text, **kw):
-		return '{{%s|%s}}' % (url, text)
+    def attachment_link(self, on, url=None, querystr=None, **kw):
+        if on:
+            return '{{ %s | ' % (self.randomID+url)
+        else: 
+            return ' }}'
 
     def attachment_image(self, url, **kw):
-		return '{{%s|}}' % (url,)
+        return '{{%s|}}' % (self.randomID+url,)
 
     def attachment_drawing(self, url, text, **kw):
-		return '{{%s|%s}}' % (url, text)
+        return '{{%s|%s}}' % (self.randomID+url, text)
 
     def text(self, text, **kw):
         self._did_para = 0
@@ -127,17 +145,25 @@ class Formatter(FormatterBase):
             self.list_type = '*'
         else:
             self.list_depth -= 1
+            if self.list_depth <= 0:
                 self.list_type = ' '
 
         return ['', '\n'][on]
 
+    # generic transclude/include:
+    def transclusion(self, on, **kw):
+        return ''
+
+    def transclusion_param(self, **kw):
+        return ''
+
     def listitem(self, on, **kw):
         # somewhy blockquote uses "listitem" call
         return [(' ' * self.list_depth * 2) + self.list_type + ' ', '\n'][not on]
 
     def code(self, on, **kw):
         """ `typewriter` or {{{typerwriter}}, for code blocks i hope codeblock works """
-		return ["''", "''"][not on]
+        return ["''%%", "%%''"][not on]
 
     def sup(self, on, **kw):
         return ['<sup>', '</sup>'][not on]
@@ -148,12 +174,20 @@ class Formatter(FormatterBase):
     def strike(self, on, **kw):
         return ['<del>', '</del>'][not on]
 
+    def small(self, on, **kw):
+        #https://www.dokuwiki.org/plugin:wrap
+        return ['<wrap lo>', '</wrap>'][not on]
+
+    def big(self, on, **kw):
+        #https://www.dokuwiki.org/plugin:wrap
+        return ['<wrap hi>', '</wrap>'][not on]
+
     def preformatted(self, on, **kw):
         FormatterBase.preformatted(self, on)
         result = ''
         if self.in_p:
             result = self.paragraph(0)
-		return result + ['<file>', '</file>\n'][not on]
+        return result + ['<code>', '</code>\n'][not on]
 
     def paragraph(self, on, **kw):
         FormatterBase.paragraph(self, on)
@@ -178,7 +212,7 @@ class Formatter(FormatterBase):
             self.in_table = 1
         else:
             self.in_table = 0
-		return ''
+        return ['', '\n'][not on]
 
     def table_row(self, on, attrs={}, **kw):
         return ['\n', '|'][not on]
@@ -187,8 +221,8 @@ class Formatter(FormatterBase):
         return ['|', ''][not on]
 
     def anchordef(self, id):
-		# not supported
-		return ''
+        # https://www.dokuwiki.org/plugin:anchor
+        return '{{anchor:'+id+'}}'
 
     def anchorlink(self, on, name='', **kw):
         # kw.id not supported, we hope the anchor matches existing heading on page
@@ -198,16 +232,18 @@ class Formatter(FormatterBase):
         return ['__', '__'][not on]
 
     def definition_list(self, on, **kw):
+        # https://www.dokuwiki.org/plugin:definitionlist
         result = ''
         if self.in_p:
             result = self.paragraph(0)
-		return result + ['<gloss>', '</gloss>'][not on]
+        return result
 
     def definition_term(self, on, compact=0, **kw):
-		return ['<label>', '</label>'][not on]
+		#MoinMoin does no wiki markup in DL-Terms
+        return ['  ;%%', '%%\n'][not on]
 
     def definition_desc(self, on, **kw):
-		return ['<item>', '</item>'][not on]
+        return ['  :', '\n'][not on]
 
     def image(self, src=None, **kw):
         valid_attrs = ['src', 'width', 'height', 'alt', 'title']
@@ -229,7 +265,7 @@ class Formatter(FormatterBase):
 
         return '{{' + url + '}}'
 
-	def code_area(self, on, code_id, code_type='code', show=0, start=-1, step=-1):
+    def code_area(self, on, code_id, code_type='code', show=0, start=-1, step=-1,msg=None):
         syntax = ''
         # switch for Python: http://simonwillison.net/2004/may/7/switch/
         try:
@@ -254,7 +290,11 @@ class Formatter(FormatterBase):
     def comment(self, text):
         # real comments (lines with two hash marks)
         if text[0:2] == '##':
+            # https://www.dokuwiki.org/plugin:comment
+            comment = text[2:].strip()
+            if len(comment)>1:
                 return "/* %s */\n" % text[2:].strip()
+            return '\n'
 
         # Some kind of Processing Instruction
         # http://moinmo.in/HelpOnProcessingInstructions
@@ -264,6 +304,7 @@ class Formatter(FormatterBase):
 
         if tokens[0] == 'acl':
             # TODO: fill acl.auth.php
+            logging.info('SKIPPING ACL: %s', text)
             return ''
 
         if tokens[0] == 'deprecated':
@@ -274,28 +315,178 @@ class Formatter(FormatterBase):
 
         if tokens[0] == 'pragma':
             # TODO: can do 'description' via 'meta' dokuwiki plugin
-			return "/* pragma: %s */\n" % " ".join(tokens[1:])
+            pargs = tokens[1].split(None, 1)
+            if pargs[0]=='section-numbers':
+                return '/* meta: %s */' % tokens
+            logging.info('SKIPPING PRAGMA: %s', tokens)
+            #return "/* pragma: %s */\n" % " ".join(tokens[1:])
+            return ''
 
-		return "/* %s */\n" % text.lstrip('#')
+        #return "/* %s */\n" % text.lstrip('#')
+        return ''
 
-	def macro(self, macro_obj, name, args):
+
+    def macro(self, macro_obj, name, args,markup):
         def email(args):
             mail = args.replace(' AT ', '@')
             mail = mail.replace(' DOT ', '.')
             return '[[%s|%s]]' % (mail, args)
 
+        def showAttachedFiles(args):
+            args = args.split(',')
+            if len(args)>1:
+                return '{{ %s | %s }}' % (self.randomID+args[0].strip(), args[1].strip())
+            else:
+                return ''
+
         # function which will just do what parent class would
         def inherit(args):
             return apply(FormatterBase.macro, (self, macro_obj, name, args))
 
+        def randomQuote(args):
+            # https://www.dokuwiki.org/plugin:xfortune
+            return '{{xfortune>quote:'+args+'.txt}}'
+
+        def monthcal(args):
+            # https://www.dokuwiki.org/plugin:monthcal
+            selfname = self.page.page_name
+            return '{{monthcal:create_links=short,namespace='+selfname.replace('/',':')+'}}'
+
+        def navigation(args):
+            # https://www.dokuwiki.org/plugin:alphaindex
+            selfname = self.page.page_name
+            args = args.split(',')
+            if len(args)>0:
+                try:
+                    result = {
+                      'slides': '[<>]',
+                      'children': '{{alphaindex>:%s#1|nons incol}}' % selfname.replace('/',':'),
+                      'siblings': '{{alphaindex>.#1|nons incol}}',
+                      'slideshow': '/* no support for slideshow navigation */'
+                    }[args[0].strip()]
+                except KeyError:
+                    result = '/* Unknown Navigation: %s #%s#*/' % args, args[0].strip()
+            else:
+                result = '/* Unsupported Navigation: %s */' % args
+            return result
+
+        def footnote(args):
+            return '((%s))' % args
+
+        def dateTimeMacro(args):
+            #https://www.dokuwiki.org/plugin:date
+            #args = args.split(',');
+            return '{{date>%%c|timestamp=strtotime("%s")|locale=de}}' % args
+
+        def dateMacro(args):
+            #https://www.dokuwiki.org/plugin:date
+            #args = args.split(',');
+            return '{{date>%%x|timestamp=strtotime("%s")|locale=de}}' % args
+
+        def includeMacro(args):
+            #https://www.dokuwiki.org/plugin:include
+            #logging.info('Include(%s)' % args)
+            args = map(unicode.strip, args.split(','));
+            #dokupage = ":".join(pagename.split("/"))
+            if len(args)==1:
+                return '{{page>%s&nodate}}' % ":".join(args[0].split("/"))
+            elif(u'titlesonly' in args):
+                #https://www.dokuwiki.org/plugin:changes
+                #https://www.dokuwiki.org/plugin:pagelist
+                selfname = self.page.page_name
+                selfNs = ":".join(selfname.split("/")).lower()
+                pairs = [arg.split('=') for arg in args]
+                # attrs = {}
+                # for key, value in pairs:
+                    # attrs[key] = value
+                #logging.info('pairs:"%s"' % pairs)
+
+                incName = ''#pairs[0]
+                incCount = -1
+                incTitlesOnly = False
+                notNamedParam = 0
+                for pair in pairs:
+                    if len(pair)==1:
+                        if u'titlesonly'==pair:
+                            notNamedParam = -1
+                            incTitlesOnly = True
+                        elif notNamedParam >=0:
+                            if notNamedParam==0:
+                                incName = pairs[notNamedParam]
+                            notNamedParam += 1;
+                    else:
+                        notNamedParam = -1
+                        if u'items'==pair[0]:
+                            incCount = int(pair[1])
+
+                resultArgs = '-h1 -textPages=""'
+                #(keys,values) = map()
+                if incCount > 0:
+                    resultArgs += ' -idAndTitle -simpleList -sortId -nbItemsMax=%d' % incCount
+                else:
+                    resultArgs += ' -nbCol=2'
+
+                ##<nspages fortran:mailarchiv -pregPagesOn="/2.*/" -h1 -nbCol=2 -textPages="">
+                ## Lister der letzten 10 Mails:
+                ##<nspages .:mailarchiv -nbItemsMax=10 -sortId -simpleList -idAndTitle -reverse -h1 -nbCol=1 -textPages=""> 
+                if incName[0]=='^':
+                    nspagedelim = incName.rfind('/')
+                    ns = ":".join(incName[1:nspagedelim].split('/')).lower()
+                    incPageReg = incName[(nspagedelim+1):]
+                    resultArgs += ' -pregPagesOn="/^%s/"' % incPageReg
+                else:
+                    ns = selfNs
+
+                return '<nspages %s %s>' % (ns, resultArgs)
+
+            else:
+                logging.info('UNSUPPORTED INCLUDE "%s"' % args)
+                return '/* Unsupported Include: %s */' % args
+
+        def fullsearch(args):
+            #args=None >> {searchform ns=}
+            #args='' >> {{backlinks>.}}
+            #args!='' >> {{search><args>}}
+            #ignore special searches. see MoinMoin page "HilfeZumSuchen"
+            if args is None:
+                return '{searchform ns=}'
+            elif ':' in args or ' ' in args:
+                logging.info('UNSUPPORTED SEARCH %s(%s)' % (name, args))
+                return '/* Unsupported Search %s(%s). may be backlinks plugin will help */' % (name, args)
+            elif args=='':
+                return '{{backlinks>.}}'
+            elif name=='PageList':
+                return '{{backlinks>%s}}' % ":".join(args.split('/')).lower();
+            else:
+                logging.info('UNSUPPORTED SEARCH %s(%s)' % (name, args))
+                return '/* Unsupported Search %s(%s) */' % (name, args)
+
         try:
             lookup = {
-				'BR' : '\\\\',
+                'BR' : ' \\\\ ',
+                'br' : ' \\\\ ',
                 'MailTo' : email,
                 'GetText' : args,
                 'ShowSmileys' : inherit,
+                'ShowAttachedFiles' : showAttachedFiles,
+                'Include' : includeMacro,
+                #no real fulltext search!
+                'FullSearch' : fullsearch,
+                'FullSearchCached' : fullsearch,
+                'PageList' : fullsearch,
+                'MonthCalendar' : monthcal,
+                'Navigation' : navigation,
+                'TableOfContents' : '',
+                'RandomQuote': randomQuote,
+                'Anchor': inherit,
+                'Action': inherit,
+                'Icon': inherit,
+                'FootNote': footnote,
+                'Date': dateMacro,
+                'DateTime': dateTimeMacro
             }[name]
         except KeyError:
+            logging.info('UNDEFINED MACRO "%s"' % name)
             lookup = '/* UndefinedMacro: %s(%s) */' % (name, args)
 
         if type(lookup) == FunctionType: