Attachment '0001-Implement-an-incremental-dump-process-6.patch'
Download 1 From 48c917b2cb9042aca63c54670b2ccdbe659e00a1 Mon Sep 17 00:00:00 2001
2 From: Paul Wise <pabs3@bonedaddy.net>
3 Date: Sat, 20 Apr 2013 13:46:50 +0800
4 Subject: [PATCH] Implement an incremental dump process
5
6 ---
7 MoinMoin/script/export/dump.py | 83 +++++++++++++++++++++++++++++++++++++-----
8 1 file changed, 74 insertions(+), 9 deletions(-)
9
10 diff --git a/MoinMoin/script/export/dump.py b/MoinMoin/script/export/dump.py
11 index d770e5b..75d0b08 100644
12 --- a/MoinMoin/script/export/dump.py
13 +++ b/MoinMoin/script/export/dump.py
14 @@ -12,11 +12,16 @@ import sys, os, time, codecs, shutil, re, errno
15 from MoinMoin import config, wikiutil, Page, user
16 from MoinMoin import script
17 from MoinMoin.action import AttachFile
18 +from MoinMoin.logfile import editlog
19 +from MoinMoin.util.filesys import touch
20
21 url_prefix_static = "."
22 logo_html = '<img src="logo.png">'
23 HTML_SUFFIX = ".html"
24
25 +timestamp_text = u'''This is a MoinMoin timestamp file.
26 +Please delete it to rebuild all pages.
27 +'''
28 page_template = u'''<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
29 <html>
30 <head>
31 @@ -126,6 +131,28 @@ General syntax: moin [options] export dump [dump-options]
32 help = "User the dump will be performed as (for ACL checks, etc)"
33 )
34
35 + def changed_pages(self, request, outputdir, timestamp):
36 + update = set()
37 + delete = set()
38 + ignore = set()
39 + version = wikiutil.timestamp2version(timestamp)
40 + log = editlog.EditLog(request)
41 + for line in log:
42 + if not request.user.may.read(line.pagename):
43 + ignore.add(line.pagename)
44 + elif line.ed_time_usecs < version:
45 + ignore.add(line.pagename)
46 + elif line.action == 'ATTDEL':
47 + name = os.path.join(outputdir, "attachments", wikiutil.quoteWikinameFS(line.pagename), line.extra)
48 + url = "attachments/%s/%s" % (wikiutil.quoteWikinameFS(line.pagename), line.extra)
49 + delete.add((name, url))
50 + elif line.action == 'SAVE/RENAME':
51 + update.add(line.pagename)
52 + update.add(line.extra)
53 + else:
54 + update.add(line.pagename)
55 + return (ignore, delete, update)
56 +
57 def mainloop(self):
58 """ moin-dump's main code. """
59
60 @@ -157,8 +184,41 @@ General syntax: moin [options] export dump [dump-options]
61 # use this user for permissions checks
62 request.user = user.User(request, name=self.options.dump_user)
63
64 - pages = request.rootpage.getPageList(user='') # get list of all pages in wiki
65 + # Check the last update timestamp
66 + timestamp = None
67 + timestamp_file = os.path.join(outputdir, 'moin-last-update')
68 + try:
69 + timestamp = os.stat(timestamp_file).st_mtime
70 + except OSError, err:
71 + if err.errno == errno.ENOENT:
72 + with open(timestamp_file, 'w') as f:
73 + f.write(timestamp_text)
74 + else:
75 + script.fatal("Cannot check last update time of '%s'!" % timestamp_file)
76 +
77 + # Setup some helper functions
78 + wikiutil.quoteWikinameURL = lambda pagename, qfn=wikiutil.quoteWikinameFS: (qfn(pagename) + HTML_SUFFIX)
79 + AttachFile.getAttachUrl = lambda pagename, filename, request, **kw: _attachment(request, pagename, filename, outputdir, **kw)
80 +
81 + # Get a list of pages
82 + if timestamp:
83 + touch(timestamp_file)
84 + ignore, delete, render = self.changed_pages(request, outputdir, timestamp)
85 + pages = list(render)
86 + for (name, url) in delete:
87 + try:
88 + os.remove(name)
89 + except OSError, err:
90 + if err.errno != errno.ENOENT:
91 + script.fatal("Cannot remove '%s'!" % url)
92 + script.log('Removed "%s"...' % url)
93 + if ignore:
94 + script.log('Ignored %s changes before the last export dump' % len(ignore))
95 + else:
96 + pages = request.rootpage.getPageList(user='') # get list of all pages in wiki
97 pages.sort()
98 +
99 + # Filter the list of pages
100 if self.options.page: # did user request a particular page or group of pages?
101 try:
102 namematch = re.compile(self.options.page)
103 @@ -168,10 +228,6 @@ General syntax: moin [options] export dump [dump-options]
104 except:
105 pages = [self.options.page]
106
107 - wikiutil.quoteWikinameURL = lambda pagename, qfn=wikiutil.quoteWikinameFS: (qfn(pagename) + HTML_SUFFIX)
108 -
109 - AttachFile.getAttachUrl = lambda pagename, filename, request, **kw: _attachment(request, pagename, filename, outputdir, **kw)
110 -
111 errfile = os.path.join(outputdir, 'error.log')
112 errlog = open(errfile, 'w')
113 errcnt = 0
114 @@ -188,11 +244,21 @@ General syntax: moin [options] export dump [dump-options]
115 for pagename in pages:
116 # we have the same name in URL and FS
117 file = wikiutil.quoteWikinameURL(pagename)
118 - script.log('Writing "%s"...' % file)
119 + filepath = os.path.join(outputdir, file)
120 + request.url = urlbase + pagename # add current pagename to url base
121 + page = Page.Page(request, pagename)
122 + if page.exists():
123 + script.log('Writing "%s"...' % file)
124 + else:
125 + try:
126 + os.remove(filepath)
127 + except OSError, err:
128 + if err.errno != errno.ENOENT:
129 + script.fatal("Cannot remove '%s'!" % file)
130 + script.log('Removed "%s"...' % file)
131 + continue
132 try:
133 pagehtml = ''
134 - request.url = urlbase + pagename # add current pagename to url base
135 - page = Page.Page(request, pagename)
136 request.page = page
137 try:
138 request.reset()
139 @@ -206,7 +272,6 @@ General syntax: moin [options] export dump [dump-options]
140 traceback.print_exc(None, errlog)
141 finally:
142 timestamp = time.strftime("%Y-%m-%d %H:%M")
143 - filepath = os.path.join(outputdir, file)
144 fileout = codecs.open(filepath, 'w', config.charset)
145 fileout.write(page_template % {
146 'charset': config.charset,
147 --
148 1.8.2.1
Attached Files
To refer to attachments on a page, use attachment:filename, as shown below in the list of files. Do NOT use the URL of the [get] link, since this is subject to change and can break easily.You are not allowed to attach a file to this page.