Attachment '0001-Implement-an-incremental-dump-process-8.patch'
Download 1 From a91f66a199f16f7b4f15ab03c88fcde6fe985dbb Mon Sep 17 00:00:00 2001
2 From: Paul Wise <pabs3@bonedaddy.net>
3 Date: Wed, 1 Jan 2014 19:30:20 +0800
4 Subject: [PATCH] Implement an incremental dump process.
5
6 This also fixes dumping of the attachments.
7
8 This also allows the dump script to be interrupted.
9 ---
10 MoinMoin/action/AttachFile.py | 9 +--
11 MoinMoin/script/export/dump.py | 133 +++++++++++++++++++++++++++++++++++------
12 2 files changed, 120 insertions(+), 22 deletions(-)
13
14 diff --git a/MoinMoin/action/AttachFile.py b/MoinMoin/action/AttachFile.py
15 index 9081c3a..0bfc865 100644
16 --- a/MoinMoin/action/AttachFile.py
17 +++ b/MoinMoin/action/AttachFile.py
18 @@ -310,7 +310,7 @@ def _access_file(pagename, request):
19 return (pagename, None, None)
20
21
22 -def _build_filelist(request, pagename, showheader, readonly, mime_type='*', filterfn=None):
23 +def _build_filelist(request, pagename, showheader, readonly, mime_type='*', filterfn=None, downloadonly=False):
24 _ = request.getText
25 fmt = request.html_formatter
26
27 @@ -372,9 +372,10 @@ def _build_filelist(request, pagename, showheader, readonly, mime_type='*', filt
28 fmt.text(label_get) +
29 fmt.url(0))
30
31 - links.append(fmt.url(1, getAttachUrl(pagename, file, request, do='view')) +
32 - fmt.text(label_view) +
33 - fmt.url(0))
34 + if may_read and not downloadonly:
35 + links.append(fmt.url(1, getAttachUrl(pagename, file, request, do='view')) +
36 + fmt.text(label_view) +
37 + fmt.url(0))
38
39 if may_write and not readonly:
40 edit_url = getAttachUrl(pagename, file, request, do='modify')
41 diff --git a/MoinMoin/script/export/dump.py b/MoinMoin/script/export/dump.py
42 index d770e5b..1e9c9c3 100644
43 --- a/MoinMoin/script/export/dump.py
44 +++ b/MoinMoin/script/export/dump.py
45 @@ -3,7 +3,8 @@
46 MoinMoin - Dump a MoinMoin wiki to static pages
47
48 @copyright: 2002-2004 Juergen Hermann <jh@web.de>,
49 - 2005-2006 MoinMoin:ThomasWaldmann
50 + 2005-2006 MoinMoin:ThomasWaldmann,
51 + 2013 Paul Wise <pabs3@bonedaddy.net>
52 @license: GNU GPL, see COPYING for details.
53 """
54
55 @@ -12,11 +13,16 @@ import sys, os, time, codecs, shutil, re, errno
56 from MoinMoin import config, wikiutil, Page, user
57 from MoinMoin import script
58 from MoinMoin.action import AttachFile
59 +from MoinMoin.logfile import editlog
60 +from MoinMoin.util.filesys import touch
61
62 url_prefix_static = "."
63 logo_html = '<img src="logo.png">'
64 HTML_SUFFIX = ".html"
65
66 +timestamp_text = u"""This is a MoinMoin timestamp file.
67 +Please delete it to rebuild all pages.
68 +"""
69 page_template = u'''<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
70 <html>
71 <head>
72 @@ -60,14 +66,19 @@ td.noborder {
73 <div id="page">
74 %(pagehtml)s
75 </div>
76 +<div id="attachments">
77 +%(attachments_html)s
78 +</div>
79 <hr>
80 %(timestamp)s
81 </body>
82 </html>
83 '''
84
85 +def _attachment_fn(outputdir, pagename, filename=''):
86 + return os.path.join(outputdir, "attachments", wikiutil.quoteWikinameFS(pagename), filename.encode(config.charset))
87
88 -def _attachment(request, pagename, filename, outputdir, **kw):
89 +def _attachment(request, pagename, filename, outputdir, copy=False, **kw):
90 filename = filename.encode(config.charset)
91 source_dir = AttachFile.getAttachDir(request, pagename)
92 source_file = os.path.join(source_dir, filename)
93 @@ -75,16 +86,17 @@ def _attachment(request, pagename, filename, outputdir, **kw):
94 dest_file = os.path.join(dest_dir, filename)
95 dest_url = "attachments/%s/%s" % (wikiutil.quoteWikinameFS(pagename), wikiutil.url_quote(filename))
96 if os.access(source_file, os.R_OK):
97 - if not os.access(dest_dir, os.F_OK):
98 - try:
99 - os.makedirs(dest_dir)
100 - except:
101 - script.fatal("Cannot create attachment directory '%s'" % dest_dir)
102 - elif not os.path.isdir(dest_dir):
103 - script.fatal("'%s' is not a directory" % dest_dir)
104 + if copy:
105 + if not os.access(dest_dir, os.F_OK):
106 + try:
107 + os.makedirs(dest_dir)
108 + except:
109 + script.fatal("Cannot create attachment directory '%s'" % dest_dir)
110 + elif not os.path.isdir(dest_dir):
111 + script.fatal("'%s' is not a directory" % dest_dir)
112
113 - shutil.copyfile(source_file, dest_file)
114 - script.log('Writing "%s"...' % dest_url)
115 + script.log('Writing "%s" attachment "%s"...' % (pagename, filename))
116 + shutil.copyfile(source_file, dest_file)
117 return dest_url
118 else:
119 return ""
120 @@ -157,8 +169,50 @@ General syntax: moin [options] export dump [dump-options]
121 # use this user for permissions checks
122 request.user = user.User(request, name=self.options.dump_user)
123
124 - pages = request.rootpage.getPageList(user='') # get list of all pages in wiki
125 + pages = request.rootpage.getPageList(user='', exists=0) # get list of all pages in wiki
126 pages.sort()
127 +
128 + # Check the last update timestamp
129 + timestamp_file = os.path.join(outputdir, 'moin-last-update')
130 + try:
131 + timestamp_value = os.stat(timestamp_file).st_mtime
132 + except OSError, err:
133 + timestamp_value = 0
134 + if err.errno != errno.ENOENT:
135 + script.fatal("Cannot check last update time of '%s' (%s)!" % (timestamp_file, str(err)))
136 +
137 + # Create a new timestamp to use if successful
138 + new_timestamp_file = timestamp_file + '.new'
139 + with open(new_timestamp_file, 'w') as f:
140 + f.write(timestamp_text)
141 +
142 + # Get a list of pages that need actions
143 + attachments = dict()
144 + if timestamp_value:
145 + pages = set()
146 + version = wikiutil.timestamp2version(timestamp_value)
147 + log = editlog.EditLog(request)
148 + for line in log:
149 + if line.ed_time_usecs < version:
150 + continue
151 + elif line.action in ('ATTNEW', 'ATTDEL'):
152 + if line.pagename not in attachments:
153 + attachments[line.pagename] = {}
154 + attachments[line.pagename][line.extra] = line.action
155 + elif line.action == 'SAVE/RENAME':
156 + attachment_from = _attachment_fn(outputdir, line.extra)
157 + attachment_to = _attachment_fn(outputdir, line.pagename)
158 + try:
159 + os.rename(attachment_from, attachment_to)
160 + except OSError, err:
161 + if err.errno != errno.ENOENT:
162 + script.fatal('Cannot move attachments from "%s" to "%s" (%s)!' % (line.extra, line.pagename, str(err)))
163 + else:
164 + script.log('Moving attachments from "%s" to "%s"' % (line.extra, line.pagename))
165 + pages.add(line.extra)
166 + pages.add(line.pagename)
167 + pages = list(pages)
168 +
169 if self.options.page: # did user request a particular page or group of pages?
170 try:
171 namematch = re.compile(self.options.page)
172 @@ -186,32 +240,72 @@ General syntax: moin [options] export dump [dump-options]
173
174 urlbase = request.url # save wiki base url
175 for pagename in pages:
176 + # Process attachments for this page
177 + copy_attachments = []
178 + delete_attachments = []
179 + if pagename in attachments:
180 + for filename, action in attachments[pagename].items():
181 + if action == 'ATTNEW':
182 + copy_attachments.append(filename)
183 + elif action == 'ATTDEL':
184 + delete_attachments.append(filename)
185 + elif not timestamp_value:
186 + copy_attachments = AttachFile._get_files(request, pagename)
187 + for filename in copy_attachments:
188 + _attachment(request, pagename, filename, outputdir, copy=True)
189 + for filename in delete_attachments:
190 + try:
191 + os.remove(_attachment_fn(outputdir, pagename, filename))
192 + except OSError, err:
193 + if err.errno != errno.ENOENT:
194 + script.fatal('Cannot remove "%s" attachment "%s" (%s)!' % (pagename, filename, str(err)))
195 + else:
196 + script.log('Removed "%s" attachment "%s"...' % (pagename, filename))
197 +
198 # we have the same name in URL and FS
199 file = wikiutil.quoteWikinameURL(pagename)
200 - script.log('Writing "%s"...' % file)
201 + filepath = os.path.join(outputdir, file)
202 + exists = os.path.exists(filepath)
203 + request.url = urlbase + pagename # add current pagename to url base
204 + page = Page.Page(request, pagename)
205 + missing = not page.exists()
206 + unreadable = not request.user.may.read(pagename)
207 + if missing or unreadable:
208 + try:
209 + os.remove(filepath)
210 + except OSError, err:
211 + if err.errno != errno.ENOENT:
212 + script.fatal("Cannot remove '%s' (%s)!" % (file, str(err)))
213 + else:
214 + script.log('Removed "%s"...' % pagename)
215 + continue
216 try:
217 + script.log('Writing "%s"...' % pagename)
218 pagehtml = ''
219 - request.url = urlbase + pagename # add current pagename to url base
220 - page = Page.Page(request, pagename)
221 request.page = page
222 try:
223 request.reset()
224 pagehtml = request.redirectedOutput(page.send_page, count_hit=0, content_only=1)
225 - except:
226 + attachments_html = AttachFile._build_filelist(request, pagename, 0, 1, downloadonly=True)
227 + if attachments_html: attachments_html = '<h2>Attached Files</h2>' + attachments_html
228 + except Exception:
229 errcnt = errcnt + 1
230 print >> sys.stderr, "*** Caught exception while writing page!"
231 print >> errlog, "~" * 78
232 - print >> errlog, file # page filename
233 + print >> errlog, pagename
234 import traceback
235 traceback.print_exc(None, errlog)
236 + except:
237 + os.remove(new_timestamp_file)
238 + raise
239 finally:
240 timestamp = time.strftime("%Y-%m-%d %H:%M")
241 - filepath = os.path.join(outputdir, file)
242 fileout = codecs.open(filepath, 'w', config.charset)
243 fileout.write(page_template % {
244 'charset': config.charset,
245 'pagename': pagename,
246 'pagehtml': pagehtml,
247 + 'attachments_html': attachments_html,
248 'logo_html': logo_html,
249 'navibar_html': navibar_html,
250 'timestamp': timestamp,
251 @@ -231,4 +325,7 @@ General syntax: moin [options] export dump [dump-options]
252 errlog.close()
253 if errcnt:
254 print >> sys.stderr, "*** %d error(s) occurred, see '%s'!" % (errcnt, errfile)
255 + os.remove(new_timestamp_file)
256 + else:
257 + os.rename(new_timestamp_file, timestamp_file)
258
259 --
260 1.8.5.2
Attached Files
To refer to attachments on a page, use attachment:filename, as shown below in the list of files. Do NOT use the URL of the [get] link, since this is subject to change and can break easily.You are not allowed to attach a file to this page.