Groups2009 code: WikiPages backend
This changes are in the mercurial queue and are not in the permanent changeset!
MoinMoin.wikidicts.Group and MoinMoin.wikidicts.GroupDict classes are moved to the MoinMoin.groups.backends.wiki_pages.
_tests are created for the backends module and appropriate test from MoinMoin._tests.test_wikidicts to test wiki_pages
Import changes to get GroupDict from the new module in /MoinMoin/events/wikidictsrescan.py and MoinMoin/web/contexts.py to pass tests
1 # HG changeset patch
2 # User Dmitrijs Milajevs <dimazest@gmail.com>
3 # Date 1243542660 -7200
4 # Node ID 0d6139c4e9917759fb6135eaa0887708a79bfe18
5 # Parent a8f09000b773566e2c21d29c1fb571c432671c2d
6 Groups 2009: WikiGroup backend
7
8 --- a/MoinMoin/events/wikidictsrescan.py
9 +++ b/MoinMoin/events/wikidictsrescan.py
10 @@ -12,7 +12,7 @@
11 logging = log.getLogger(__name__)
12
13 from MoinMoin import events as ev
14 -from MoinMoin import wikidicts
15 +from MoinMoin.groups.backends.wiki_pages import GroupDict
16
17 def handle(event):
18 # "changed" includes creation, deletion, renamed and copied
19 @@ -35,7 +35,7 @@
20
21 logging.debug("groupsdicts changed: %r, scan_dicts started", page.page_name)
22 del request.dicts
23 - gd = wikidicts.GroupDict(request)
24 + gd = GroupDict(request)
25 gd.scan_dicts()
26 logging.debug("groupsdicts changed: scan_dicts finished")
27
28 new file mode 100644
29 copy from MoinMoin/_tests/test_wikidicts.py
30 copy to MoinMoin/groups/backends/_tests/test_wiki_pages.py
31 --- a/MoinMoin/_tests/test_wikidicts.py
32 +++ b/MoinMoin/groups/backends/_tests/test_wiki_pages.py
33 @@ -11,6 +11,7 @@
34 import re
35 import shutil
36
37 +from MoinMoin.groups.backends.wiki_pages import Group
38 from MoinMoin import wikidicts
39 from MoinMoin import Page
40 from MoinMoin.PageEditor import PageEditor
41 @@ -68,7 +69,7 @@
42 assert self.getMembers(text) == ['take this']
43
44 def getMembers(self, text):
45 - group = wikidicts.Group(self.request, '')
46 + group = Group(self.request, '')
47 group.initFromText(text)
48 return group.members()
49
50 @@ -107,7 +108,7 @@
51 Assume that the SystemPagesGroup is in the data or the underlay dir.
52 """
53 assert Page.Page(self.request, 'SystemPagesGroup').exists(), "SystemPagesGroup is missing, Can't run test"
54 - systemPages = wikidicts.Group(self.request, 'SystemPagesGroup')
55 + systemPages = Group(self.request, 'SystemPagesGroup')
56 #print repr(systemPages)
57 #print repr(self.request.dicts['SystemPagesGroup'])
58 for member in systemPages.members():
59 @@ -126,7 +127,7 @@
60 become_trusted(request)
61 page = create_page(request, u'SomeGroup', u" * ExampleUser")
62 page.renamePage('AnotherGroup')
63 - group = wikidicts.Group(request, '')
64 + group = Group(request, '')
65 isgroup = request.cfg.cache.page_group_regexact.search
66 grouppages = request.rootpage.getPageList(user='', filter=isgroup)
67 result = request.dicts.has_member(u'AnotherGroup', u'ExampleUser')
68 @@ -142,7 +143,7 @@
69 become_trusted(request)
70 page = create_page(request, u'SomeGroup', u" * ExampleUser")
71 page.copyPage(u'OtherGroup')
72 - group = wikidicts.Group(request, '')
73 + group = Group(request, '')
74 isgroup = request.cfg.cache.page_group_regexact.search
75 grouppages = request.rootpage.getPageList(user='', filter=isgroup)
76 result = request.dicts.has_member(u'OtherGroup', u'ExampleUser')
77 @@ -227,5 +228,4 @@
78
79 assert result is True
80
81 -coverage_modules = ['MoinMoin.wikidicts']
82 -
83 +coverage_modules = ['MoinMoin.groups.backends.wiki_pages']
84 copy from MoinMoin/wikidicts.py
85 copy to MoinMoin/groups/backends/wiki_pages.py
86 --- a/MoinMoin/wikidicts.py
87 +++ b/MoinMoin/groups/backends/wiki_pages.py
88 @@ -6,63 +6,11 @@
89 2003 by Gustavo Niemeyer
90 @license: GNU GPL, see COPYING for details.
91 """
92 +
93 import re, time
94
95 -from MoinMoin import caching, Page
96 -
97 -# Version of the internal data structure which is pickled.
98 -# Please increment if you have changed the structure.
99 -DICTS_PICKLE_VERSION = 6
100 -
101 -
102 -class DictBase(dict):
103 - """ Base class for wiki dicts
104 -
105 - To use this class, subclass it and override regex and initFromText.
106 - """
107 - def __init__(self, request=None, pagename=None):
108 - dict.__init__(self)
109 - self.name = None
110 - if request is not None and pagename is not None:
111 - self.loadFromPage(request, pagename)
112 -
113 - # Regular expression used to parse text - subclass must override this
114 - regex = None # re.compile(u'...', re.MULTILINE | re.UNICODE)
115 -
116 - def loadFromPage(self, request, name):
117 - """ load the dict from wiki page <name>'s content """
118 - self.name = name
119 - text = Page.Page(request, name).get_raw_body()
120 - self.initFromText(text)
121 -
122 - def initFromText(self, text):
123 - """ parse the wiki page text and init the dict from it """
124 - raise NotImplementedError('subclasses should override this')
125 -
126 -
127 -class Dict(DictBase):
128 - """ Mapping of keys to values in a wiki page.
129 -
130 - How a Dict definition page should look like:
131 -
132 - any text ignored
133 - key1:: value1
134 - * ignored, too
135 - key2:: value2 containing spaces
136 - ...
137 - keyn:: ....
138 - any text ignored
139 - """
140 - # Key:: Value - ignore all but key:: value pairs, strip whitespace, exactly one space after the :: is required
141 - regex = re.compile(ur'^ (?P<key>.+?):: (?P<val>.*?) *$', re.MULTILINE | re.UNICODE)
142 -
143 - def initFromText(self, text):
144 - for match in self.regex.finditer(text):
145 - key, val = match.groups()
146 - self[key] = val
147 -
148 - def __repr__(self):
149 - return "<Dict name=%r items=%r>" % (self.name, self.items())
150 +from MoinMoin.wikidicts import DictBase, DictDict, DICTS_PICKLE_VERSION
151 +from MoinMoin import caching
152
153
154 class Group(DictBase):
155 @@ -121,70 +69,6 @@
156 return "<Group name=%r items=%r>" % (self.name, self._list)
157
158
159 -class DictDict:
160 - """ a dictionary of Dict objects
161 -
162 - Config:
163 - cfg.page_dict_regex
164 - Default: ".*Dict$" Defs$ Vars$ ???????????????????
165 - """
166 -
167 - def __init__(self):
168 - self.reset()
169 -
170 - def reset(self):
171 - self.dictdict = {}
172 - self.namespace_timestamp = 0
173 - self.pageupdate_timestamp = 0
174 - self.base_timestamp = 0
175 - self.picklever = DICTS_PICKLE_VERSION
176 -
177 - def has_key(self, dictname, key):
178 - """ check if we have key <key> in dict <dictname> """
179 - d = self.dictdict.get(dictname)
180 - return d and d.has_key(key)
181 -
182 - def keys(self, dictname):
183 - """ get keys of dict <dictname> """
184 - try:
185 - d = self.dictdict[dictname]
186 - except KeyError:
187 - return []
188 - return d.keys()
189 -
190 - def values(self, dictname):
191 - """ get values of dict <dictname> """
192 - try:
193 - d = self.dictdict[dictname]
194 - except KeyError:
195 - return []
196 - return d.values()
197 -
198 - def dict(self, dictname):
199 - """ get dict <dictname> """
200 - try:
201 - d = self.dictdict[dictname]
202 - except KeyError:
203 - return {}
204 - return d
205 -
206 - def adddict(self, request, dictname):
207 - """ add a new dict (will be read from the wiki page) """
208 - self.dictdict[dictname] = Dict(request, dictname)
209 -
210 - def has_dict(self, dictname):
211 - """ check if we have a dict <dictname> """
212 - return self.dictdict.has_key(dictname)
213 -
214 - def keydict(self, key):
215 - """ list all dicts that contain key """
216 - dictlist = []
217 - for d in self.dictdict.values():
218 - if d.has_key(key):
219 - dictlist.append(d.name)
220 - return dictlist
221 -
222 -
223 class GroupDict(DictDict):
224 """ a dictionary of Group objects
225
226 --- a/MoinMoin/web/contexts.py
227 +++ b/MoinMoin/web/contexts.py
228 @@ -371,8 +371,8 @@
229
230 def dicts(self):
231 """ Lazy initialize the dicts on the first access """
232 - from MoinMoin import wikidicts
233 - dicts = wikidicts.GroupDict(self)
234 + from MoinMoin.groups.backends.wiki_pages import GroupDict
235 + dicts = GroupDict(self)
236 dicts.load_dicts()
237 return dicts
238 dicts = EnvironProxy(dicts)
239 --- a/MoinMoin/wikidicts.py
240 +++ b/MoinMoin/wikidicts.py
241 @@ -65,62 +65,6 @@
242 return "<Dict name=%r items=%r>" % (self.name, self.items())
243
244
245 -class Group(DictBase):
246 - """ Group of users, of pages, of whatever.
247 -
248 - How a Group definition page should look like:
249 -
250 - any text ignored
251 - * member1
252 - * ignored, too
253 - * member2
254 - * ....
255 - * memberN
256 - any text ignored
257 -
258 - If there are any free links using [[free link]] notation, the markup
259 - is stripped from the member.
260 - """
261 - # * Member - ignore all but first level list items, strip whitespace, strip free links markup
262 - regex = re.compile(ur'^ \* +(?:\[\[)?(?P<member>.+?)(?:\]\])? *$', re.MULTILINE | re.UNICODE)
263 -
264 - def __init__(self, request=None, pagename=None):
265 - self._list = []
266 - DictBase.__init__(self, request, pagename)
267 -
268 - def initFromText(self, text):
269 - for match in self.regex.finditer(text):
270 - member = match.group('member')
271 - self.addmember(member)
272 -
273 - def update(self, members):
274 - self.addmembers(members.keys())
275 -
276 - def __iter__(self):
277 - return iter(self._list)
278 -
279 - def members(self):
280 - """ return the group's members """
281 - return self._list[:]
282 -
283 - def addmembers(self, members):
284 - """ add a list of members to the group """
285 - for m in members:
286 - self.addmember(m)
287 -
288 - def addmember(self, member):
289 - """ add a member to the group """
290 - self[member] = 1
291 - self._list.append(member)
292 -
293 - def has_member(self, member):
294 - """ check if the group has member <member> """
295 - return self.has_key(member)
296 -
297 - def __repr__(self):
298 - return "<Group name=%r items=%r>" % (self.name, self._list)
299 -
300 -
301 class DictDict:
302 """ a dictionary of Dict objects
303
304 @@ -183,177 +127,3 @@
305 if d.has_key(key):
306 dictlist.append(d.name)
307 return dictlist
308 -
309 -
310 -class GroupDict(DictDict):
311 - """ a dictionary of Group objects
312 -
313 - Config:
314 - cfg.page_group_regex
315 - Default: ".*Group$"
316 - """
317 -
318 - def __init__(self, request):
319 - self.cfg = request.cfg
320 - self.request = request
321 -
322 - def reset(self):
323 - self.dictdict = {}
324 - self.groupdict = {} # unexpanded groups
325 - self.picklever = DICTS_PICKLE_VERSION
326 - self.disk_cache_id = None
327 -
328 - def has_member(self, groupname, member):
329 - """ check if we have <member> as a member of group <groupname> """
330 - group = self.dictdict.get(groupname)
331 - return group and group.has_member(member)
332 -
333 - def members(self, groupname):
334 - """ get members of group <groupname> """
335 - try:
336 - group = self.dictdict[groupname]
337 - except KeyError:
338 - return []
339 - return group.members()
340 -
341 - def addgroup(self, request, groupname):
342 - """ add a new group (will be read from the wiki page) """
343 - grp = Group(request, groupname)
344 - self.groupdict[groupname] = grp
345 - self.expand_groups()
346 -
347 - def hasgroup(self, groupname):
348 - """ check if we have a dict <dictname> """
349 - return self.groupdict.has_key(groupname)
350 -
351 - def __getitem__(self, name):
352 - return self.groupdict[name]
353 -
354 - def membergroups(self, member):
355 - """ list all groups where member is a member of """
356 - grouplist = []
357 - for group in self.dictdict.values():
358 - if group.has_member(member):
359 - grouplist.append(group.name)
360 - return grouplist
361 -
362 - def expand_groups(self):
363 - """ copy expanded groups to self.dictdict """
364 - for name in self.groupdict:
365 - members, groups = self.expand_group(name)
366 - members.update(groups)
367 - grp = Group()
368 - grp.update(members)
369 - self.dictdict[name] = grp
370 -
371 - def expand_group(self, name):
372 - """ Recursively expand group <name>, using the groupdict (which is a not expanded
373 - dict of all group names -> group dicts). We return a flat list of group member
374 - names and group names.
375 -
376 - Given a groupdict (self) with two groups:
377 -
378 - MainGroup: [A, SubGroup]
379 - SubGroup: [B, C]
380 -
381 - MainGroup is expanded to:
382 -
383 - self.expand_group('MainGroup') -> [A, B, C], [MainGroup, SubGroup]
384 - """
385 - groups = {name: 1}
386 - members = {}
387 - groupmembers = self[name].keys()
388 - for member in groupmembers:
389 - # Skip duplicates
390 - if member in groups:
391 - continue
392 - # Add member and its children
393 - if self.hasgroup(member):
394 - new_members, new_groups = self.expand_group(member)
395 - groups.update(new_groups)
396 - members.update(new_members)
397 - else:
398 - members[member] = 1
399 - return members, groups
400 -
401 - def load_dicts(self):
402 - """ load the dict from the cache """
403 - request = self.request
404 - rescan = False
405 - arena = 'wikidicts'
406 - key = 'dicts_groups'
407 - cache = caching.CacheEntry(request, arena, key, scope='wiki', use_pickle=True)
408 - current_disk_cache_id = cache.uid()
409 - try:
410 - self.__dict__.update(self.cfg.cache.DICTS_DATA)
411 - if (current_disk_cache_id is None or
412 - current_disk_cache_id != self.disk_cache_id):
413 - self.reset()
414 - raise AttributeError # not fresh, force load from disk
415 - else:
416 - return
417 - except AttributeError:
418 - try:
419 - data = cache.content()
420 - self.__dict__.update(data)
421 - self.disk_cache_id = current_disk_cache_id
422 -
423 - # invalidate the cache if the pickle version changed
424 - if self.picklever != DICTS_PICKLE_VERSION:
425 - raise # force rescan
426 - except:
427 - self.reset()
428 - rescan = True
429 -
430 - if rescan:
431 - self.scan_dicts()
432 - self.load_dicts() # try again
433 - return
434 -
435 - data = {
436 - "disk_cache_id": self.disk_cache_id,
437 - "dictdict": self.dictdict,
438 - "groupdict": self.groupdict,
439 - "picklever": self.picklever
440 - }
441 -
442 - # remember it (persistent environments)
443 - self.cfg.cache.DICTS_DATA = data
444 -
445 - def scan_dicts(self):
446 - """ scan all pages matching the dict / group regex and
447 - cache the results on disk
448 - """
449 - request = self.request
450 - self.reset()
451 -
452 - # XXX get cache write lock here
453 - scan_begin_time = time.time()
454 -
455 - # Get all pages in the wiki - without user filtering using filter
456 - # function - this makes the page list about 10 times faster.
457 - isdict = self.cfg.cache.page_dict_regexact.search
458 - dictpages = request.rootpage.getPageList(user='', filter=isdict)
459 - for pagename in dictpages:
460 - self.adddict(request, pagename)
461 -
462 - isgroup = self.cfg.cache.page_group_regexact.search
463 - grouppages = request.rootpage.getPageList(user='', filter=isgroup)
464 - for pagename in grouppages:
465 - self.addgroup(request, pagename)
466 -
467 - scan_end_time = time.time()
468 - self.expand_groups()
469 -
470 - arena = 'wikidicts'
471 - key = 'dicts_groups'
472 - cache = caching.CacheEntry(request, arena, key, scope='wiki', use_pickle=True)
473 - data = {
474 - "scan_begin_time": scan_begin_time,
475 - "scan_end_time": scan_end_time,
476 - "dictdict": self.dictdict,
477 - "groupdict": self.groupdict,
478 - "picklever": self.picklever
479 - }
480 - cache.update(data)
481 - # XXX release cache write lock here
482