MoinMoin: attachment:subscribercache.patch of MoinMoinBugs/GetSubscribersSlow

   1 Description: Use a cache for storing and retrieving subscriber
   2  information to boost performance
   3 Origin: http://moinmo.in/MoinMoinBugs/GetSubscribersSlow
   4 Author: Vitaliy Shchupak, updated by Steve McIntyre to add locking
   5 Last-Update: 2012-04-29
   6 
   7 --- a/MoinMoin/Page.py	2012-04-29 19:17:26.113325108 +0100
   8 +++ b/MoinMoin/Page.py	2012-04-29 19:00:55.781245226 +0100
   9 @@ -811,7 +811,7 @@
  10  
  11          return link
  12  
  13 -    def getSubscribers(self, request, **kw):
  14 +    def getSubscribersUncached(self, request, **kw):  # original function renamed
  15          """ Get all subscribers of this page.
  16  
  17          @param request: the request object
  18 @@ -834,6 +834,8 @@
  19          else:
  20              from MoinMoin.security import Default as UserPerms
  21  
  22 +        request.clock.start("getSubscribersUncached")
  23 +
  24          # get email addresses of the all wiki user which have a profile stored;
  25          # add the address only if the user has subscribed to the page and
  26          # the user is not the current editor
  27 @@ -872,6 +874,113 @@
  28              else:
  29                  subscriber_list[lang].append(subscriber.email)
  30  
  31 +        request.clock.stop("getSubscribersUncached")
  32 +
  33 +        return subscriber_list
  34 +
  35 +    # cached version of getSubscribers
  36 +    def getSubscribers(self, request, **kw):
  37 +        """ Get all subscribers of this page.
  38 +
  39 +        @param request: the request object
  40 +        @keyword include_self: if 1, include current user (default: 0)
  41 +        @keyword return_users: if 1, return user instances (default: 0)
  42 +        @rtype: dict
  43 +        @return: lists of subscribed email addresses in a dict by language key
  44 +        """
  45 +        include_self = kw.get('include_self', self.include_self)
  46 +        return_users = kw.get('return_users', 0)
  47 +
  48 +        request.clock.start('getSubscribersCached')
  49 +        # extract categories of this page
  50 +        pageList = self.getCategories(request)
  51 +
  52 +        # add current page name for list matching
  53 +        pageList.append(self.page_name)
  54 +
  55 +        arena = 'user'
  56 +        key = 'page_sub'
  57 +
  58 +        # get or create cache file
  59 +        page_sub = {}
  60 +        cache = caching.CacheEntry(request, arena, key, scope='wiki', use_pickle=True)
  61 +        if cache.exists():
  62 +            page_sub = cache.content()
  63 +        else:
  64 +            #build a cache if it doesn't exist
  65 +            cache = caching.CacheEntry(request, arena, key, scope='wiki', use_pickle=True, do_locking=False)
  66 +            # lock to stop anybody else interfering with the data while we're working
  67 +            cache.lock('w')
  68 +            userlist = user.getUserList(request)
  69 +            for uid in userlist:
  70 +                subscriber = user.User(request, uid)
  71 +                # we don't care about storing entries for users without any page subscriptions
  72 +                if subscriber.subscribed_pages:
  73 +                    page_sub[subscriber.id] = {
  74 +                        'name': subscriber.name,
  75 +                        'email': subscriber.email,
  76 +                        'subscribed_pages': subscriber.subscribed_pages
  77 +                    }
  78 +            cache.update(page_sub)
  79 +            cache.unlock()
  80 +            cache.lock('r') # to go back to the same mode as if it had existed all along
  81 +
  82 +        if self.cfg.SecurityPolicy:
  83 +            UserPerms = self.cfg.SecurityPolicy
  84 +        else:
  85 +            from MoinMoin.security import Default as UserPerms
  86 +
  87 +        # get email addresses of the all wiki user which have a profile stored;
  88 +        # add the address only if the user has subscribed to the page and
  89 +        # the user is not the current editor
  90 +        userlist = page_sub
  91 +        subscriber_list = {}
  92 +
  93 +        pages = pageList[:]
  94 +        if request.cfg.interwikiname:
  95 +            pages += ["%s:%s" % (request.cfg.interwikiname, pagename) for pagename in pageList]
  96 +        # Create text for regular expression search
  97 +        text = '\n'.join(pages)
  98 +
  99 +        for uid in userlist.keys():
 100 +            if uid == request.user.id and not include_self:
 101 +                continue # no self notification
 102 +
 103 +            isSubscribed = False
 104 +
 105 +            # This is a bit wrong if return_users=1 (which implies that the caller will process
 106 +            # user attributes and may, for example choose to send an SMS)
 107 +            # So it _should_ be "not (subscriber.email and return_users)" but that breaks at the moment.
 108 +            if not userlist[uid]['email']:
 109 +                continue # skip empty email addresses
 110 +
 111 +            # now check patters for actual match
 112 +            for pattern in userlist[uid]['subscribed_pages']:
 113 +                if pattern in pages:
 114 +                    isSubscribed = True
 115 +                try:
 116 +                    pattern = re.compile(r'^%s$' % pattern, re.M)
 117 +                except re.error:
 118 +                    continue
 119 +                if pattern.search(text):
 120 +                    isSubscribed = True
 121 +
 122 +            # only if subscribed, then read user info from file
 123 +            if isSubscribed:
 124 +                subscriber = user.User(request, uid)
 125 +
 126 +                if not UserPerms(subscriber).read(self.page_name):
 127 +                    continue
 128 +
 129 +                lang = subscriber.language or request.cfg.language_default
 130 +                if not lang in subscriber_list:
 131 +                    subscriber_list[lang] = []
 132 +                if return_users:
 133 +                    subscriber_list[lang].append(subscriber)
 134 +                else:
 135 +                    subscriber_list[lang].append(subscriber.email)
 136 +
 137 +        request.clock.stop('getSubscribersCached')
 138          return subscriber_list
 139  
 140      def parse_processing_instructions(self):
 141 --- a/MoinMoin/user.py	2012-04-29 19:17:28.053323498 +0100
 142 +++ b/MoinMoin/user.py	2012-04-29 19:08:40.921246380 +0100
 143 @@ -618,6 +618,9 @@
 144              event = events.UserCreatedEvent(self._request, self)
 145              events.send_event(event)
 146  
 147 +        # update page subscriber's cache after saving user preferences
 148 +        self.updatePageSubCache()
 149 +
 150      # -----------------------------------------------------------------
 151      # Time and date formatting
 152  
 153 @@ -805,6 +808,36 @@
 154              self.save()
 155          return not self.isSubscribedTo([pagename])
 156  
 157 +    # update page subscribers cache
 158 +    def updatePageSubCache(self):
 159 +        """ When a user changes his preferences, we update the
 160 +        page subscriber's cache
 161 +        """
 162 +
 163 +        arena = 'user'
 164 +        key = 'page_sub'
 165 +
 166 +        page_sub = {}
 167 +        cache = caching.CacheEntry(self._request, arena, key, scope='wiki', use_pickle=True, do_locking=False)
 168 +        if not cache.exists():
 169 +            return  # if no cache file, just don't do anything
 170 +
 171 +        cache.lock('w')
 172 +        page_sub = cache.content()
 173 +
 174 +        # we don't care about storing entries for users without any page subscriptions
 175 +        if self.subscribed_pages:
 176 +            page_sub[self.id] = {
 177 +                'name': self.name,
 178 +                'email': self.email,
 179 +                'subscribed_pages': self.subscribed_pages
 180 +            }
 181 +        elif page_sub.get(self.id):
 182 +            del page_sub[self.id]
 183 +
 184 +        cache.update(page_sub)
 185 +        cache.unlock()
 186 +
 187      # -----------------------------------------------------------------
 188      # Quicklinks
 189