MoinMoin: attachment:mpl-1.0.py of ParserMarket/matplotlib

   1 # -*- coding: iso-8859-1 -*-
   2 """
   3     MoinMoin - Matplotlib integration
   4 
   5     example:   {{{#!mpl src=off,run=on,prefix=mpl,display=inline,klass=mpl,tablestyle="",rowstyle="",style="",persistence=True,debug=False
   6                 ...
   7                 ...
   8                 }}}
   9 
  10     keyword parameters:
  11 
  12     @param display: off|link|inline (default:inline)
  13     @param columns: arange images in table with No columns (default:1) 
  14                     if more than 1 image and display is inline
  15     @param src: on|off (display mpl source, or not, default: off)
  16     @param run: on|off (if off, execution is disabled, default:on)
  17 
  18     @prefix: prefix for files generated (default: mpl). If you have more than one
  19              image on a page which is not persistent, you should use a unique prefix
  20              for each chart.
  21 
  22     @params klass: class attribute for div
  23     @params tablestyle: style attribute for table
  24     @params rowstyle: style attribute for table row
  25     @params style: style attribute for table cell
  26 
  27     @params persistence: if True create delete.me.to.regenerate.images marker (default: True)
  28     @params debug: if True keep files after processing (default: False)
  29 
  30 
  31     Directives:
  32     -----------
  33 
  34     #! include(pagename)
  35     #! attach(pagename/*.xls)
  36     #! page(pagename)
  37     #! icaedoc(icaedoc-name)
  38 
  39     Mpl scripts can be made very modular using the '#! include(pagename)' directive.
  40     You can see this similar to the import statement in python. In the Mpl script the
  41     whole page content (#format python) is included. Use this to store your standard
  42     settings as reuse them for consistent plotting.
  43 
  44     example: #! include(MplSettings)
  45 
  46 
  47     Data sources:
  48     ============
  49 
  50     1) Page attachments
  51     *******************
  52 
  53     # --------------------------------
  54     #! attach(pagename/*.xls) 
  55     # --------------------------------
  56 
  57     The IN object contains all attached input files in the IN['files'] dictionary with the filenames as keys.
  58     If you want to make a page attachment available for the plotting process, 
  59     use the '#! attach(filename)' directive and reference the file in the open
  60     statement as shown in the example.
  61 
  62     example:
  63         #! attach('filename')
  64         files = IN['files']
  65         f = open(files['filename'])
  66 
  67     2) Page raw data
  68     ****************
  69 
  70     # --------------------------------
  71     #! page(pagename) 
  72     # --------------------------------
  73 
  74     The IN object contains the page raw data (text) in the IN['pages'] dictionary with the pagenames as keys
  75     and the page content (as obtained from page.get_raw_body()). Absolute page names are used as keys.
  76 
  77     example:
  78         #! page(PageWithData)
  79         pgData = IN['pages']['PageWithData']
  80 
  81     3) Form data
  82     ************
  83 
  84     The IN object also contains the form dictionary from the current request.
  85     example: value = IN['form'].get('parname',['0'])[0]
  86 
  87     4) iCAE document objects
  88     ************************
  89 
  90     # --------------------------------
  91     #! icaedoc(iCAE-doc-name)
  92     # --------------------------------
  93 
  94     You can add iCAE documents with the correct short path. The documents are made
  95     available in the IN['files'] dictionary in the same way in the '#! attach()' directive.
  96 
  97     example:
  98         #! icaedoc('P000000-MainEngineData.xls')
  99         files = IN['files']
 100         f = open(files['P000000-MainEngineData.xls'])
 101 
 102     The IN object contains also the current user name from the request in IN['user']
 103 
 104     @copyright: 2008 F. Zieher
 105     @license: GNU GPL, see COPYING for details.
 106 
 107 """
 108 
 109 import os, re
 110 import sys
 111 import exceptions
 112 import tempfile
 113 import sha
 114 import pickle
 115 
 116 from path import path
 117 
 118 from MoinMoin.Page import Page
 119 from MoinMoin import config, wikiutil
 120 from MoinMoin.action import AttachFile
 121 from MoinMoin import log
 122 
 123 # -------------------------------------------------------------------------------
 124 
 125 loggin = log.getLogger(__name__)
 126 
 127 # ---------------------------------------------------------------------- 
 128 
 129 # some config settings, XXX should be moved to the moin cfg object
 130 if 'win' in sys.platform:
 131     FURL = 'e:/home/zieherf/_ipython/security/ipcontroller-tc.furl'
 132 else:
 133     FURL = '/s330/moin/furls/ipcontroller-tc.furl'
 134 
 135 Dependencies = ['time']
 136 
 137 # ---------------------------------------------------------------------- 
 138 
 139 MPL_HELP = """
 140 # ------------------------------------------------------------------------------------
 141 # Predefined objects:
 142 #    mpl ... matplotlib object
 143 #    plt ... matplotlib.pyplot object
 144 #    np  ... numpy object
 145 #    IN  ... dictionary containing 'files' dictionary, 'pages' dictionary,
 146 #            'form' dictionary (copy of request.form) and 'user' name.
 147 #            - Access attached file with IN['files']['filename']
 148 #            - Accrss page data with IN['pages']['PageName']
 149 #            - Access form parameter with IN['form'].get(param,['default-value'])[0]
 150 #    mm  ... use mm.imgName(imgno=0) and mm.nextImg() in plt.savefig
 151 #            examples: plt.savefig(mm.imgName()), and for all consecutive
 152 #                      plt.savefig(mm.nextImg())
 153 # ------------------------------------------------------------------------------------
 154 """
 155 
 156 # ---------------------------------------------------------------------- 
 157 
 158 class SandBox:
 159     """Implement Sandbox for executing python code.
 160     Sandbox has inbox/outbox directories. inbox is where ipengine
 161     reads all data from. outbox is where ipengine creates
 162     output files (i.e. image files from matplotlib).
 163     """
 164 
 165     fmode = 0666
 166     dmode = 0777
 167 
 168     def __init__(self,request,sbpath='',prefix=''):
 169         """Instantiate sandbox
 170 
 171         @param sbpath: if given, path to sandbox, otherwise a temporary name will be created
 172         @param prefix: prefix to use when sandbox name is created automatically
 173         """
 174         self.request = request
 175         self.sb      = sbpath and path(sbpath) or path(tempfile.NamedTemporaryFile(prefix=prefix).name)
 176         self.inbox   = self.sb.joinpath('inbox')
 177         self.outbox  = self.sb.joinpath('outbox')
 178         self.create()
 179         
 180     def create(self):
 181         """Create sandbox
 182         """
 183         for d in (self.sb,self.inbox,self.outbox):
 184             d.mkdir()
 185             d.chmod(self.dmode)
 186 
 187     def emptyInbox(self):
 188         """Empty (delete) all files in inbox
 189         """
 190         for f in self.inbox.walkfiles():
 191             f.remove()
 192 
 193     def emptyOutbox(self):
 194         """Empty (delete) all files in outbox
 195         """
 196         for f in self.outbox.walkfiles():
 197             f.remove()
 198 
 199     def remove(self):
 200         """Remove sandbox
 201         """
 202         self.sb.rmtree()
 203 
 204     def sendTo(self,srcname,dstname=''):
 205         """Copy file srcname into the inbox. 
 206         If dstname is not given, srcname is used.
 207         """
 208         f = path(srcname)
 209         if f.isfile() and f.access(os.R_OK):
 210             dst = dstname and self.inbox.joinpath(dstname) or self.inbox.joinpath(f.basename())
 211             f.copy(dst)
 212             dst.chmod(self.fmode)
 213 
 214     @property
 215     def outboxFiles(self):
 216         """Return files in the outbox
 217         """
 218         return self.outbox.files()
 219 
 220     @property
 221     def inDict(self):
 222         """Input object sent to ipengine"""
 223         
 224         indict = dict(files={},form={},user="")
 225 
 226         # file objects from attach and icaedoc directives
 227         for f in self.inbox.files():
 228             indict['files'][f.basename()] = f.encode(config.charset).replace('\\','/')
 229 
 230         return indict
 231 
 232 # ---------------------------------------------------------------------- 
 233 
 234 class MplClient:
 235     """Client interface to ipengines.
 236 
 237     1) Create sandbox
 238     2) Copy all required input data into inbox
 239     3) Supply python script file to run --> copy into inbox as run.py
 240     4) Run run.py with TaskClient and pull output out (dictionary)
 241     5) Get png files from outbox and attach to page name
 242     6) Cleanup
 243     """
 244 
 245     furl = FURL
 246 
 247     def __init__(self,request,pagename,script,fmt='png',prefix='mpl',debug=False):
 248 
 249         self.request    = request
 250         self.pagename   = pagename
 251         self.fmt        = fmt
 252         self.prefix     = prefix
 253         self.debug      = debug
 254         self.attach_dir = path(AttachFile.getAttachDir(self.request,self.pagename))
 255         self.sb     = None
 256         self.script = ''
 257         self.inbox  = []
 258         self.pageData = {}
 259         self.imgdata = []
 260         self.logdata = None
 261         self.outdata = {}
 262 
 263         # base script   
 264         self.script = script
 265 
 266         # treat attachments referenced in script and create input list
 267         # XXX could be done more efficient (parse once :-)
 268         self.parseAttachments()
 269         self.parseIcaeDocs()
 270         self.parsePages()
 271         self.parseIncludes()
 272 
 273     @property
 274     def imgPrefix(self):
 275         raw = self.script
 276         for f in self.inbox:
 277             raw += str(f.stat().st_mtime)
 278         for pg,data in self.pageData.items():
 279             raw += data
 280         self.imgPrefix = self.prefix + "_" + sha.new(raw).hexdigest() + '_chart'
 281         return self.imgPrefix
 282 
 283     def addInput(self,infile):
 284         """Add infile to the set of input files.
 285         """
 286         f = path(infile)
 287         if f.isfile() and f.access(os.R_OK):
 288             self.inbox.append(f)
 289 
 290     def parseAttachments(self):
 291         """Collect input files from "#! attach(file-attachment-path)" in self.inbox
 292 
 293         The path can reference an attachment on a differenet page than the
 294         current. It supports relative path syntax, i.e. "../file.dat".
 295         The file part can include widcard character that are used for
 296         globing more than one file.
 297         
 298         Don't use quotation marks on the file-path.
 299         """
 300         rec = re.compile(r"""(#! *attach\()(?P<fname>[A-Za-z0-9 ._=+*/-]+)(\).*)""")
 301         for line in self.script.splitlines():
 302             m = rec.match(line)
 303             if m:
 304                 # find page and attach_dir where attachments are stored
 305                 aName = m.group('fname')
 306 
 307                 # get pageName and attchDir
 308                 fparts = aName.split('/')
 309                 pageName = '/'.join(fparts[0:-1])
 310 
 311                 if not pageName:
 312                     pageName = self.pagename
 313                 else:
 314                     if pageName.endswith('..'):
 315                         pageName += '/'
 316                     pageName = wikiutil.AbsPageName(self.pagename,pageName)
 317                 filePat  = fparts[-1]
 318 
 319                 attachDir = AttachFile.getAttachDir(self.request,pageName)
 320 
 321                 # attachment found, could be pattern
 322                 for fname in path(attachDir).files(filePat):
 323                     self.inbox.append(fname)
 324 
 325     def parseIcaeDocs(self):
 326         """Collect input files from "#! icaedoc(file-attachment-path)" in self.inbox
 327         The path must be a conform icae document path, i.e. having the short path
 328         prepended. Don't use quotation marks on the file-path.
 329         example:
 330             #! icaedoc(P000000_4010-benchmark.xls) 
 331         """
 332         from icaebase import getPathFromSPath
 333 
 334         rec = re.compile(r"""(#! *icaedoc\()(?P<fname>[A-Za-z0-9 ._=+*/-]+)(\).*)""")
 335         for line in self.script.splitlines():
 336             m = rec.match(line)
 337             if m:
 338                 fname = m.group('fname')
 339                 try:
 340                     spath,doc = fname.split('-')
 341                 except:
 342                     continue
 343 
 344                 fname = getPathFromSPath(spath).joinpath('0-documentation',fname)
 345                 if fname.exists():
 346                     # attachment found, could be pattern
 347                     self.inbox.append(fname)
 348 
 349     def parsePages(self):
 350         """Read data from #! page(pagename) and store text data in self.pages
 351         """
 352         rec = re.compile(r"""(#! *page\()(?P<pname>[A-Za-z0-9 ._=+*/-]+)(\).*)""")
 353         script = self.script.splitlines()
 354 
 355         for lno,line in enumerate(script):
 356             m = rec.match(line)
 357             if m:
 358                 # page directive detected
 359                 pageName = m.group('pname')
 360                 if pageName.endswith('..'):
 361                     pageName += '/'
 362                 pageName  = wikiutil.AbsPageName(self.pagename,pageName)
 363                 self.pageData[pageName] = Page(self.request, pageName).get_raw_body()
 364 
 365     def parseIncludes(self):
 366         """Replace #! include(pagename) with the python src from page.
 367         """
 368         rec = re.compile(r"""(#! *include\()(?P<pname>[A-Za-z0-9 ._=+*/-]+)(\).*)""")
 369         script = self.script.splitlines()
 370 
 371         for lno,line in enumerate(script):
 372             m = rec.match(line)
 373             if m:
 374                 # include detected
 375                 pageName = m.group('pname')
 376                 if pageName.endswith('..'):
 377                     pageName += '/'
 378                 pageName  = wikiutil.AbsPageName(self.pagename,pageName)
 379                 pageTxt = Page(self.request, pageName).get_raw_body()
 380                 pageTxt = '\n'.join(pageTxt.splitlines()[1:])
 381                 script[lno] = pageTxt 
 382 
 383         self.script = '\n'.join(script)
 384 
 385 # ------------------- assyncclient ----------------------------------------------------------
 386 
 387     def run(self):
 388         """Execute matplotlib script by ipengine
 389         """
 390 
 391         # make sandbox
 392         self.sb = SandBox(self.request,prefix='mpl_')  
 393 
 394         # send files (data files and python files)
 395         for f in self.inbox:
 396             self.sb.sendTo(f)
 397 
 398         # -------------------------------------------------------------
 399         # run StringTask on ipengine / most cool feature :-)
 400         # -------------------------------------------------------------
 401 
 402         # IN object 
 403         inDict = self.sb.inDict.copy()
 404         inDict['pages']   = self.pageData
 405         inDict['form']    = self.request.form.copy()
 406         inDict['user']    = self.request.user.name
 407         inDict['imgBase'] = str(self.sb.outbox.joinpath('mplplot')).replace('\\','/')
 408         inDict['fmt']     = self.fmt
 409 
 410         # execute script on ipengine and obtain output (pullObjs)
 411         script   = str(MplScript(self.script).mplScript)
 412         pushObjs = dict(IN=inDict)
 413         pullObjs = ('OUT',)
 414 
 415         # the tricky part of handling the execution on the client
 416         # this is valid when using twisted as web framework in moin
 417         if hasattr(self.request,'reactor'):
 418             from IPython.kernel import asyncclient
 419             from twisted.internet.threads import blockingCallFromThread
 420             tc  = blockingCallFromThread(self.request.reactor,asyncclient.get_task_client,self.furl)
 421             tc  = tc.adapt_to_blocking_client()
 422             st  = asyncclient.StringTask(script,clear_before=True,pull=pullObjs,push=pushObjs)
 423         else:
 424             from IPython.kernel import client
 425             tc  = client.TaskClient(self.furl)
 426             st  = client.StringTask(script,clear_before=True,pull=pullObjs,push=pushObjs)
 427 
 428         tid = tc.run(st)
 429         res = tc.get_task_result(tid,block=True)
 430 
 431         if res.failure:
 432             self.failure = res.failure
 433             self.logdata = res.failure.getTraceback()
 434 
 435         if res.results.get('OUT',{}):
 436             self.outdata = res.results.get('OUT',{})
 437 
 438         # -------------------------------------------------------------
 439 
 440         # read image data
 441         self.imgdata = []
 442         for no,imgfile in enumerate(self.sb.outbox.files('mplplot-*.%s'%self.fmt)):
 443             self.imgdata.append(file(imgfile,'rb').read())
 444 
 445         # finally cleanup
 446         self.cleanup()
 447 
 448     def cleanup(self):
 449         """Cleanup sandbox
 450         """
 451         if not self.debug:
 452             self.sb.remove()
 453 
 454 
 455 # ---------------------------------------------------------------------- 
 456 
 457 class MplScript:
 458 
 459     pre = """
 460 # --- PRE CODE -------------------------------------------
 461 import matplotlib as mpl
 462 mpl.use('Agg')
 463 import matplotlib.pyplot as plt
 464 import numpy as np
 465 
 466 # reset mpl default settings
 467 mpl.rcdefaults()
 468 
 469 class MoinMpl:
 470     def __init__(self,base,fmt='png'):
 471         self.base = base
 472         self.imgno  = 0
 473         self.fmt    = fmt
 474     def imgName(self,imgno=0):
 475         self.imgno = int(imgno)
 476         return '%s-%d.%s'% (self.base,self.imgno,self.fmt)
 477     def nextImg(self):
 478         return self.imgName(self.imgno+1)
 479 # ------------------------------------------
 480 
 481 mm = MoinMpl(IN['imgBase'],IN['fmt'])
 482 OUT = {} # output will be handed to caller
 483 # --- END PRE CODE ---------------------------------------
 484 
 485 """
 486     post = """
 487 
 488 # --- POST CODE ------------------------------------------
 489 plt.close()
 490 # --- END POST CODE --------------------------------------
 491 """
 492 
 493     def __init__(self,pyscript=''):
 494         self.script = pyscript
 495 
 496     @property
 497     def mplScript(self):
 498         return self.pre+self.script+self.post
 499 
 500 # ---------------------------------------------------------------------- 
 501 
 502 def mpl_settings(run='on',src='off',mxsrc=9999,fmt='png',display='inline',columns=1,prefix='mpl',
 503                  klass="mpl",tablestyle="",rowstyle="",style="",persistence=True,debug=False):
 504     """
 505     Initialize default parameters.
 506     """
 507     return locals()
 508 
 509 # ---------------------------------------------------------------------- 
 510 
 511 class Parser:
 512     """
 513         Sends plot images generated by matplotlib
 514     """
 515     
 516     extensions = []
 517     Dependencies = Dependencies
 518 
 519     def __init__(self, raw, request, **kw):
 520         self.raw = raw
 521         self.request = request
 522 
 523         args = kw.get('format_args', '')
 524         # we use a macro definition to initialize the default init parameters
 525         # if a user enters a wrong parameter the failure is shown by the exception
 526         try:
 527             pyplot = mpl_settings()
 528             for k,v in pyplot.iteritems():
 529                 setattr(self,k,v)
 530             settings = wikiutil.invoke_extension_function(request, mpl_settings, args)
 531             for k, v in settings.iteritems():
 532                 if v != None: 
 533                     pyplot[k]=v
 534                     setattr(self,k,v)
 535 
 536         except ValueError, err:
 537             msg = u"matplotlib: %s" % err.args[0]
 538             request.write(self.request.formatter.text(msg))
 539 
 540     def format(self, formatter):
 541         """ Send the text. """
 542 
 543         self.request.flush() # to identify error text
 544         self.formatter = formatter
 545 
 546         if self.src.lower() in ('on','1','true'):
 547             # use highlight parser to show python code
 548             from highlight import Parser as HLParser
 549             hlp = HLParser(MPL_HELP+self.raw,self.request,format_args='python')
 550             hlp.format(formatter)
 551 
 552         if self.run.lower() in ('off','0','false'):
 553             self.request.write(formatter.preformatted(1)+'Execution disabled, set run=on.'+formatter.preformatted(0))
 554             return
 555 
 556         self.pagename = formatter.page.page_name
 557         self.attach_dir=AttachFile.getAttachDir(self.request,self.pagename,create=1)
 558 
 559         # -------------------- 
 560 
 561         # mplclient is created here, need to include data files in hexdigest
 562         self.mpl = MplClient(self.request,self.pagename,self.raw,self.fmt,prefix=self.prefix,debug=self.debug)
 563         update, charts = self._updateImgs(formatter)
 564 
 565         if update:
 566 
 567             self.mpl.run()
 568 
 569             if self.mpl.logdata:
 570                 # error occured
 571                 self.request.write(formatter.preformatted(1)+self.mpl.logdata+formatter.preformatted(0))
 572                 return
 573 
 574             # attach generated image(s)
 575             for no,imgdata in enumerate(self.mpl.imgdata):
 576                 imgName = "%s-%d.%s" % (self.mpl.imgPrefix,no,self.fmt)
 577                 attached_file = file(self.attach_dir + "/" + imgName, 'wb')
 578                 attached_file.write(imgdata)
 579                 attached_file.close()
 580                 charts.append(imgName)
 581 
 582         self.renderCharts(charts)
 583 
 584     def renderCharts(self,charts):
 585         """Render charts according to settings
 586         """
 587     
 588         if self.display.lower() in ('off','0','false') or not charts:
 589             return
 590 
 591         fmt = self.formatter
 592 
 593         html = []
 594         html.append(fmt.div(1,attr={'class':self.klass}))
 595 
 596         if self.display.lower() == 'link':
 597             # link display
 598             html.append(fmt.bullet_list(1))
 599             for chart in charts:
 600                 url = AttachFile.getAttachUrl(self.pagename, chart, self.request)
 601                 html.append(fmt.listitem(1))
 602                 html.append(fmt.url(1,url)+chart+fmt.url(0))
 603                 html.append(fmt.listitem(0))
 604             html.append(fmt.bullet_list(0))
 605         
 606         else:
 607             # inline display in table form
 608             noImgs = len(charts)
 609             rows  = noImgs / self.columns
 610             rows += noImgs%self.columns and 1 or 0
 611 
 612             T = fmt.table
 613             R = fmt.table_row
 614             C = fmt.table_cell
 615 
 616             html.append(T(1,style=self.tablestyle))
 617             id = 0
 618             for row in range(rows):
 619                 html.append(R(1,style=self.rowstyle))
 620                 for col in range(self.columns):
 621                     chart = charts[id]
 622                     url = AttachFile.getAttachUrl(self.pagename, chart, self.request)
 623                     html.append(C(1,style=self.style)+fmt.url(1,url)+fmt.image(src="%s" % url, alt=chart)+fmt.url(0)+C(0))
 624                     id += 1
 625                     if id >= noImgs:
 626                         break
 627                 html.append(R(0))
 628                 if id >= noImgs:
 629                     break
 630             html.append(T(0))
 631 
 632         html.append(fmt.div(0))
 633         self.request.write('\n'.join(html))
 634 
 635     def _removeChart(self,chart):
 636         """Remove chart attachment from page and from xapian index.
 637         @param chart: chart name (without path)
 638         """
 639         fpath = os.path.join(self.attach_dir, chart).encode(config.charset)
 640         os.remove(fpath)
 641         if self.request.cfg.xapian_search:
 642             from MoinMoin.search.Xapian import Index
 643             index = Index(self.request)
 644             if index.exists:
 645                 index.remove_item(self.pagename, chart)
 646 
 647     def _updateImgs(self, formatter):
 648         """Delete outdated charts
 649         @param formatter: formatter object
 650         """
 651         imgPrefix = self.mpl.imgPrefix
 652 
 653         # use delete.me.to.regenerate.images trick from dot.py
 654         dm2ri = self.attach_dir + '/' + "%s.delete.me.to.regenerate.images"%self.prefix
 655         charts = []
 656 
 657         updateImgs = True
 658         if not self.persistence and path(dm2ri).exists():
 659             path(dm2ri).remove()
 660 
 661         deleteImgs = not path(dm2ri).exists()
 662 
 663         # delete.me. ... exists, if with pref exists, then continue and do not 
 664         # rerun mpl execution
 665         attach_files = AttachFile._get_files(self.request, self.pagename)
 666         reChart  = re.compile(r"%s_.*_chart-[0-9]+.%s"%(self.prefix,self.fmt))
 667         for chart in attach_files:
 668             if reChart.match(chart) and deleteImgs:
 669                 self._removeChart(chart)
 670 
 671             if not deleteImgs and imgPrefix in chart:
 672                 charts.append(chart)
 673                 updateImgs = False
 674 
 675         # create persistence marker if not existing and persistence requested
 676         if deleteImgs and self.persistence:
 677             open(dm2ri,'w').close()
 678 
 679         return updateImgs,charts