MoinMoin: attachment:mpl.py of ParserMarket/matplotlib

   1 # -*- coding: iso-8859-1 -*-
   2 """
   3     MoinMoin - Matplotlib integration
   4 
   5     example:   {{{#!mpl src=off, run=on, display=inline, klass=mpl, tablestyle="", rowstyle="",style=""
   6                 ...
   7                 ...
   8                 }}}
   9 
  10 
  11     IN object contains all input files with filenames as key.
  12     If you use a file, use the '#! attach(filename)' command and
  13     reference it in the open statement with 
  14     f = open(IN['filename'])
  15 
  16     keyword parameters:
  17 
  18     @param display: off|link|inline (default:inline)
  19     @param columns: arange images in table with No columns (default:1) 
  20                     if more than 1 image and display is inline
  21     @param src: on|off (display mpl source, or not, default: off)
  22     @param run: on|off (if off, execution is disabled, default:on)
  23 
  24     @params klass: class attribute for div
  25     @params tablestyle: style attribute for table
  26     @params rowstyle: style attribute for table row
  27     @params style: style attribute for table cell
  28 
  29     @copyright: 2008 F. Zieher
  30     @license: GNU GPL, see COPYING for details.
  31 """
  32 
  33 import os, re
  34 import sys
  35 import exceptions
  36 import tempfile
  37 import subprocess
  38 import sha
  39 import pickle
  40 
  41 from path import path
  42 
  43 from MoinMoin.Page import Page
  44 from MoinMoin import config, wikiutil
  45 from MoinMoin.action import AttachFile
  46 from MoinMoin import log
  47 loggin = log.getLogger(__name__)
  48 
  49 # ---------------------------------------------------------------------- 
  50 
  51 # some config settings
  52 if 'win' in sys.platform:
  53     FURL = 'e:/home/zieherf/_ipython/security/ipcontroller-tc.furl'
  54     PYTHONCMD = 'd:/Python25/python.exe'
  55 else:
  56     FURL = '/s330/moin/furls/ipcontroller-tc.furl'
  57     PYTHONCMD = 'python'
  58 
  59 Dependencies = ['time']
  60 
  61 # ---------------------------------------------------------------------- 
  62 
  63 _mplCode = """
  64 # -*- coding: iso-8859-1 -*-
  65 from IPython.kernel import client
  66 import pickle
  67 
  68 mmPref = "%(prefix)s"
  69 mmFmt  = "%(fmt)s"
  70 inobj  = %(inobj)s
  71 
  72 tc     = client.TaskClient('%(furl)s')
  73 script = '''
  74 %(script)s
  75 '''
  76 
  77 # execute script on ipengine and obtain output (pullObjs)
  78 pushObjs = dict(IN=inobj,mmPref=mmPref,mmFmt=mmFmt)
  79 pullObjs = ('out',)
  80 st = client.StringTask(script,clear_before=True,pull=pullObjs,push=pushObjs)
  81 res = tc.run(st,block=True)
  82 
  83 if res.failure:
  84     f = open('%(log)s','w')
  85     f.write(res.failure.getTraceback())
  86     f.close()
  87 else:
  88     f = open('%(out)s','wb')
  89     pickle.dump(res.results.get('out',{}),f)
  90     f.close()
  91 """
  92 
  93 MPL_HELP = """
  94 # ------------------------------------------------------------------------------------
  95 # Predefined objects:
  96 #    mpl ... matplotlib object
  97 #    plt ... matplotlib.pyplot object
  98 #    np  ... numpy object
  99 #    IN  ... dictionary of absolute paths of input files keyed by filename
 100 #    mm  ... use mm.imgName(imgno=0) and mm.nextImg() in plt.savefig
 101 #            examples: plt.savefig(mm.imgName(1)), and for all consecutive
 102 #                      plt.savefig(img.nextImg())
 103 # ------------------------------------------------------------------------------------
 104 """
 105 
 106 # ---------------------------------------------------------------------- 
 107 
 108 class SandBox:
 109     """Implement Sandbox for executing python code.
 110     Sandbox has inbox/outbox directories. inbox is where ipengine
 111     reads all data from. outbox is where ipengine creates
 112     output files (i.e. image files from matplotlib).
 113     """
 114 
 115     fmode = 0666
 116     dmode = 0777
 117 
 118     def __init__(self,sbpath='',prefix=''):
 119         """Instantiate sandbox
 120 
 121         @param sbpath: if given, path to sandbox, otherwise a temporary name will be created
 122         @param prefix: prefix to use when snadbox name is created automatically
 123         """
 124         self.sb     = sbpath and path(sbpath) or path(tempfile.NamedTemporaryFile(prefix=prefix).name)
 125         self.inbox  = self.sb.joinpath('inbox')
 126         self.outbox = self.sb.joinpath('outbox')
 127         self.create()
 128         
 129     def create(self):
 130         """Create sandbox
 131         """
 132         for d in (self.sb,self.inbox,self.outbox):
 133             d.mkdir()
 134             d.chmod(self.dmode)
 135 
 136     def emptyInbox(self):
 137         """Empty (delete) all files in inbox
 138         """
 139         for f in self.inbox.walkfiles():
 140             f.remove()
 141 
 142     def emptyOutbox(self):
 143         """Empty (delete) all files in outbox
 144         """
 145         for f in self.outbox.walkfiles():
 146             f.remove()
 147 
 148     def remove(self):
 149         """Remove sandbox
 150         """
 151         self.sb.rmtree()
 152 
 153     def sendTo(self,srcname,dstname=''):
 154         """Copy file srcname into the inbox. 
 155         If dstname is not given, srcname is used.
 156         """
 157         f = path(srcname)
 158         if f.isfile() and f.access(os.R_OK):
 159             dst = dstname and self.inbox.joinpath(dstname) or self.inbox.joinpath(f.basename())
 160             f.copy(dst)
 161             dst.chmod(self.fmode)
 162 
 163     @property
 164     def outboxFiles(self):
 165         """Return files in the outbox
 166         """
 167         return self.outbox.files()
 168 
 169     @property
 170     def INobj(self):
 171         IN = {}
 172         for f in self.inbox.files():
 173             IN[f.basename()] = f.encode(config.charset).replace('\\','/')
 174         # skip run.py from IN object
 175         if 'run.py' in IN:
 176             del IN['run.py']
 177         return IN
 178 
 179 # ---------------------------------------------------------------------- 
 180 
 181 class MplClient:
 182     """Client interface to ipengines.
 183 
 184     1) Create sandbox
 185     2) Copy all required input data into inbox
 186     3) Supply python script file to run --> copy into inbox as run.py
 187     4) Run run.py with TaskClient and pull output out (dictionary)
 188     5) Get png files from outbox and attach to page name
 189     6) Cleanup
 190     """
 191 
 192     furl = FURL
 193 
 194     def __init__(self,request,pagename,script,fmt='png'):
 195 
 196         self.request    = request
 197         self.pagename   = pagename
 198         self.fmt        = fmt
 199         self.attach_dir = path(AttachFile.getAttachDir(self.request,self.pagename))
 200         self.sb     = None
 201         self.script = ''
 202         self.inbox  = []
 203         self.imgdata = []
 204         self.logdata = None
 205         self.outdata = {}
 206 
 207         # base script   
 208         self.script = script
 209 
 210         # treat attachments referenced in script and create input list
 211         self.parseAttachments()
 212         self.parseIncludes()
 213 
 214     @property
 215     def imgPrefix(self):
 216         raw = self.script
 217         for f in self.inbox:
 218             raw += str(f.stat().st_mtime)
 219         self.imgPrefix = 'mpl_' + sha.new(raw).hexdigest() + '_chart'
 220         return self.imgPrefix
 221 
 222     def addInput(self,infile):
 223         """Add infile to the set of input files.
 224         """
 225         f = path(infile)
 226         if f.isfile() and f.access(os.R_OK):
 227             self.inbox.append(f)
 228 
 229     def parseAttachments(self):
 230         """Collect input files from "#! attach(file-attachment-path)" in self.inbox
 231 
 232         The path can reference an attachment on a differenet page than the
 233         current. It supports relative path syntax, i.e. "../file.dat".
 234         The file part can include widcard character that are used for
 235         globing more than one file.
 236         
 237         Don't use quotation marks on the file-path.
 238         """
 239         rec = re.compile(r"""(#! *attach\()(?P<fname>[A-Za-z0-9 ._=+*/-]+)(\).*)""")
 240         for line in self.script.splitlines():
 241             m = rec.match(line)
 242             if m:
 243                 # find page and attach_dir where attachments are stored
 244                 aName = m.group('fname')
 245 
 246                 # get pageName and attchDir
 247                 fparts = aName.split('/')
 248                 pageName = '/'.join(fparts[0:-1])
 249 
 250                 if not pageName:
 251                     pageName = self.pagename
 252                 else:
 253                     if pageName.endswith('..'):
 254                         pageName += '/'
 255                     pageName = wikiutil.AbsPageName(self.pagename,pageName)
 256                 filePat  = fparts[-1]
 257 
 258                 attachDir = AttachFile.getAttachDir(self.request,pageName)
 259 
 260                 # attachment found, could be pattern
 261                 for fname in path(attachDir).files(filePat):
 262                     self.inbox.append(fname)
 263 
 264     def parseIncludes(self):
 265         """Replace #! include(pagename) with the python src from page.
 266         """
 267         rec = re.compile(r"""(#! *include\()(?P<pname>[A-Za-z0-9 ._=+*/-]+)(\).*)""")
 268         script = self.script.splitlines()
 269 
 270         for lno,line in enumerate(script):
 271             m = rec.match(line)
 272             if m:
 273                 # include detected
 274                 pageName = m.group('pname')
 275                 if pageName.endswith('..'):
 276                     pageName += '/'
 277                 pageName  = wikiutil.AbsPageName(self.pagename,pageName)
 278                 pageTxt = Page(self.request, pageName).get_raw_body()
 279                 pageTxt = '\n'.join(pageTxt.splitlines()[1:])
 280                 script[lno] = pageTxt 
 281 
 282         self.script = '\n'.join(script)
 283 
 284     def run(self):
 285         """Execute script in sandbox by ipengine
 286         """
 287 
 288         # make sandbox
 289         self.sb = SandBox(prefix='mpl_')  
 290         self.prefix  = str(self.sb.outbox.joinpath('mplplot')).replace('\\','/')
 291         self.logfile = self.sb.outbox.joinpath('error.log')
 292         self.outfile = self.sb.outbox.joinpath('mpl.out')
 293 
 294         # make mpl script
 295         self.mplScript = MplScript(self.script).mplScript        
 296 
 297         # send files (data files and python files)
 298         for f in self.inbox:
 299             self.sb.sendTo(f)
 300 
 301         # run
 302         self.runProcess()
 303 
 304         # read image data
 305         self.imgdata = []
 306         for no,imgfile in enumerate(self.sb.outbox.files('mplplot-*.%s'%self.fmt)):
 307             self.imgdata.append(file(imgfile,'rb').read())
 308 
 309         # read pickled output object
 310         if self.outfile.exists():
 311             f = open(self.outfile,'rb')
 312             self.outdata = pickle.load(f)
 313             f.close()
 314 
 315         # read logfile
 316         if self.logfile.exists():
 317             self.logdata = file(self.logfile,'r').read()
 318 
 319         # finally cleanup
 320         self.cleanup()
 321 
 322     def runProcess(self):
 323         """Run python/mpl script on an external ipengine
 324         Direct use of TaskClient is not possible because of 
 325         twisted web server used for moin application.
 326         """
 327 
 328         # build run script in sandbox
 329         self.pyScript = self.sb.inbox.joinpath('run.py')
 330         logfile = str(self.logfile).replace('\\','/')
 331         outfile = str(self.outfile).replace('\\','/')
 332         f = open(self.pyScript,'w')
 333         params = dict(furl=self.furl,script=self.mplScript,prefix=self.prefix,
 334                 fmt=self.fmt,log=logfile,out=outfile,inobj=self.sb.INobj)
 335         f.write( _mplCode % params)
 336         f.close()
 337 
 338         # execute sandbox script 
 339         self.failure = None
 340         p = subprocess.Popen([PYTHONCMD,self.pyScript])
 341         try:        
 342             p.wait()
 343         except exceptions.EnvironmentError, ex:
 344             os.kill(p.pid, 9)
 345             self.failure = ex
 346 
 347     def cleanup(self):
 348         """Cleanup sandbox
 349         """
 350         self.sb.remove()
 351 
 352 # ---------------------------------------------------------------------- 
 353 
 354 class MplScript:
 355 
 356     pre = """
 357 # --- PRE CODE -------------------------------------------
 358 import matplotlib as mpl
 359 mpl.use('Agg')
 360 import matplotlib.pyplot as plt
 361 import numpy as np
 362 
 363 class MoinMpl:
 364     def __init__(self,prefix,fmt='png'):
 365         self.prefix = prefix
 366         self.imgno  = 0
 367         self.fmt    = fmt
 368     def imgName(self,imgno=0):
 369         self.imgno = int(imgno)
 370         return '%s-%d.%s'% (self.prefix,self.imgno,self.fmt)
 371     def nextImg(self):
 372         return self.imgName(self.imgno+1)
 373 # ------------------------------------------
 374 
 375 mpl.rcdefaults()
 376 mm = MoinMpl(mmPref,mmFmt)
 377 out = {} # output will be handed to caller
 378 # --- END PRE CODE ---------------------------------------
 379 
 380 """
 381     post = """
 382 
 383 # --- POST CODE ------------------------------------------
 384 plt.close()
 385 # --- END POST CODE --------------------------------------
 386 """
 387 
 388     def __init__(self,pyscript=''):
 389         self.script = pyscript
 390 
 391     @property
 392     def mplScript(self):
 393         return self.pre+self.script+self.post
 394 
 395 # ---------------------------------------------------------------------- 
 396 
 397 def mpl_settings(run='on',src='off',mxsrc=9999,fmt='png',display='inline',columns=1,klass="mpl",tablestyle="",rowstyle="",style=""):
 398     """
 399     Initialize default parameters. The parameters are checked for wrong input.
 400     @param run: don't execute script of run == 'off' (default on)
 401     @param src: show src, default=off
 402     @param mxsrc: number of lines to show, default=9999  all lines basically 
 403     """
 404     return locals()
 405 
 406 # ---------------------------------------------------------------------- 
 407 
 408 class Parser:
 409     """
 410         Sends plot images generated by matplotlib
 411     """
 412     
 413     extensions = []
 414     Dependencies = Dependencies
 415 
 416     def __init__(self, raw, request, **kw):
 417         self.raw = raw
 418         self.request = request
 419 
 420         args = kw.get('format_args', '')
 421         # we use a macro definition to initialize the default init parameters
 422         # if a user enters a wrong parameter the failure is shown by the exception
 423         try:
 424             pyplot = mpl_settings()
 425             for k,v in pyplot.iteritems():
 426                 setattr(self,k,v)
 427             settings = wikiutil.invoke_extension_function(request, mpl_settings, args)
 428             for k, v in settings.iteritems():
 429                 if v: 
 430                     pyplot[k]=v
 431                     setattr(self,k,v)
 432 
 433         except ValueError, err:
 434             msg = u"matplotlib: %s" % err.args[0]
 435             request.write(self.request.formatter.text(msg))
 436 
 437     def format(self, formatter):
 438         """ Send the text. """
 439 
 440         self.request.flush() # to identify error text
 441         self.formatter = formatter
 442 
 443         if self.src.lower() in ('on','1','true'):
 444             # use highlight parser to show python code
 445             from highlight import Parser as HLParser
 446             #pre = MplScript().pre
 447             #post= MplScript().post
 448             #hlp = HLParser(pre+self.raw+post,self.request,format_args='python')
 449             hlp = HLParser(MPL_HELP+self.raw,self.request,format_args='python')
 450             hlp.format(formatter)
 451 
 452         if self.run.lower() in ('off','0','false'):
 453             self.request.write(formatter.preformatted(1)+'Execution disabled, set run=on.'+formatter.preformatted(0))
 454             return
 455 
 456         self.pagename = formatter.page.page_name
 457         self.attach_dir=AttachFile.getAttachDir(self.request,self.pagename,create=1)
 458 
 459         # -------------------- 
 460 
 461         # mplclient is created here, need to include data files in hexdigest
 462         mpl = MplClient(self.request,self.pagename,self.raw,self.fmt)
 463         imgPrefix = mpl.imgPrefix
 464         update, charts = self._updateImgs(formatter,imgPrefix)
 465 
 466         if update:
 467 
 468             mpl.run()
 469 
 470             if mpl.logdata:
 471                 # error occured
 472                 self.request.write(formatter.preformatted(1)+mpl.logdata+formatter.preformatted(0))
 473                 return
 474 
 475             # attach generated image(s)
 476             for no,imgdata in enumerate(mpl.imgdata):
 477                 imgName = "%s-%d.%s" % (imgPrefix,no,self.fmt)
 478                 attached_file = file(self.attach_dir + "/" + imgName, 'wb')
 479                 attached_file.write(imgdata)
 480                 attached_file.close()
 481                 charts.append(imgName)
 482 
 483         self.renderCharts(charts)
 484 
 485     def renderCharts(self,charts):
 486         """Render charts according to settings
 487         """
 488     
 489         if self.display.lower() in ('off','0','false') or not charts:
 490             return
 491 
 492         fmt = self.formatter
 493 
 494         html = []
 495         html.append(fmt.div(1,attr={'class':self.klass}))
 496 
 497         if self.display.lower() == 'link':
 498             # link display
 499             html.append(fmt.bullet_list(1))
 500             for chart in charts:
 501                 url = AttachFile.getAttachUrl(self.pagename, chart, self.request)
 502                 html.append(fmt.listitem(1))
 503                 html.append(fmt.url(1,url)+chart+fmt.url(0))
 504                 html.append(fmt.listitem(0))
 505             html.append(fmt.bullet_list(0))
 506         
 507         else:
 508             # inline display in table form
 509             noImgs = len(charts)
 510             rows  = noImgs / self.columns
 511             rows += noImgs%self.columns and 1 or 0
 512 
 513             T = fmt.table
 514             R = fmt.table_row
 515             C = fmt.table_cell
 516 
 517             html.append(T(1,style=self.tablestyle))
 518             id = 0
 519             for row in range(rows):
 520                 html.append(R(1,style=self.rowstyle))
 521                 for col in range(self.columns):
 522                     chart = charts[id]
 523                     url = AttachFile.getAttachUrl(self.pagename, chart, self.request)
 524                     html.append(C(1,style=self.style)+fmt.url(1,url)+fmt.image(src="%s" % url, alt=chart)+fmt.url(0)+C(0))
 525                     id += 1
 526                     if id >= noImgs:
 527                         break
 528                 html.append(R(0))
 529                 if id >= noImgs:
 530                     break
 531             html.append(T(0))
 532 
 533         html.append(fmt.div(0))
 534         self.request.write('\n'.join(html))
 535 
 536     def _updateImgs(self, formatter,imgPrefix):
 537         """Delete outdated charts
 538         @param formatter: formatter object
 539         """
 540 
 541         # use delete.me.to.regenerate.images trick from dot.py
 542         dm2ri = self.attach_dir + '/' + "delete.me.to.regenerate.images"
 543         charts = []
 544 
 545         updateImgs = True
 546         deleteImgs = not path(dm2ri).exists()
 547         if deleteImgs:
 548             open(dm2ri,'w').close()
 549 
 550         # delete.me. ... exists, if with pref exists, then continue and do not 
 551         # rerun mpl execution
 552         attach_files = AttachFile._get_files(self.request, self.pagename)
 553         reChart = re.compile(r"mpl_.*_chart-[0-9]+.%s"%self.fmt)
 554         for chart in attach_files:
 555             if reChart.match(chart) and deleteImgs:
 556                 fullpath = os.path.join(self.attach_dir, chart).encode(config.charset)
 557                 os.remove(fullpath)
 558 
 559             if not deleteImgs and imgPrefix in chart:
 560                 charts.append(chart)
 561                 updateImgs = False
 562 
 563         return updateImgs,charts