Attachment 'mpl-1.0_fixed.py'
Download 1 # -*- coding: iso-8859-1 -*-
2 """
3 MoinMoin - Matplotlib integration
4
5 example: {{{#!mpl src=off,run=on,prefix=mpl,display=inline,klass=mpl,tablestyle="",rowstyle="",style="",persistence=True,debug=False
6 ...
7 ...
8 }}}
9
10 keyword parameters:
11
12 @param display: off|link|inline (default:inline)
13 @param columns: arange images in table with No columns (default:1)
14 if more than 1 image and display is inline
15 @param src: on|off (display mpl source, or not, default: off)
16 @param run: on|off (if off, execution is disabled, default:on)
17
18 @prefix: prefix for files generated (default: mpl). If you have more than one
19 image on a page which is not persistent, you should use a unique prefix
20 for each chart.
21
22 @params klass: class attribute for div
23 @params tablestyle: style attribute for table
24 @params rowstyle: style attribute for table row
25 @params style: style attribute for table cell
26
27 @params persistence: if True create delete.me.to.regenerate.images marker (default: True)
28 @params debug: if True keep files after processing (default: False)
29
30
31 Directives:
32 -----------
33
34 #! include(pagename)
35 #! attach(pagename/*.xls)
36 #! page(pagename)
37 #! icaedoc(icaedoc-name)
38
39 Mpl scripts can be made very modular using the '#! include(pagename)' directive.
40 You can see this similar to the import statement in python. In the Mpl script the
41 whole page content (#format python) is included. Use this to store your standard
42 settings as reuse them for consistent plotting.
43
44 example: #! include(MplSettings)
45
46
47 Data sources:
48 ============
49
50 1) Page attachments
51 *******************
52
53 # --------------------------------
54 #! attach(pagename/*.xls)
55 # --------------------------------
56
57 The IN object contains all attached input files in the IN['files'] dictionary with the filenames as keys.
58 If you want to make a page attachment available for the plotting process,
59 use the '#! attach(filename)' directive and reference the file in the open
60 statement as shown in the example.
61
62 example:
63 #! attach('filename')
64 files = IN['files']
65 f = open(files['filename'])
66
67 2) Page raw data
68 ****************
69
70 # --------------------------------
71 #! page(pagename)
72 # --------------------------------
73
74 The IN object contains the page raw data (text) in the IN['pages'] dictionary with the pagenames as keys
75 and the page content (as obtained from page.get_raw_body()). Absolute page names are used as keys.
76
77 example:
78 #! page(PageWithData)
79 pgData = IN['pages']['PageWithData']
80
81 3) Form data
82 ************
83
84 The IN object also contains the form dictionary from the current request.
85 example: value = IN['form'].get('parname',['0'])[0]
86
87 4) iCAE document objects
88 ************************
89
90 # --------------------------------
91 #! icaedoc(iCAE-doc-name)
92 # --------------------------------
93
94 You can add iCAE documents with the correct short path. The documents are made
95 available in the IN['files'] dictionary in the same way in the '#! attach()' directive.
96
97 example:
98 #! icaedoc('P000000-MainEngineData.xls')
99 files = IN['files']
100 f = open(files['P000000-MainEngineData.xls'])
101
102 The IN object contains also the current user name from the request in IN['user']
103
104 @copyright: 2008 F. Zieher
105 @license: GNU GPL, see COPYING for details.
106
107 """
108
109 import os, re
110 import sys
111 import exceptions
112 import tempfile
113 import sha
114 import pickle
115
116 from path import path
117
118 from MoinMoin.Page import Page
119 from MoinMoin import config, wikiutil
120 from MoinMoin.action import AttachFile
121 from MoinMoin import log
122
123 # -------------------------------------------------------------------------------
124
125 loggin = log.getLogger(__name__)
126
127 # ----------------------------------------------------------------------
128
129 # some config settings, XXX should be moved to the moin cfg object
130 if 'win' in sys.platform:
131 FURL = 'e:/home/zieherf/_ipython/security/ipcontroller-tc.furl'
132 else:
133 FURL = '/s330/moin/furls/ipcontroller-tc.furl'
134
135 Dependencies = ['time']
136
137 # ----------------------------------------------------------------------
138
139 MPL_HELP = """
140 # ------------------------------------------------------------------------------------
141 # Predefined objects:
142 # mpl ... matplotlib object
143 # plt ... matplotlib.pyplot object
144 # np ... numpy object
145 # IN ... dictionary containing 'files' dictionary, 'pages' dictionary,
146 # 'form' dictionary (copy of request.form) and 'user' name.
147 # - Access attached file with IN['files']['filename']
148 # - Accrss page data with IN['pages']['PageName']
149 # - Access form parameter with IN['form'].get(param,['default-value'])[0]
150 # mm ... use mm.imgName(imgno=0) and mm.nextImg() in plt.savefig
151 # examples: plt.savefig(mm.imgName()), and for all consecutive
152 # plt.savefig(mm.nextImg())
153 # ------------------------------------------------------------------------------------
154 """
155
156 # ----------------------------------------------------------------------
157
158 class SandBox:
159 """Implement Sandbox for executing python code.
160 Sandbox has inbox/outbox directories. inbox is where ipengine
161 reads all data from. outbox is where ipengine creates
162 output files (i.e. image files from matplotlib).
163 """
164
165 fmode = 0666
166 dmode = 0777
167
168 def __init__(self,request,sbpath='',prefix=''):
169 """Instantiate sandbox
170
171 @param sbpath: if given, path to sandbox, otherwise a temporary name will be created
172 @param prefix: prefix to use when sandbox name is created automatically
173 """
174 self.request = request
175 self.sb = sbpath and path(sbpath) or path(tempfile.NamedTemporaryFile(prefix=prefix).name)
176 self.inbox = self.sb + '/inbox'
177 self.outbox = self.sb + '/outbox'
178 self.create()
179
180 def create(self):
181 """Create sandbox
182 """
183 for d in (self.sb,self.inbox,self.outbox):
184 d.mkdir()
185 d.chmod(self.dmode)
186
187 def emptyInbox(self):
188 """Empty (delete) all files in inbox
189 """
190 for f in self.inbox.walkfiles():
191 f.remove()
192
193 def emptyOutbox(self):
194 """Empty (delete) all files in outbox
195 """
196 for f in self.outbox.walkfiles():
197 f.remove()
198
199 def remove(self):
200 """Remove sandbox
201 """
202 self.sb.rmtree()
203
204 def sendTo(self,srcname,dstname=''):
205 """Copy file srcname into the inbox.
206 If dstname is not given, srcname is used.
207 """
208 f = path(srcname)
209 if f.isfile() and f.access(os.R_OK):
210 dst = dstname and self.inbox + dstname or self.inbox + f.basename()
211 f.copy(dst)
212 dst.chmod(self.fmode)
213
214 @property
215 def outboxFiles(self):
216 """Return files in the outbox
217 """
218 return self.outbox.files()
219
220 @property
221 def inDict(self):
222 """Input object sent to ipengine"""
223
224 indict = dict(files={},form={},user="")
225
226 # file objects from attach and icaedoc directives
227 for f in self.inbox.files():
228 indict['files'][f.basename()] = f.encode(config.charset).replace('\\','/')
229
230 return indict
231
232 # ----------------------------------------------------------------------
233
234 class MplClient:
235 """Client interface to ipengines.
236
237 1) Create sandbox
238 2) Copy all required input data into inbox
239 3) Supply python script file to run --> copy into inbox as run.py
240 4) Run run.py with TaskClient and pull output out (dictionary)
241 5) Get png files from outbox and attach to page name
242 6) Cleanup
243 """
244
245 furl = FURL
246
247 def __init__(self,request,pagename,script,fmt='png',prefix='mpl',debug=False):
248
249 self.request = request
250 self.pagename = pagename
251 self.fmt = fmt
252 self.prefix = prefix
253 self.debug = debug
254 self.attach_dir = path(AttachFile.getAttachDir(self.request,self.pagename))
255 self.sb = None
256 self.script = ''
257 self.inbox = []
258 self.pageData = {}
259 self.imgdata = []
260 self.logdata = None
261 self.outdata = {}
262
263 # base script
264 self.script = script
265
266 # treat attachments referenced in script and create input list
267 # XXX could be done more efficient (parse once :-)
268 self.parseAttachments()
269 self.parseIcaeDocs()
270 self.parsePages()
271 self.parseIncludes()
272
273 @property
274 def imgPrefix(self):
275 raw = self.script
276 for f in self.inbox:
277 raw += str(f.stat().st_mtime)
278 for pg,data in self.pageData.items():
279 raw += data
280 self.imgPrefix = self.prefix + "_" + sha.new(raw).hexdigest() + '_chart'
281 return self.imgPrefix
282
283 def addInput(self,infile):
284 """Add infile to the set of input files.
285 """
286 f = path(infile)
287 if f.isfile() and f.access(os.R_OK):
288 self.inbox.append(f)
289
290 def parseAttachments(self):
291 """Collect input files from "#! attach(file-attachment-path)" in self.inbox
292
293 The path can reference an attachment on a differenet page than the
294 current. It supports relative path syntax, i.e. "../file.dat".
295 The file part can include widcard character that are used for
296 globing more than one file.
297
298 Don't use quotation marks on the file-path.
299 """
300 rec = re.compile(r"""(#! *attach\()(?P<fname>[A-Za-z0-9 ._=+*/-]+)(\).*)""")
301 for line in self.script.splitlines():
302 m = rec.match(line)
303 if m:
304 # find page and attach_dir where attachments are stored
305 aName = m.group('fname')
306
307 # get pageName and attchDir
308 fparts = aName.split('/')
309 pageName = '/'.join(fparts[0:-1])
310
311 if not pageName:
312 pageName = self.pagename
313 else:
314 if pageName.endswith('..'):
315 pageName += '/'
316 pageName = wikiutil.AbsPageName(self.pagename,pageName)
317 filePat = fparts[-1]
318
319 attachDir = AttachFile.getAttachDir(self.request,pageName)
320
321 # attachment found, could be pattern
322 for fname in path(attachDir).files(filePat):
323 self.inbox.append(fname)
324
325 def parseIcaeDocs(self):
326 """Collect input files from "#! icaedoc(file-attachment-path)" in self.inbox
327 The path must be a conform icae document path, i.e. having the short path
328 prepended. Don't use quotation marks on the file-path.
329 example:
330 #! icaedoc(P000000_4010-benchmark.xls)
331 """
332 return
333
334 from icaebase import getPathFromSPath
335
336 rec = re.compile(r"""(#! *icaedoc\()(?P<fname>[A-Za-z0-9 ._=+*/-]+)(\).*)""")
337 for line in self.script.splitlines():
338 m = rec.match(line)
339 if m:
340 fname = m.group('fname')
341 try:
342 spath,doc = fname.split('-')
343 except:
344 continue
345
346 fname = (getPathFromSPath(spath) + '0-documentation') + fname
347 if fname.exists():
348 # attachment found, could be pattern
349 self.inbox.append(fname)
350
351 def parsePages(self):
352 """Read data from #! page(pagename) and store text data in self.pages
353 """
354 rec = re.compile(r"""(#! *page\()(?P<pname>[A-Za-z0-9 ._=+*/-]+)(\).*)""")
355 script = self.script.splitlines()
356
357 for lno,line in enumerate(script):
358 m = rec.match(line)
359 if m:
360 # page directive detected
361 pageName = m.group('pname')
362 if pageName.endswith('..'):
363 pageName += '/'
364 pageName = wikiutil.AbsPageName(self.pagename,pageName)
365 self.pageData[pageName] = Page(self.request, pageName).get_raw_body()
366
367 def parseIncludes(self):
368 """Replace #! include(pagename) with the python src from page.
369 """
370 rec = re.compile(r"""(#! *include\()(?P<pname>[A-Za-z0-9 ._=+*/-]+)(\).*)""")
371 script = self.script.splitlines()
372
373 for lno,line in enumerate(script):
374 m = rec.match(line)
375 if m:
376 # include detected
377 pageName = m.group('pname')
378 if pageName.endswith('..'):
379 pageName += '/'
380 pageName = wikiutil.AbsPageName(self.pagename,pageName)
381 pageTxt = Page(self.request, pageName).get_raw_body()
382 pageTxt = '\n'.join(pageTxt.splitlines()[1:])
383 script[lno] = pageTxt
384
385 self.script = '\n'.join(script)
386
387 # ------------------- assyncclient ----------------------------------------------------------
388
389 def run(self):
390 """Execute matplotlib script by ipengine
391 """
392
393 # make sandbox
394 self.sb = SandBox(self.request,prefix='mpl_')
395
396 # send files (data files and python files)
397 for f in self.inbox:
398 self.sb.sendTo(f)
399
400 # -------------------------------------------------------------
401 # run StringTask on ipengine / most cool feature :-)
402 # -------------------------------------------------------------
403
404 # IN object
405 inDict = self.sb.inDict.copy()
406 inDict['pages'] = self.pageData
407 inDict['form'] = self.request.form.copy()
408 inDict['user'] = self.request.user.name
409 inDict['imgBase'] = str(self.sb.outbox + '/mplplot').replace('\\','/')
410 inDict['fmt'] = self.fmt
411
412 # execute script on ipengine and obtain output (pullObjs)
413 script = str(MplScript(self.script).mplScript)
414 pushObjs = {'IN':inDict}
415 pushObjs['OUT'] = {}
416
417 # the tricky part of handling the execution on the client
418 # this is valid when using twisted as web framework in moin
419
420 #import sys
421 #sys.path.append("/home/zjfhach/usr/lib/python2.6/site-packages")
422 #if True:#hasattr(self.request,'reactor'):
423 # from IPython.kernel import asyncclient
424 # from twisted.internet.threads import blockingCallFromThread
425 # tc = blockingCallFromThread(self.request.reactor,asyncclient.get_task_client,self.furl)
426 # tc = tc.adapt_to_blocking_client()
427 # st = asyncclient.StringTask(script,clear_before=True,pull=pullObjs,push=pushObjs)
428 #else:
429 # from IPython.kernel import client
430 # tc = client.TaskClient(self.furl)
431 # st = client.StringTask(script,clear_before=True,pull=pullObjs,push=pushObjs)
432
433 #tid = tc.run(st)
434 exec(script,pushObjs) #tc.get_task_result(tid,block=True)
435
436 #if res.failure:
437 # self.failure = res.failure
438 # self.logdata = res.failure.getTraceback()
439
440 #if res.results.get('OUT',{}):
441 # self.outdata = res.results.get('OUT',{})
442 self.outdata = pushObjs['OUT']
443 # -------------------------------------------------------------
444
445 # read image data
446 self.imgdata = []
447 for no,imgfile in enumerate(self.sb.outbox.files('mplplot-*.%s'%self.fmt)):
448 self.imgdata.append(file(imgfile,'rb').read())
449
450 # finally cleanup
451 self.cleanup()
452
453 def cleanup(self):
454 """Cleanup sandbox
455 """
456 if not self.debug:
457 self.sb.remove()
458
459
460 # ----------------------------------------------------------------------
461
462 class MplScript:
463
464 pre = """
465 # --- PRE CODE -------------------------------------------
466 import matplotlib as mpl
467 mpl.use('Agg')
468 import matplotlib.pyplot as plt
469 import numpy as np
470
471 # reset mpl default settings
472 mpl.rcdefaults()
473
474 class MoinMpl:
475 def __init__(self,base,fmt='png'):
476 self.base = base
477 self.imgno = 0
478 self.fmt = fmt
479 def imgName(self,imgno=0):
480 self.imgno = int(imgno)
481 return '%s-%d.%s'% (self.base,self.imgno,self.fmt)
482 def nextImg(self):
483 return self.imgName(self.imgno+1)
484 # ------------------------------------------
485
486 mm = MoinMpl(IN['imgBase'],IN['fmt'])
487 OUT = {} # output will be handed to caller
488 # --- END PRE CODE ---------------------------------------
489
490 """
491 post = """
492
493 # --- POST CODE ------------------------------------------
494 plt.close()
495 # --- END POST CODE --------------------------------------
496 """
497
498 def __init__(self,pyscript=''):
499 self.script = pyscript
500
501 @property
502 def mplScript(self):
503 return self.pre+self.script+self.post
504
505 # ----------------------------------------------------------------------
506
507 def mpl_settings(run='on',src='off',mxsrc=9999,fmt='png',display='inline',columns=1,prefix='mpl',
508 klass="mpl",tablestyle="",rowstyle="",style="",persistence=True,debug=False):
509 """
510 Initialize default parameters.
511 """
512 return locals()
513
514 # ----------------------------------------------------------------------
515
516 class Parser:
517 """
518 Sends plot images generated by matplotlib
519 """
520
521 extensions = []
522 Dependencies = Dependencies
523
524 def __init__(self, raw, request, **kw):
525 self.raw = raw
526 self.request = request
527
528 args = kw.get('format_args', '')
529 # we use a macro definition to initialize the default init parameters
530 # if a user enters a wrong parameter the failure is shown by the exception
531 try:
532 pyplot = mpl_settings()
533 for k,v in pyplot.iteritems():
534 setattr(self,k,v)
535 settings = wikiutil.invoke_extension_function(request, mpl_settings, args)
536 for k, v in settings.iteritems():
537 if v != None:
538 pyplot[k]=v
539 setattr(self,k,v)
540
541 except ValueError, err:
542 msg = u"matplotlib: %s" % err.args[0]
543 request.write(self.request.formatter.text(msg))
544
545 def format(self, formatter):
546 """ Send the text. """
547
548 #self.request.flush() # to identify error text
549 self.formatter = formatter
550
551 if self.src.lower() in ('on','1','true'):
552 # use highlight parser to show python code
553 from MoinMoin.parser.highlight import Parser as HLParser
554 hlp = HLParser(MPL_HELP+self.raw,self.request,format_args='python')
555 hlp.format(formatter)
556
557 if self.run.lower() in ('off','0','false'):
558 self.request.write(formatter.preformatted(1)+'Execution disabled, set run=on.'+formatter.preformatted(0))
559 return
560
561 self.pagename = formatter.page.page_name
562 self.attach_dir=AttachFile.getAttachDir(self.request,self.pagename,create=1)
563
564 # --------------------
565
566 # mplclient is created here, need to include data files in hexdigest
567 self.mpl = MplClient(self.request,self.pagename,self.raw,self.fmt,prefix=self.prefix,debug=self.debug)
568 update, charts = self._updateImgs(formatter)
569
570 if update:
571
572 self.mpl.run()
573
574 if self.mpl.logdata:
575 # error occured
576 self.request.write(formatter.preformatted(1)+self.mpl.logdata+formatter.preformatted(0))
577 return
578
579 # attach generated image(s)
580 for no,imgdata in enumerate(self.mpl.imgdata):
581 imgName = "%s-%d.%s" % (self.mpl.imgPrefix,no,self.fmt)
582 attached_file = file(self.attach_dir + "/" + imgName, 'wb')
583 attached_file.write(imgdata)
584 attached_file.close()
585 charts.append(imgName)
586
587 self.renderCharts(charts)
588
589 def renderCharts(self,charts):
590 """Render charts according to settings
591 """
592
593 if self.display.lower() in ('off','0','false') or not charts:
594 return
595
596 fmt = self.formatter
597
598 html = []
599 html.append(fmt.div(1,attr={'class':self.klass}))
600
601 if self.display.lower() == 'link':
602 # link display
603 html.append(fmt.bullet_list(1))
604 for chart in charts:
605 url = AttachFile.getAttachUrl(self.pagename, chart, self.request)
606 html.append(fmt.listitem(1))
607 html.append(fmt.url(1,url)+chart+fmt.url(0))
608 html.append(fmt.listitem(0))
609 html.append(fmt.bullet_list(0))
610
611 else:
612 # inline display in table form
613 noImgs = len(charts)
614 rows = noImgs / self.columns
615 rows += noImgs%self.columns and 1 or 0
616
617 T = fmt.table
618 R = fmt.table_row
619 C = fmt.table_cell
620
621 html.append(T(1,style=self.tablestyle))
622 id = 0
623 for row in range(rows):
624 html.append(R(1,style=self.rowstyle))
625 for col in range(self.columns):
626 chart = charts[id]
627 url = AttachFile.getAttachUrl(self.pagename, chart, self.request)
628 html.append(C(1,style=self.style)+fmt.url(1,url)+fmt.image(src="%s" % url, alt=chart)+fmt.url(0)+C(0))
629 id += 1
630 if id >= noImgs:
631 break
632 html.append(R(0))
633 if id >= noImgs:
634 break
635 html.append(T(0))
636
637 html.append(fmt.div(0))
638 self.request.write('\n'.join(html))
639
640 def _removeChart(self,chart):
641 """Remove chart attachment from page and from xapian index.
642 @param chart: chart name (without path)
643 """
644 fpath = os.path.join(self.attach_dir, chart).encode(config.charset)
645 os.remove(fpath)
646 if self.request.cfg.xapian_search:
647 from MoinMoin.search.Xapian import Index
648 index = Index(self.request)
649 if index.exists:
650 index.remove_item(self.pagename, chart)
651
652 def _updateImgs(self, formatter):
653 """Delete outdated charts
654 @param formatter: formatter object
655 """
656 imgPrefix = self.mpl.imgPrefix
657
658 # use delete.me.to.regenerate.images trick from dot.py
659 dm2ri = self.attach_dir + '/' + "%s.delete.me.to.regenerate.images"%self.prefix
660 charts = []
661
662 updateImgs = True
663 if not self.persistence and path(dm2ri).exists():
664 path(dm2ri).remove()
665
666 deleteImgs = not path(dm2ri).exists()
667
668 # delete.me. ... exists, if with pref exists, then continue and do not
669 # rerun mpl execution
670 attach_files = AttachFile._get_files(self.request, self.pagename)
671 reChart = re.compile(r"%s_.*_chart-[0-9]+.%s"%(self.prefix,self.fmt))
672 for chart in attach_files:
673 if reChart.match(chart) and deleteImgs:
674 self._removeChart(chart)
675
676 if not deleteImgs and imgPrefix in chart:
677 charts.append(chart)
678 updateImgs = False
679
680 # create persistence marker if not existing and persistence requested
681 if deleteImgs and self.persistence:
682 open(dm2ri,'w').close()
683
684 return updateImgs,charts
Attached Files
To refer to attachments on a page, use attachment:filename, as shown below in the list of files. Do NOT use the URL of the [get] link, since this is subject to change and can break easily.You are not allowed to attach a file to this page.