# -*- coding: iso-8859-1 -*- """ MoinMoin - convert various input formats into output formats <> Features: * fetches source file from urls or page attachments * converts the input file via a local conversion tool (dia, inkscape, ...) to the output format * checks timestamp of content for updating cached image * uses cache for input and output file Requires the external conversion program: pdf -> svg: pdf2svg pdf -> png: pstoimg dia -> (svg|png): dia dot -> (svg|png): dot eps -> (svg|png): inkscape xcf -> (svg|png): xcftools Most of the code is based on the pdf2img macro created by Reimar Bauer. @copyright: 2011 MoinMoin:ReimarBauer @copyright: 2014 MoinMoin:LarsKruse @license: GNU GPL v3 or later ---- Changelog: v1.0 - 2014/09/22 * initial release * the following conversions are supported: * pdf|dia|dot|eps -> svg|png * xcf -> png ---- Usage: <> <> <> <> """ from MoinMoin import log logging = log.getLogger(__name__) import os import urllib2 import httplib import subprocess from urlparse import urlparse from MoinMoin import caching, config from MoinMoin.util.SubProcess import Popen from MoinMoin.action import AttachFile, cache CACHE_ARENA = 'sendcache' CACHE_SCOPE = 'wiki' # list all input and output document types here CONTENT_TYPE_MAP = { "svg": 'image/svg+xml', "png": 'image/png', "dia": 'application/dia', "dot": 'application/graphviz', "eps": 'application/postscript', "pdf": 'application/pdf', "xcf": 'application/x-pdf', } # map pairs of (input, output) to conversion functions # we use a function instead of a dict for lazy evaluation def get_conversion_map(): return { ("dot", "svg"): dot_converter, ("dot", "png"): dot_converter, ("dia", "svg"): dia_converter, ("dia", "png"): dia_converter, ("pdf", "png"): pdf2png_converter, ("pdf", "svg"): pdf2svg_converter, #("xcf", "png"): xcf2png_converter_imagemagick, ("xcf", "png"): xcf2png_converter_xcftools, ("eps", "svg"): ps_converter, ("eps", "png"): ps_converter, ("ps", "svg"): ps_converter, ("ps", "png"): ps_converter, } class SourceNotFound(KeyError): pass class AttachmentSourceNotFound(SourceNotFound): pass class ConversionFailed(RuntimeError): pass def exec_cmd_tokens(_, args): try: proc = Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = proc.communicate() return stdout, stderr, proc.returncode except OSError: return "", _("Failed to run '%s'") % args[0], -1 def url_exists(url): try: item = urllib2.urlopen(url) content = item.read(size=1) item.close() return len(content) > 0 except (IOError, urllib2.HTTPError, ValueError): return False def last_modified(request=None, pagename=None, attachment=None, url=""): if not url: pdf_file = os.path.join(AttachFile.getAttachDir(request, pagename), attachment).encode(config.charset) filestat = os.stat(pdf_file) return request.user.getFormattedDateTime(filestat.st_mtime) parse_result = urlparse(url) conn = httplib.HTTPConnection(parse_result.netloc) conn.request("GET", parse_result.path) response = conn.getresponse() return response.getheader('last-modified') def get_cache_item_filename(request, cache_key): """ prepares the cache file and returns its file name """ return get_cache_data(request, cache_key)._fname def get_cache_data(request, cache_key): """ get the file location of a cache item """ return caching.CacheEntry(request, CACHE_ARENA, "%s.data" % cache_key, CACHE_SCOPE, do_locking=False) def fetch_source_item(request, url, content_type, timestamp): """ fetches the source item and stores it as cache file """ cache_key = cache.key(request, itemname="multi_convert", content=(url + timestamp)) try: item = urllib2.urlopen(url) except (IOError, urllib2.HTTPError, ValueError), err: logging.info(url) logging.debug("%s: %s" % (url, err)) return None else: cache.put(request, cache_key, item.read(), content_type=content_type) item.close() return get_cache_item_filename(request, cache_key) def _parse_extra_options(text): result = {} for token in text.split(): if ":" in token: key, value = token.split(":", 1) else: key, value = token, True result[key] = value return result def is_attachment(source): return not "://" in source def _get_source_and_cache(macro, source, source_content_type, cache_key_suffix): """ Generate the cache key to be used for storing data and put the source data (specified as an attachment or remote URL) into the cache. @param macro: the original macro object containing the current request @param source: use the specified attachment or URL as a source @param source_content_type: content type of the source data (e.g. "application/pdf") @param cache_key_suffix: a unique unicode string to be used for this specific conversion all relevant options need to be included in this string (e.g. output format, size, ...) """ request = macro.request pagename = request.page.page_name # determine source URL and cache key if is_attachment(source): page_name, filename = AttachFile.absoluteName(source, pagename) if not AttachFile.exists(request, page_name, filename): raise AttachmentSourceNotFound("attachment: %s does not exists" % source) identifier = last_modified(request, page_name, filename) source_url = AttachFile.getAttachUrl(page_name, filename, request) source_file = AttachFile.getFilename(request, page_name, filename).encode(config.charset) else: if not url_exists(source): raise SourceNotFound("url: %s does not exists" % source) identifier = last_modified(url=source) source_url = source # store the source data in a local download cache source_file = fetch_source_item(request, source, source_content_type, identifier) logging.debug("%s: %s" % (source_url, identifier)) target_cache_key = cache.key(request, itemname=pagename, content="%s.%s.%s.%s" % (macro.name, source_url, identifier, cache_key_suffix)) return source_file, target_cache_key def _do_conversion(request, source_file, target_cache_key, conversion_func, target_content_type, conversion_details): """ Run the given conversion function and store the result in the cache. @param request: a moinmoin request instance @source_file: local name of the input file @cache_key: the key to be used for retrieving the cache content that should be displayed @conversion_func: a conversion function that expects four parameters: request.getText, input_filename, output_filename, details @param target_content_type: content type of the target (e.g. "image/png") """ _ = request.getText if cache.exists(request, target_cache_key): # nothing to be done return target_file = get_cache_item_filename(request, target_cache_key) output_msg, error_msg, returncode = conversion_func(_, source_file, target_file, conversion_details) if returncode != 0: raise ConversionFailed(_("Conversion failed: %s") % error_msg) # explicitly transfer the file to the cache (just to be sure that moin's cache is updated) target_handle = open(target_file, 'rb') cache.put(request, target_cache_key, target_handle.read(), content_type=target_content_type) target_handle.close() def _get_conversion_result(request, width, height, cache_key, attachment_name=None): """ Return a formatted representation of a cache item via a selected style. @param request: a moinmoin request instance @param target_content_type: content type of the target (e.g. "image/png") @width: the width of the embedded conversion visualization (only for 'do_embed == True') @height: the height of the embedded conversion visualization (only for 'do_embed == True') @cache_key: the key to be used for retrieving the cache content that should be displayed """ if cache.exists(request, cache_key): formatter = request.formatter result = "" args = {} if width: args["width"] = width if height: args["height"] = height if attachment_name: result += formatter.attachment_link(1, url=attachment_name) result += formatter.image(src=cache.url(request, cache_key), alt="", **args) if attachment_name: result += formatter.attachment_link(0) return result else: # failed to convert input return "" def dia_converter(_, source_file, target_file, details): target_type = details["target_type"] args = ["dia", "--nosplash", "--export", target_file] if target_type == "png": args += ["--filter", "png"] # target size is only available for png output format width = details.get("width", 0) height = details.get("height", 0) if width or height: args += ["--size", "%sx%s" % (width or "", height or "")] elif target_type == "svg": args += ["--filter", "svg"] else: raise ConversionFailed(_("Unsupported output format for dia converter: %s" % target_type)) args += [source_file] return exec_cmd_tokens(_, args) def dot_converter(_, source_file, target_file, details): target_type = details["target_type"] args = ["dot"] if target_type == "png": # TODO: implement width/height args += ["-Tpng"] elif target_type == "svg": args += ["-Tsvg"] else: raise ConversionFailed(_("Unsupported output format for dot converter: %s" % target_type)) args += ["-o%s" % target_file, source_file] return exec_cmd_tokens(_, args) def pdf2png_converter(_, source_file, target_file, details): # TODO: implement width/height # TODO: handle "page" return exec_cmd_tokens(_, ["pstoimg", "-quiet", "-crop", "tblr", "-density", 200, "-type", "png", source_file, "-out", target_file]) def pdf2svg_converter(_, source_file, target_file, details): # 'pageno' can be a number or a page specification (e.g. 'iii') page = details.get("page", 1) return exec_cmd_tokens(_, ["pdf2svg", source_file, target_file, str(page)]) def xcf2png_converter_xcftools(_, source_file, target_file, details): return exec_cmd_tokens(_, ["xcf2png", "-o", target_file, source_file]) def xcf2png_converter_imagemagick(_, source_file, target_file, details): return exec_cmd_tokens(_, ["convert", source_file, "-alpha", "on", "-background", "none", "-layers", "merge", "png:%s" % target_file]) def ps_converter(_, source_file, target_file, details): target_type = details["target_type"] args = ["inkscape", "--without-gui"] if target_type == "png": args += ["--export-png", target_file] width = details.get("width", 0) if width: args += ["--export-width", width] height = details.get("height", 0) if height: args += ["--export-height", height] elif target_type == "svg": args += ["--export-plain-svg", target_file] else: raise ConversionFailed(_("Unsupported output format for postscript converter: %s" % target_type)) args += [source_file] return exec_cmd_tokens(_, args) def macro_FormatConverter(macro, source, source_type, target_type, width=0, height=0, extras=""): """ converts input data from urls or attachments to image files using MoinMoins cache and renders from there Retrieve the source data, run the conversion function and return the formatted output. @param macro: the original macro object containing the current request @param source: use the specified attachment or URL as a source @param source_type: content type of the source data (e.g. "pdf") @param target_type: content type of the target (e.g. "png") @width: the width of the embedded conversion visualization (only for 'do_embed == True') @height: the height of the embedded conversion visualization (only for 'do_embed == True') @extras: a space-separated list of 'key:value' pairs - e.g. used for specifying a page number """ _ = macro.request.getText source_type = source_type.lower() target_type = target_type.lower() if not extras: extras = "" try: conversion_func = get_conversion_map()[(source_type, target_type)] except KeyError: return _("The requested conversion is not supported: <%s> to <%s>. Use any of the following instead: %s") % \ (source_type, target_type, get_conversion_map().keys()) source_content_type = CONTENT_TYPE_MAP[source_type] target_content_type = CONTENT_TYPE_MAP[target_type] cache_key_suffix = " ".join(["width=%d" % width, "height=%d" % height, extras]) try: source_file, target_cache_key = _get_source_and_cache(macro, source, source_content_type, cache_key_suffix) except AttachmentSourceNotFound: # display an "upload missing attachment" link result = macro.request.formatter.attachment_link(1, url=source) result += macro.request.formatter.text(_("Upload source file")) result += macro.request.formatter.attachment_link(0) return result except SourceNotFound, err_msg: return _("Could not find source (%s) for conversion: %s") % (source, err_msg) # parse 'extras' string extra_tokens = _parse_extra_options(extras) # run conversion conversion_details = { "width": width, "height": height, "source_type": source_type, "target_type": target_type, } conversion_details.update(extra_tokens) _do_conversion(macro.request, source_file, target_cache_key, conversion_func, target_content_type, conversion_details) # return formatted result string if is_attachment(source): attachment_name = source else: attachment_name = None return _get_conversion_result(macro.request, width, height, target_cache_key, attachment_name=attachment_name)