2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/087a08c43cb5/ changeset: 087a08c43cb5 user: jmchilton date: 2012-12-07 18:48:07 summary: Fix upload of files to history through API (as outlined here http://dev.list.galaxyproject.org/Uploading-large-files-to-history-through-A...). affected #: 2 files diff -r 330f05e523aee275c91f3e5de1177f8eb832d512 -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 lib/galaxy/web/framework/__init__.py --- a/lib/galaxy/web/framework/__init__.py +++ b/lib/galaxy/web/framework/__init__.py @@ -142,6 +142,14 @@ named_args, _, _, _ = inspect.getargspec(func) for arg in named_args: payload.pop(arg, None) + for k, v in payload.iteritems(): + if isinstance(v, (str, unicode)): + try: + payload[k] = simplejson.loads(v) + except: + # may not actually be json, just continue + pass + payload = util.recursively_stringify_dictionary_keys( payload ) else: # Assume application/json content type and parse request body manually, since wsgi won't do it. However, the order of this check # should ideally be in reverse, with the if clause being a check for application/json and the else clause assuming a standard encoding diff -r 330f05e523aee275c91f3e5de1177f8eb832d512 -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 lib/galaxy/webapps/galaxy/api/tools.py --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -72,6 +72,11 @@ # Set up inputs. inputs = payload[ 'inputs' ] + # Find files coming in as multipart file data and add to inputs. + for k, v in payload.iteritems(): + if k.startswith("files_"): + inputs[k] = v + # HACK: add run button so that tool.handle_input will run tool. inputs['runtool_btn'] = 'Execute' # TODO: encode data ids and decode ids. https://bitbucket.org/galaxy/galaxy-central/commits/caaab0382447/ changeset: caaab0382447 user: jmchilton date: 2013-01-10 17:51:57 summary: Merge. affected #: 253 files diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 .hgignore --- a/.hgignore +++ b/.hgignore @@ -74,6 +74,10 @@ */variables.less static/june_2007_style/blue/base_sprites.less +# Testing +selenium-server.jar +selenium_results.html + # Documentation build files. doc/build diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 doc/Makefile --- a/doc/Makefile +++ b/doc/Makefile @@ -6,6 +6,9 @@ SPHINXBUILD = sphinx-build PAPER = BUILDDIR = build +UPDATEWORKDIR = /tmp/galaxySphinxUpdate +UPDATEWORKSOURCELIB = $(UPDATEWORKDIR)/source/lib +SPHINXAPIDOC = sphinx-apidoc # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 @@ -21,7 +24,7 @@ $(TOOLDATASHAREDDIR)/ncbi/builds.txt \ $(TOOLDATASHAREDDIR)/ucsc/publicbuilds.txt -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext updaterst # Sphinx wants the build files to be there; Copy the sample files into @@ -51,9 +54,13 @@ @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " updaterst to update sphinx rst to reflect code structure changes" + +# might also want to do +# cd source/lib; hg revert; rm *.rst.orig; or not. clean: - -rm -rf $(BUILDDIR)/* + -rm -rf $(BUILDDIR)/* $(UPDATEWORKDIR) html: $(TOOLDATABUILDFILES) $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @@ -165,3 +172,20 @@ $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." + +# Targets for updating the structure of the Sphinx RST doc for lib/ + +$(UPDATEWORKSOURCELIB): + mkdir -p $(UPDATEWORKSOURCELIB) + +# Create a fresh version of the RST files for the lib, and then create a +# unified patch file (ignore all emacs leftovers). +# Feed that to our custom version of patch.py, which applies patches that +# are only adds, and reports everything else to the user to deal with manually +# +# Note: this is still a very rough process. the run of patch.py gets some +# errors that don't mean anything to us. And the manual process is not fun. +updaterst: $(UPDATEWORKSOURCELIB) + $(SPHINXAPIDOC) -o $(UPDATEWORKSOURCELIB) ../lib + -diff -x '*.rst~' -ru source/lib $(UPDATEWORKSOURCELIB) > $(UPDATEWORKDIR)/alldifs.patch + ./patch.py $(UPDATEWORKDIR)/alldifs.patch diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 doc/patch.py --- /dev/null +++ b/doc/patch.py @@ -0,0 +1,1085 @@ +#!/usr/bin/env python +""" Patch utility to apply unified diffs + + Brute-force line-by-line non-recursive parsing + + Copyright (c) 2008-2012 anatoly techtonik + Available under the terms of MIT license + + Project home: http://code.google.com/p/python-patch/ + + + $Id: patch.py 181 2012-11-23 16:03:05Z techtonik $ + $HeadURL: https://python-patch.googlecode.com/svn/trunk/patch.py $ + + This program needs further tweaking for how we use it at Galaxy. +""" + +__author__ = "anatoly techtonik <techtonik@gmail.com>" +__version__ = "1.12.11" + +import copy +import logging +import re +# cStringIO doesn't support unicode in 2.5 +from StringIO import StringIO +import urllib2 + +from os.path import exists, isfile, abspath +import os +import shutil + +#------------------------------------------------ +# Logging is controlled by logger named after the +# module name (e.g. 'patch' for patch.py module) + +debugmode = False + +logger = logging.getLogger(__name__) + +debug = logger.debug +info = logger.info +warning = logger.warning + +class NullHandler(logging.Handler): + """ Copied from Python 2.7 to avoid getting + `No handlers could be found for logger "patch"` + http://bugs.python.org/issue16539 + """ + def handle(self, record): + pass + def emit(self, record): + pass + def createLock(self): + self.lock = None + +logger.addHandler(NullHandler()) + +#------------------------------------------------ +# Constants for Patch/PatchSet types + +DIFF = PLAIN = "plain" +GIT = "git" +HG = MERCURIAL = "mercurial" +SVN = SUBVERSION = "svn" +# mixed type is only actual when PatchSet contains +# Patches of different type +MIXED = MIXED = "mixed" + + +#------------------------------------------------ +# Helpers (these could come with Python stdlib) + +# x...() function are used to work with paths in +# cross-platform manner - all paths use forward +# slashes even on Windows. + +def xisabs(filename): + """ Cross-platform version of `os.path.isabs()` + Returns True if `filename` is absolute on + Linux, OS X or Windows. + """ + if filename.startswith('/'): # Linux/Unix + return True + elif filename.startswith('\\'): # Windows + return True + elif re.match(r'\w:[\\/]', filename): # Windows + return True + return False + +def xnormpath(path): + """ Cross-platform version of os.path.normpath """ + return os.path.normpath(path).replace(os.sep, '/') + +def xstrip(filename): + """ Make relative path out of absolute by stripping + prefixes used on Linux, OS X and Windows. + + This function is critical for security. + """ + while xisabs(filename): + # strip windows drive with all slashes + if re.match(r'\w:[\\/]', filename): + filename = re.sub(r'^\w+:[\\/]+', '', filename) + # strip all slashes + elif re.match(r'[\\/]', filename): + filename = re.sub(r'^[\\/]+', '', filename) + return filename + +#----------------------------------------------- +# Main API functions + +def fromfile(filename): + """ Parse patch file. If successful, returns + PatchSet() object. Otherwise returns False. + """ + patchset = PatchSet() + debug("reading %s" % filename) + fp = open(filename, "rb") + res = patchset.parse(fp) + fp.close() + if res == True: + return patchset + return False + + +def fromstring(s): + """ Parse text string and return PatchSet() + object (or False if parsing fails) + """ + ps = PatchSet( StringIO(s) ) + if ps.errors == 0: + return ps + return False + + +def fromurl(url): + """ Parse patch from an URL, return False + if an error occured. Note that this also + can throw urlopen() exceptions. + """ + ps = PatchSet( urllib2.urlopen(url) ) + if ps.errors == 0: + return ps + return False + + +# --- Utility functions --- +# [ ] reuse more universal pathsplit() +def pathstrip(path, n): + """ Strip n leading components from the given path """ + pathlist = [path] + while os.path.dirname(pathlist[0]) != '': + pathlist[0:1] = os.path.split(pathlist[0]) + return '/'.join(pathlist[n:]) +# --- /Utility function --- + + +class Hunk(object): + """ Parsed hunk data container (hunk starts with @@ -R +R @@) """ + + def __init__(self): + self.startsrc=None #: line count starts with 1 + self.linessrc=None + self.starttgt=None + self.linestgt=None + self.invalid=False + self.hasplus=False # True if any "+" lines in hunk + self.hasminus=False # True if any "-" lines in hunk + self.text=[] + + def originalText(self): + + return("@@ -" + str(self.startsrc) + + "," + str(self.linessrc) + + " +" + str(self.starttgt) + + "," + str(self.linestgt) + + "\n" + + self.printableText()) + + def printableText(self): + """Reformat text into printable text""" + + # yeah, there must be a better way to do this. + printable = "" + for line in self.text: + printable += line + + return printable + + + +# def apply(self, estream): +# """ write hunk data into enumerable stream +# return strings one by one until hunk is +# over +# +# enumerable stream are tuples (lineno, line) +# where lineno starts with 0 +# """ +# pass + + +class Patch(object): + """ Patch for a single file """ + def __init__(self): + self.source = None + self.target = None + self.hunks = [] + self.hunkends = [] + self.header = [] + + self.type = None + + +class PatchSet(object): + + def __init__(self, stream=None): + # --- API accessible fields --- + + # name of the PatchSet (filename or ...) + self.name = None + # patch set type - one of constants + self.type = None + + # list of Patch objects + self.items = [] + + self.errors = 0 # fatal parsing errors + self.warnings = 0 # non-critical warnings + # --- /API --- + + if stream: + self.parse(stream) + + def __len__(self): + return len(self.items) + + def parse(self, stream): + """ parse unified diff + return True on success + """ + lineends = dict(lf=0, crlf=0, cr=0) + nexthunkno = 0 #: even if index starts with 0 user messages number hunks from 1 + + p = None + hunk = None + # hunkactual variable is used to calculate hunk lines for comparison + hunkactual = dict(linessrc=None, linestgt=None) + + + class wrapumerate(enumerate): + """Enumerate wrapper that uses boolean end of stream status instead of + StopIteration exception, and properties to access line information. + """ + + def __init__(self, *args, **kwargs): + # we don't call parent, it is magically created by __new__ method + + self._exhausted = False + self._lineno = False # after end of stream equal to the num of lines + self._line = False # will be reset to False after end of stream + + def next(self): + """Try to read the next line and return True if it is available, + False if end of stream is reached.""" + if self._exhausted: + return False + + try: + self._lineno, self._line = super(wrapumerate, self).next() + except StopIteration: + self._exhausted = True + self._line = False + return False + return True + + @property + def is_empty(self): + return self._exhausted + + @property + def line(self): + return self._line + + @property + def lineno(self): + return self._lineno + + # define states (possible file regions) that direct parse flow + headscan = True # start with scanning header + filenames = False # lines starting with --- and +++ + + hunkhead = False # @@ -R +R @@ sequence + hunkbody = False # + hunkskip = False # skipping invalid hunk mode + + hunkparsed = False # state after successfully parsed hunk + + # regexp to match start of hunk, used groups - 1,3,4,6 + re_hunk_start = re.compile("^@@ -(\d+)(,(\d+))? \+(\d+)(,(\d+))?") + + self.errors = 0 + # temp buffers for header and filenames info + header = [] + srcname = None + tgtname = None + + # start of main cycle + # each parsing block already has line available in fe.line + fe = wrapumerate(stream) + while fe.next(): + + # -- deciders: these only switch state to decide who should process + # -- line fetched at the start of this cycle + if hunkparsed: + hunkparsed = False + if re_hunk_start.match(fe.line): + hunkhead = True + elif fe.line.startswith("--- "): + filenames = True + else: + headscan = True + # -- ------------------------------------ + + # read out header + if headscan: + while not fe.is_empty and not fe.line.startswith("--- "): + header.append(fe.line) + fe.next() + if fe.is_empty: + if p == None: + debug("no patch data found") # error is shown later + self.errors += 1 + else: + info("%d unparsed bytes left at the end of stream" % len(''.join(header))) + self.warnings += 1 + # TODO check for \No new line at the end.. + # TODO test for unparsed bytes + # otherwise error += 1 + # this is actually a loop exit + continue + + headscan = False + # switch to filenames state + filenames = True + + line = fe.line + lineno = fe.lineno + + + # hunkskip and hunkbody code skipped until definition of hunkhead is parsed + if hunkbody: + # [x] treat empty lines inside hunks as containing single space + # (this happens when diff is saved by copy/pasting to editor + # that strips trailing whitespace) + if line.strip("\r\n") == "": + debug("expanding empty line in a middle of hunk body") + self.warnings += 1 + line = ' ' + line + + # process line first + if re.match(r"^[- \+\\]", line): + # gather stats about line endings + if line.endswith("\r\n"): + p.hunkends["crlf"] += 1 + elif line.endswith("\n"): + p.hunkends["lf"] += 1 + elif line.endswith("\r"): + p.hunkends["cr"] += 1 + + if line.startswith("-"): + hunkactual["linessrc"] += 1 + hunk.hasminus = True + elif line.startswith("+"): + hunkactual["linestgt"] += 1 + hunk.hasplus = True + elif not line.startswith("\\"): + hunkactual["linessrc"] += 1 + hunkactual["linestgt"] += 1 + hunk.text.append(line) + # todo: handle \ No newline cases + else: + warning("invalid hunk no.%d at %d for target file %s" % (nexthunkno, lineno+1, p.target)) + # add hunk status node + hunk.invalid = True + p.hunks.append(hunk) + self.errors += 1 + # switch to hunkskip state + hunkbody = False + hunkskip = True + + # check exit conditions + if hunkactual["linessrc"] > hunk.linessrc or hunkactual["linestgt"] > hunk.linestgt: + warning("extra lines for hunk no.%d at %d for target %s" % (nexthunkno, lineno+1, p.target)) + # add hunk status node + hunk.invalid = True + p.hunks.append(hunk) + self.errors += 1 + # switch to hunkskip state + hunkbody = False + hunkskip = True + elif hunk.linessrc == hunkactual["linessrc"] and hunk.linestgt == hunkactual["linestgt"]: + # hunk parsed successfully + p.hunks.append(hunk) + # switch to hunkparsed state + hunkbody = False + hunkparsed = True + + # detect mixed window/unix line ends + ends = p.hunkends + if ((ends["cr"]!=0) + (ends["crlf"]!=0) + (ends["lf"]!=0)) > 1: + warning("inconsistent line ends in patch hunks for %s" % p.source) + self.warnings += 1 + if debugmode: + debuglines = dict(ends) + debuglines.update(file=p.target, hunk=nexthunkno) + debug("crlf: %(crlf)d lf: %(lf)d cr: %(cr)d\t - file: %(file)s hunk: %(hunk)d" % debuglines) + # fetch next line + continue + + if hunkskip: + if re_hunk_start.match(line): + # switch to hunkhead state + hunkskip = False + hunkhead = True + elif line.startswith("--- "): + # switch to filenames state + hunkskip = False + filenames = True + if debugmode and len(self.items) > 0: + debug("- %2d hunks for %s" % (len(p.hunks), p.source)) + + if filenames: + if line.startswith("--- "): + if srcname != None: + # XXX testcase + warning("skipping false patch for %s" % srcname) + srcname = None + # XXX header += srcname + # double source filename line is encountered + # attempt to restart from this second line + re_filename = "^--- ([^\t]+)" + match = re.match(re_filename, line) + # todo: support spaces in filenames + if match: + srcname = match.group(1).strip() + else: + warning("skipping invalid filename at line %d" % lineno) + self.errors += 1 + # XXX p.header += line + # switch back to headscan state + filenames = False + headscan = True + elif not line.startswith("+++ "): + if srcname != None: + warning("skipping invalid patch with no target for %s" % srcname) + self.errors += 1 + srcname = None + # XXX header += srcname + # XXX header += line + else: + # this should be unreachable + warning("skipping invalid target patch") + filenames = False + headscan = True + else: + if tgtname != None: + # XXX seems to be a dead branch + warning("skipping invalid patch - double target at line %d" % lineno) + self.errors += 1 + srcname = None + tgtname = None + # XXX header += srcname + # XXX header += tgtname + # XXX header += line + # double target filename line is encountered + # switch back to headscan state + filenames = False + headscan = True + else: + re_filename = "^\+\+\+ ([^\t]+)" + match = re.match(re_filename, line) + if not match: + warning("skipping invalid patch - no target filename at line %d" % lineno) + self.errors += 1 + srcname = None + # switch back to headscan state + filenames = False + headscan = True + else: + if p: # for the first run p is None + self.items.append(p) + p = Patch() + p.source = srcname + srcname = None + p.target = match.group(1).strip() + p.header = header + header = [] + # switch to hunkhead state + filenames = False + hunkhead = True + nexthunkno = 0 + p.hunkends = lineends.copy() + continue + + if hunkhead: + match = re.match("^@@ -(\d+)(,(\d+))? \+(\d+)(,(\d+))?", line) + if not match: + if not p.hunks: + warning("skipping invalid patch with no hunks for file %s" % p.source) + self.errors += 1 + # XXX review switch + # switch to headscan state + hunkhead = False + headscan = True + continue + else: + # TODO review condition case + # switch to headscan state + hunkhead = False + headscan = True + else: + hunk = Hunk() + hunk.startsrc = int(match.group(1)) + hunk.linessrc = 1 + if match.group(3): hunk.linessrc = int(match.group(3)) + hunk.starttgt = int(match.group(4)) + hunk.linestgt = 1 + if match.group(6): hunk.linestgt = int(match.group(6)) + hunk.invalid = False + hunk.text = [] + + hunkactual["linessrc"] = hunkactual["linestgt"] = 0 + + # switch to hunkbody state + hunkhead = False + hunkbody = True + nexthunkno += 1 + continue + + # /while fe.next() + + if p: + self.items.append(p) + + if not hunkparsed: + if hunkskip: + warning("warning: finished with errors, some hunks may be invalid") + elif headscan: + if len(self.items) == 0: + warning("error: no patch data found!") + return False + else: # extra data at the end of file + pass + else: + warning("error: patch stream is incomplete!") + self.errors += 1 + if len(self.items) == 0: + return False + + if debugmode and len(self.items) > 0: + debug("- %2d hunks for %s" % (len(p.hunks), p.source)) + + # XXX fix total hunks calculation + debug("total files: %d total hunks: %d" % (len(self.items), + sum(len(p.hunks) for p in self.items))) + + # ---- detect patch and patchset types ---- + for idx, p in enumerate(self.items): + self.items[idx].type = self._detect_type(p) + + types = set([p.type for p in self.items]) + if len(types) > 1: + self.type = MIXED + else: + self.type = types.pop() + # -------- + + self._normalize_filenames() + + return (self.errors == 0) + + def _detect_type(self, p): + """ detect and return type for the specified Patch object + analyzes header and filenames info + + NOTE: must be run before filenames are normalized + """ + + # check for SVN + # - header starts with Index: + # - next line is ===... delimiter + # - filename is followed by revision number + # TODO add SVN revision + if (len(p.header) > 1 and p.header[-2].startswith("Index: ") + and p.header[-1].startswith("="*67)): + return SVN + + # common checks for both HG and GIT + DVCS = ((p.source.startswith('a/') or p.source == '/dev/null') + and (p.target.startswith('b/') or p.target == '/dev/null')) + + # GIT type check + # - header[-2] is like "diff --git a/oldname b/newname" + # - header[-1] is like "index <hash>..<hash><mode>" + # TODO add git rename diffs and add/remove diffs + # add git diff with spaced filename + # TODO http://www.kernel.org/pub/software/scm/git/docs/git-diff.html + + # detect the start of diff header - there might be some comments before + if len(p.header) > 1: + for idx in reversed(range(len(p.header))): + if p.header[idx].startswith("diff --git"): + break + if re.match(r'diff --git a/[\w/.]+ b/[\w/.]+', p.header[idx]): + if (idx+1 < len(p.header) + and re.match(r'index \w{7}..\w{7} \d{6}', p.header[idx+1])): + if DVCS: + return GIT + + # HG check + # + # - for plain HG format header is like "diff -r b2d9961ff1f5 filename" + # - for Git-style HG patches it is "diff --git a/oldname b/newname" + # - filename starts with a/, b/ or is equal to /dev/null + # - exported changesets also contain the header + # # HG changeset patch + # # User name@example.com + # ... + # TODO add MQ + # TODO add revision info + if len(p.header) > 0: + if DVCS and re.match(r'diff -r \w{12} .*', p.header[-1]): + return HG + if DVCS and p.header[-1].startswith('diff --git a/'): + if len(p.header) == 1: # native Git patch header len is 2 + return HG + elif p.header[0].startswith('# HG changeset patch'): + return HG + + return PLAIN + + + def _normalize_filenames(self): + """ sanitize filenames, normalizing paths, i.e.: + 1. strip a/ and b/ prefixes from GIT and HG style patches + 2. remove all references to parent directories (with warning) + 3. translate any absolute paths to relative (with warning) + + [x] always use forward slashes to be crossplatform + (diff/patch were born as a unix utility after all) + + return None + """ + for i,p in enumerate(self.items): + if p.type in (HG, GIT): + # TODO: figure out how to deal with /dev/null entries + debug("stripping a/ and b/ prefixes") + if p.source != '/dev/null': + if not p.source.startswith("a/"): + warning("invalid source filename") + else: + p.source = p.source[2:] + if p.target != '/dev/null': + if not p.target.startswith("b/"): + warning("invalid target filename") + else: + p.target = p.target[2:] + + p.source = xnormpath(p.source) + p.target = xnormpath(p.target) + + sep = '/' # sep value can be hardcoded, but it looks nice this way + + # references to parent are not allowed + if p.source.startswith(".." + sep): + warning("error: stripping parent path for source file patch no.%d" % (i+1)) + self.warnings += 1 + while p.source.startswith(".." + sep): + p.source = p.source.partition(sep)[2] + if p.target.startswith(".." + sep): + warning("error: stripping parent path for target file patch no.%d" % (i+1)) + self.warnings += 1 + while p.target.startswith(".." + sep): + p.target = p.target.partition(sep)[2] + # absolute paths are not allowed + if xisabs(p.source) or xisabs(p.target): + warning("error: absolute paths are not allowed - file no.%d" % (i+1)) + self.warnings += 1 + if xisabs(p.source): + warning("stripping absolute path from source name '%s'" % p.source) + p.source = xstrip(p.source) + if xisabs(p.target): + warning("stripping absolute path from target name '%s'" % p.target) + p.target = xstrip(p.target) + + self.items[i].source = p.source + self.items[i].target = p.target + + + def diffstat(self): + """ calculate diffstat and return as a string + Notes: + - original diffstat ouputs target filename + - single + or - shouldn't escape histogram + """ + names = [] + insert = [] + delete = [] + namelen = 0 + maxdiff = 0 # max number of changes for single file + # (for histogram width calculation) + for patch in self.items: + i,d = 0,0 + for hunk in patch.hunks: + for line in hunk.text: + if line.startswith('+'): + i += 1 + elif line.startswith('-'): + d += 1 + names.append(patch.target) + insert.append(i) + delete.append(d) + namelen = max(namelen, len(patch.target)) + maxdiff = max(maxdiff, i+d) + output = '' + statlen = len(str(maxdiff)) # stats column width + for i,n in enumerate(names): + # %-19s | %-4d %s + format = " %-" + str(namelen) + "s | %" + str(statlen) + "s %s\n" + + hist = '' + # -- calculating histogram -- + width = len(format % ('', '', '')) + histwidth = max(2, 80 - width) + if maxdiff < histwidth: + hist = "+"*insert[i] + "-"*delete[i] + else: + iratio = (float(insert[i]) / maxdiff) * histwidth + dratio = (float(delete[i]) / maxdiff) * histwidth + + # make sure every entry gets at least one + or - + iwidth = 1 if 0 < iratio < 1 else int(iratio) + dwidth = 1 if 0 < dratio < 1 else int(dratio) + #print iratio, dratio, iwidth, dwidth, histwidth + hist = "+"*int(iwidth) + "-"*int(dwidth) + # -- /calculating +- histogram -- + output += (format % (names[i], insert[i] + delete[i], hist)) + + output += (" %d files changed, %d insertions(+), %d deletions(-)" + % (len(names), sum(insert), sum(delete))) + return output + + + def apply(self, strip=0): + """ apply parsed patch + return True on success + """ + + total = len(self.items) + errors = 0 + if strip: + # [ ] test strip level exceeds nesting level + # [ ] test the same only for selected files + # [ ] test if files end up being on the same level + try: + strip = int(strip) + except ValueError: + errors += 1 + warning("error: strip parameter '%s' must be an integer" % strip) + strip = 0 + + #for fileno, filename in enumerate(self.source): + for i,p in enumerate(self.items): + f2patch = p.source + if strip: + debug("stripping %s leading component from '%s'" % (strip, f2patch)) + f2patch = pathstrip(f2patch, strip) + if not exists(f2patch): + f2patch = p.target + if strip: + debug("stripping %s leading component from '%s'" % (strip, f2patch)) + f2patch = pathstrip(f2patch, strip) + if not exists(f2patch): + warning("source/target file does not exist\n--- %s\n+++ %s" % (p.source, f2patch)) + errors += 1 + continue + if not isfile(f2patch): + warning("not a file - %s" % f2patch) + errors += 1 + continue + filename = f2patch + + debug("processing %d/%d:\t %s" % (i+1, total, filename)) + + # validate before patching + f2fp = open(filename) + hunkno = 0 + hunk = p.hunks[hunkno] + hunkfind = [] + hunkreplace = [] + validhunks = 0 + canpatch = False + for lineno, line in enumerate(f2fp): + if lineno+1 < hunk.startsrc: + continue + elif lineno+1 == hunk.startsrc: + hunkfind = [x[1:].rstrip("\r\n") for x in hunk.text if x[0] in " -"] + hunkreplace = [x[1:].rstrip("\r\n") for x in hunk.text if x[0] in " +"] + #pprint(hunkreplace) + hunklineno = 0 + + # todo \ No newline at end of file + + # check hunks in source file + if lineno+1 < hunk.startsrc+len(hunkfind)-1: + if line.rstrip("\r\n") == hunkfind[hunklineno]: + hunklineno+=1 + else: + info("file %d/%d:\t %s" % (i+1, total, filename)) + info(" hunk no.%d doesn't match source file at line %d" % (hunkno+1, lineno)) + info(" expected: %s" % hunkfind[hunklineno]) + info(" actual : %s" % line.rstrip("\r\n")) + # not counting this as error, because file may already be patched. + # check if file is already patched is done after the number of + # invalid hunks if found + # TODO: check hunks against source/target file in one pass + # API - check(stream, srchunks, tgthunks) + # return tuple (srcerrs, tgterrs) + + # continue to check other hunks for completeness + hunkno += 1 + if hunkno < len(p.hunks): + hunk = p.hunks[hunkno] + continue + else: + break + + # check if processed line is the last line + if lineno+1 == hunk.startsrc+len(hunkfind)-1: + debug(" hunk no.%d for file %s -- is ready to be patched" % (hunkno+1, filename)) + hunkno+=1 + validhunks+=1 + if hunkno < len(p.hunks): + hunk = p.hunks[hunkno] + else: + if validhunks == len(p.hunks): + # patch file + canpatch = True + break + else: + if hunkno < len(p.hunks): + warning("premature end of source file %s at hunk %d" % (filename, hunkno+1)) + errors += 1 + + f2fp.close() + + if validhunks < len(p.hunks): + if self._match_file_hunks(filename, p.hunks): + warning("already patched %s" % filename) + else: + warning("source file is different - %s" % filename) + errors += 1 + if canpatch: + backupname = filename+".orig" + if exists(backupname): + warning("can't backup original file to %s - aborting" % backupname) + else: + import shutil + shutil.move(filename, backupname) + if self.write_hunks(backupname, filename, p.hunks): + info("successfully patched %d/%d:\t %s" % (i+1, total, filename)) + os.unlink(backupname) + else: + errors += 1 + warning("error patching file %s" % filename) + shutil.copy(filename, filename+".invalid") + warning("invalid version is saved to %s" % filename+".invalid") + # todo: proper rejects + shutil.move(backupname, filename) + + # todo: check for premature eof + return (errors == 0) + + + def can_patch(self, filename): + """ Check if specified filename can be patched. Returns None if file can + not be found among source filenames. False if patch can not be applied + clearly. True otherwise. + + :returns: True, False or None + """ + filename = abspath(filename) + for p in self.items: + if filename == abspath(p.source): + return self._match_file_hunks(filename, p.hunks) + return None + + + def _match_file_hunks(self, filepath, hunks): + matched = True + fp = open(abspath(filepath)) + + class NoMatch(Exception): + pass + + lineno = 1 + line = fp.readline() + hno = None + try: + for hno, h in enumerate(hunks): + # skip to first line of the hunk + while lineno < h.starttgt: + if not len(line): # eof + debug("check failed - premature eof before hunk: %d" % (hno+1)) + raise NoMatch + line = fp.readline() + lineno += 1 + for hline in h.text: + if hline.startswith("-"): + continue + if not len(line): + debug("check failed - premature eof on hunk: %d" % (hno+1)) + # todo: \ No newline at the end of file + raise NoMatch + if line.rstrip("\r\n") != hline[1:].rstrip("\r\n"): + debug("file is not patched - failed hunk: %d" % (hno+1)) + raise NoMatch + line = fp.readline() + lineno += 1 + + except NoMatch: + matched = False + # todo: display failed hunk, i.e. expected/found + + fp.close() + return matched + + + def patch_stream(self, instream, hunks): + """ Generator that yields stream patched with hunks iterable + + Converts lineends in hunk lines to the best suitable format + autodetected from input + """ + + # todo: At the moment substituted lineends may not be the same + # at the start and at the end of patching. Also issue a + # warning/throw about mixed lineends (is it really needed?) + + hunks = iter(hunks) + + srclineno = 1 + + lineends = {'\n':0, '\r\n':0, '\r':0} + def get_line(): + """ + local utility function - return line from source stream + collecting line end statistics on the way + """ + line = instream.readline() + # 'U' mode works only with text files + if line.endswith("\r\n"): + lineends["\r\n"] += 1 + elif line.endswith("\n"): + lineends["\n"] += 1 + elif line.endswith("\r"): + lineends["\r"] += 1 + return line + + for hno, h in enumerate(hunks): + debug("hunk %d" % (hno+1)) + if h.hasminus: + warning("Change removes/replaces some text; INVESTIGATE AND APPLY (OR NOT) MANUALLY") + warning("Change:") + changeText = h.originalText() + if len(changeText) > 1000: + changeText = changeText[0:999] + "...\n" + warning(changeText) + else: + # skip to line just before hunk starts + while srclineno < h.startsrc: + yield get_line() + srclineno += 1 + + for hline in h.text: + # todo: check \ No newline at the end of file + if hline.startswith("-") or hline.startswith("\\"): + get_line() + srclineno += 1 + continue + else: + if not hline.startswith("+"): + get_line() + srclineno += 1 + line2write = hline[1:] + # detect if line ends are consistent in source file + if sum([bool(lineends[x]) for x in lineends]) == 1: + newline = [x for x in lineends if lineends[x] != 0][0] + yield line2write.rstrip("\r\n")+newline + else: # newlines are mixed + yield line2write + + for line in instream: + yield line + + + def write_hunks(self, srcname, tgtname, hunks): + src = open(srcname, "rb") + tgt = open(tgtname, "wb") + + debug("processing target file %s" % tgtname) + + tgt.writelines(self.patch_stream(src, hunks)) + + tgt.close() + src.close() + # [ ] TODO: add test for permission copy + shutil.copymode(srcname, tgtname) + return True + + + +if __name__ == "__main__": + from optparse import OptionParser + from os.path import exists + import sys + + opt = OptionParser(usage="1. %prog [options] unified.diff\n" + " 2. %prog [options] http://host/patch\n" + " 3. %prog [options] -- < unified.diff", + version="python-patch %s" % __version__) + opt.add_option("-q", "--quiet", action="store_const", dest="verbosity", + const=0, help="print only warnings and errors", default=1) + opt.add_option("-v", "--verbose", action="store_const", dest="verbosity", + const=2, help="be verbose") + opt.add_option("--debug", action="store_true", dest="debugmode", help="debug mode") + opt.add_option("--diffstat", action="store_true", dest="diffstat", + help="print diffstat and exit") + opt.add_option("-p", "--strip", type="int", metavar='N', default=0, + help="strip N path components from filenames") + (options, args) = opt.parse_args() + + if not args and sys.argv[-1:] != ['--']: + opt.print_version() + opt.print_help() + sys.exit() + readstdin = (sys.argv[-1:] == ['--'] and not args) + + debugmode = options.debugmode + + verbosity_levels = {0:logging.WARNING, 1:logging.INFO, 2:logging.DEBUG} + loglevel = verbosity_levels[options.verbosity] + logformat = "%(message)s" + if debugmode: + loglevel = logging.DEBUG + logformat = "%(levelname)8s %(message)s" + logger.setLevel(loglevel) + loghandler = logging.StreamHandler() + loghandler.setFormatter(logging.Formatter(logformat)) + logger.addHandler(loghandler) + + + if readstdin: + patch = PatchSet(sys.stdin) + else: + patchfile = args[0] + urltest = patchfile.split(':')[0] + if (':' in patchfile and urltest.isalpha() + and len(urltest) > 1): # one char before : is a windows drive letter + patch = fromurl(patchfile) + else: + if not exists(patchfile) or not isfile(patchfile): + sys.exit("patch file does not exist - %s" % patchfile) + patch = fromfile(patchfile) + + if options.diffstat: + print patch.diffstat() + sys.exit(0) + + #pprint(patch) + patch.apply(options.strip) or sys.exit(-1) + + # todo: document and test line ends handling logic - patch.py detects proper line-endings + # for inserted hunks and issues a warning if patched file has incosistent line ends diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -274,6 +274,6 @@ return Mock() # adding pbs_python, DRMAA_python, markupsafe, and drmaa here had no effect. -MOCK_MODULES = ['tables', 'decorator'] +MOCK_MODULES = ['tables', 'decorator', 'numpy'] for mod_name in MOCK_MODULES: sys.modules[mod_name] = Mock() diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,12 +1,41 @@ Galaxy Code Documentation ************************* -Galaxy is an open, web-based platform for accessible, reproducible, and +Galaxy_ is an open, web-based platform for accessible, reproducible, and transparent computational biomedical research. -- Accessible: Users without programming experience can easily specify parameters and run tools and workflows. -- Reproducible: Galaxy captures information so that any user can repeat and understand a complete computational analysis. -- Transparent: Users share and publish analyses via the web and create Pages, interactive, web-based documents that describe a complete analysis. +- *Accessible:* Users without programming experience can easily specify parameters and run tools and workflows. +- *Reproducible:* Galaxy captures information so that any user can repeat and understand a complete computational analysis. +- *Transparent:* Users share and publish analyses via the web and create Pages, interactive, web-based documents that describe a complete analysis. + +Two copies of the Galaxy code doumentation are published by the Galaxy Project + +- Galaxy-Dist_: This describes the code in the `most recent official release`_ of Galaxy. +- Galaxy-Central_: Describes the `current code in the development branch`_ of Galaxy. This is the latest checkin, bleeding edge version of the code. The documentation should never be more than an hour behind the code. + +Both copies are hosted at ReadTheDocs_, a publicly supported web site for hosting project documentation. + +If you have your own copy of the Galaxy source code, you can also generate your own version of this documentation: + +:: + + $ cd doc + $ make html + +The generated documentation will be in ``doc/build/html/`` and can be viewed with a web browser. Note that you will need to install Sphinx and a fair number of module dependencies before this will produce output. + +.. _Galaxy: http://galaxyproject.org/ +.. _Galaxy-Dist: https://galaxy-dist.readthedocs.org/ +.. _most recent official release: https://bitbucket.org/galaxy/galaxy-dist +.. _Galaxy-Central: https://galaxy-central.readthedocs.org/ +.. _current code in the development branch: https://bitbucket.org/galaxy/galaxy-central +.. _ReadTheDocs: https://readthedocs.org/ + + +For more on the Galaxy Project, please visit the `project home page`_. + +.. _project home page: http://galaxyproject.org/ + Contents ======== diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 doc/source/lib/galaxy.jobs.rst --- a/doc/source/lib/galaxy.jobs.rst +++ b/doc/source/lib/galaxy.jobs.rst @@ -48,6 +48,7 @@ galaxy.jobs.actions galaxy.jobs.deferred + galaxy.jobs.rules galaxy.jobs.runners galaxy.jobs.splitters diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 doc/source/lib/galaxy.jobs.runners.rst --- a/doc/source/lib/galaxy.jobs.runners.rst +++ b/doc/source/lib/galaxy.jobs.runners.rst @@ -57,14 +57,6 @@ :undoc-members: :show-inheritance: -:mod:`sge` Module ------------------ - -.. automodule:: galaxy.jobs.runners.sge - :members: - :undoc-members: - :show-inheritance: - :mod:`tasks` Module ------------------- diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 doc/source/lib/galaxy.tool_shed.rst --- a/doc/source/lib/galaxy.tool_shed.rst +++ b/doc/source/lib/galaxy.tool_shed.rst @@ -9,6 +9,14 @@ :undoc-members: :show-inheritance: +:mod:`common_util` Module +------------------------- + +.. automodule:: galaxy.tool_shed.common_util + :members: + :undoc-members: + :show-inheritance: + :mod:`encoding_util` Module --------------------------- diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 doc/source/lib/galaxy.util.rst --- a/doc/source/lib/galaxy.util.rst +++ b/doc/source/lib/galaxy.util.rst @@ -25,6 +25,14 @@ :undoc-members: :show-inheritance: +:mod:`debugging` Module +----------------------- + +.. automodule:: galaxy.util.debugging + :members: + :undoc-members: + :show-inheritance: + :mod:`expressions` Module ------------------------- @@ -113,6 +121,14 @@ :undoc-members: :show-inheritance: +:mod:`shed_util_common` Module +------------------------------ + +.. automodule:: galaxy.util.shed_util_common + :members: + :undoc-members: + :show-inheritance: + :mod:`streamball` Module ------------------------ diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 doc/source/lib/galaxy.webapps.community.rst --- a/doc/source/lib/galaxy.webapps.community.rst +++ b/doc/source/lib/galaxy.webapps.community.rst @@ -42,4 +42,5 @@ galaxy.webapps.community.framework galaxy.webapps.community.model galaxy.webapps.community.security + galaxy.webapps.community.util diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 doc/source/lib/galaxy.webapps.community.util.rst --- /dev/null +++ b/doc/source/lib/galaxy.webapps.community.util.rst @@ -0,0 +1,27 @@ +util Package +============ + +:mod:`container_util` Module +---------------------------- + +.. automodule:: galaxy.webapps.community.util.container_util + :members: + :undoc-members: + :show-inheritance: + +:mod:`hgweb_config` Module +-------------------------- + +.. automodule:: galaxy.webapps.community.util.hgweb_config + :members: + :undoc-members: + :show-inheritance: + +:mod:`shed_statistics` Module +----------------------------- + +.. automodule:: galaxy.webapps.community.util.shed_statistics + :members: + :undoc-members: + :show-inheritance: + diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 doc/source/lib/galaxy.webapps.galaxy.api.rst --- a/doc/source/lib/galaxy.webapps.galaxy.api.rst +++ b/doc/source/lib/galaxy.webapps.galaxy.api.rst @@ -293,6 +293,14 @@ :undoc-members: :show-inheritance: +:mod:`item_tags` Module +----------------------- + +.. automodule:: galaxy.webapps.galaxy.api.item_tags + :members: + :undoc-members: + :show-inheritance: + :mod:`libraries` Module ----------------------- diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 eggs.ini --- a/eggs.ini +++ b/eggs.ini @@ -29,6 +29,7 @@ simplejson = 2.1.1 threadframe = 0.2 guppy = 0.1.8 +; msgpack_python = 0.2.4 [eggs:noplatform] amqplib = 0.6.1 @@ -65,6 +66,7 @@ Babel = 0.9.4 wchartype = 0.1 Whoosh = 0.3.18 +; fluent_logger = 0.3.3 ; extra version information [tags] diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -23,11 +23,13 @@ """Encapsulates the state of a Universe application""" def __init__( self, **kwargs ): print >> sys.stderr, "python path is: " + ", ".join( sys.path ) + self.name = 'galaxy' self.new_installation = False # Read config file and check for errors self.config = config.Configuration( **kwargs ) self.config.check() config.configure_logging( self.config ) + self.configure_fluent_log() # Determine the database url if self.config.database_connection: db_url = self.config.database_connection @@ -53,7 +55,8 @@ db_url, self.config.database_engine_options, database_query_profiling_proxy = self.config.database_query_profiling_proxy, - object_store = self.object_store ) + object_store = self.object_store, + trace_logger=self.trace_logger ) # Manage installed tool shed repositories. self.installed_repository_manager = galaxy.tool_shed.InstalledRepositoryManager( self ) # Create an empty datatypes registry. @@ -148,6 +151,7 @@ self.job_stop_queue = self.job_manager.job_stop_queue # Initialize the external service types self.external_service_types = external_service_types.ExternalServiceTypesCollection( self.config.external_service_type_config_file, self.config.external_service_type_path, self ) + def shutdown( self ): self.job_manager.shutdown() self.object_store.shutdown() @@ -160,3 +164,10 @@ os.unlink( self.datatypes_registry.integrated_datatypes_configs ) except: pass + + def configure_fluent_log( self ): + if self.config.fluent_log: + from galaxy.util.log.fluent_log import FluentTraceLogger + self.trace_logger = FluentTraceLogger( 'galaxy', self.config.fluent_host, self.config.fluent_port ) + else: + self.trace_logger = None diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -261,6 +261,10 @@ self.api_folders = string_as_bool( kwargs.get( 'api_folders', False ) ) # This is for testing new library browsing capabilities. self.new_lib_browse = string_as_bool( kwargs.get( 'new_lib_browse', False ) ) + # Logging with fluentd + self.fluent_log = string_as_bool( kwargs.get( 'fluent_log', False ) ) + self.fluent_host = kwargs.get( 'fluent_host', 'localhost' ) + self.fluent_port = int( kwargs.get( 'fluent_port', 24224 ) ) def __read_tool_job_config( self, global_conf_parser, section, key ): try: diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -114,7 +114,8 @@ self.datatype_elems.remove( in_memory_elem ) else: # Keep an in-memory list of datatype elems to enable persistence. - self.datatype_elems.append( elem ) + if extension not in self.datatypes_by_extension: + self.datatype_elems.append( elem ) if extension and extension in self.datatypes_by_extension and deactivate: # We are deactivating an installed tool shed repository, so eliminate the datatype from the registry. # TODO: Handle deactivating datatype converters, etc before removing from self.datatypes_by_extension. diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 lib/galaxy/datatypes/tabular.py --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -267,20 +267,20 @@ def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None): if chunk: return self.get_chunk(trans, dataset, chunk) + elif to_ext or not preview: + return self._serve_raw(trans, dataset, to_ext) elif dataset.metadata.columns > 50: #Fancy tabular display is only suitable for datasets without an incredibly large number of columns. #We should add a new datatype 'matrix', with it's own draw method, suitable for this kind of data. #For now, default to the old behavior, ugly as it is. Remove this after adding 'matrix'. max_peek_size = 1000000 # 1 MB - if not preview or os.stat( dataset.file_name ).st_size < max_peek_size: + if os.stat( dataset.file_name ).st_size < max_peek_size: return open( dataset.file_name ) else: trans.response.set_content_type( "text/html" ) return trans.stream_template_mako( "/dataset/large_file.mako", truncated_data = open( dataset.file_name ).read(max_peek_size), data = dataset) - elif to_ext or not preview: - return self._serve_raw(trans, dataset, to_ext) else: column_names = 'null' if dataset.metadata.column_names: diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -686,7 +686,7 @@ if self.app.config.set_metadata_externally: self.external_output_metadata.cleanup_external_metadata( self.sa_session ) galaxy.tools.imp_exp.JobExportHistoryArchiveWrapper( self.job_id ).cleanup_after_job( self.sa_session ) - galaxy.tools.imp_exp.JobImportHistoryArchiveWrapper( self.job_id ).cleanup_after_job( self.sa_session ) + galaxy.tools.imp_exp.JobImportHistoryArchiveWrapper( self.app, self.job_id ).cleanup_after_job() galaxy.tools.genome_index.GenomeIndexToolWrapper( self.job_id ).postprocessing( self.sa_session, self.app ) self.app.object_store.delete(self.get_job(), base_dir='job_work', entire_dir=True, dir_only=True, extra_dir=str(self.job_id)) except: diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 lib/galaxy/jobs/handler.py --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -61,7 +61,8 @@ # Helper for interruptable sleep self.sleeper = Sleeper() self.running = True - self.monitor_thread = threading.Thread( target=self.__monitor ) + self.monitor_thread = threading.Thread( name="JobHandlerQueue.monitor_thread", target=self.__monitor ) + self.monitor_thread.setDaemon( True ) def start( self ): """ @@ -353,7 +354,8 @@ # Helper for interruptable sleep self.sleeper = Sleeper() self.running = True - self.monitor_thread = threading.Thread( target=self.monitor ) + self.monitor_thread = threading.Thread( name="JobHandlerStopQueue.monitor_thread", target=self.monitor ) + self.monitor_thread.setDaemon( True ) self.monitor_thread.start() log.info( "job handler stop queue started" ) diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 lib/galaxy/jobs/manager.py --- a/lib/galaxy/jobs/manager.py +++ b/lib/galaxy/jobs/manager.py @@ -68,7 +68,8 @@ # Helper for interruptable sleep self.sleeper = Sleeper() self.running = True - self.monitor_thread = threading.Thread( target=self.__monitor ) + self.monitor_thread = threading.Thread( name="JobManagerQueue.monitor_thread", target=self.__monitor ) + self.monitor_thread.setDaemon( True ) # Recover jobs at startup self.__check_jobs_at_startup() # Start the queue @@ -219,7 +220,8 @@ # Helper for interruptable sleep self.sleeper = Sleeper() self.running = True - self.monitor_thread = threading.Thread( target=self.monitor ) + self.monitor_thread = threading.Thread( name="JobManagerStopQueue.monitor_thread", target=self.monitor ) + self.monitor_thread.setDaemon( True ) self.monitor_thread.start() log.info( "job manager stop queue started" ) diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 lib/galaxy/jobs/runners/drmaa.py --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -105,13 +105,14 @@ self.monitor_queue = Queue() self.ds = drmaa.Session() self.ds.initialize() - self.monitor_thread = threading.Thread( target=self.monitor ) + self.monitor_thread = threading.Thread( name="DRMAAJobRunner.monitor_thread", target=self.monitor ) + self.monitor_thread.setDaemon( True ) self.monitor_thread.start() self.work_queue = Queue() self.work_threads = [] nworkers = app.config.cluster_job_queue_workers for i in range( nworkers ): - worker = threading.Thread( target=self.run_next ) + worker = threading.Thread( name=( "DRMAAJobRunner.work_threads-%d" % i ), target=self.run_next ) worker.start() self.work_threads.append( worker ) log.debug( "%d workers ready" % nworkers ) diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 lib/galaxy/jobs/runners/local.py --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -37,7 +37,8 @@ nworkers = app.config.local_job_queue_workers log.info( "starting workers" ) for i in range( nworkers ): - worker = threading.Thread( target=self.run_next ) + worker = threading.Thread( name=( "LocalJobRunner.threads-%d" % i ), target=self.run_next ) + worker.setDaemon( True ) worker.start() self.threads.append( worker ) log.debug( "%d workers ready", nworkers ) diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 lib/galaxy/jobs/runners/lwr.py --- a/lib/galaxy/jobs/runners/lwr.py +++ b/lib/galaxy/jobs/runners/lwr.py @@ -229,7 +229,8 @@ nworkers = app.config.local_job_queue_workers log.info( "starting workers" ) for i in range( nworkers ): - worker = threading.Thread( target=self.run_next ) + worker = threading.Thread( ( name="LwrJobRunner.thread-%d" % i ), target=self.run_next ) + worker.setDaemon( True ) worker.start() self.threads.append( worker ) log.debug( "%d workers ready", nworkers ) diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 lib/galaxy/jobs/runners/tasks.py --- a/lib/galaxy/jobs/runners/tasks.py +++ b/lib/galaxy/jobs/runners/tasks.py @@ -29,7 +29,8 @@ nworkers = app.config.local_task_queue_workers log.info( "Starting tasked-job runners" ) for i in range( nworkers ): - worker = threading.Thread( target=self.run_next ) + worker = threading.Thread( name=( "TaskedJobRunner-%d" % i ), target=self.run_next ) + worker.setDaemon( True ) worker.start() self.threads.append( worker ) log.debug( "%d workers ready", nworkers ) diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1335,7 +1335,9 @@ # Loop through sources until viable one is found. for source in source_list: msg = self.convert_dataset( trans, source ) - if msg == self.conversion_messages.PENDING: + # No message or PENDING means that source is viable. No + # message indicates conversion was done and is successful. + if not msg or msg == self.conversion_messages.PENDING: data_source = source break @@ -3014,6 +3016,7 @@ installation_status = Bunch( NEW='New', CLONING='Cloning', SETTING_TOOL_VERSIONS='Setting tool versions', + INSTALLING_REPOSITORY_DEPENDENCIES='Installing repository dependencies', INSTALLING_TOOL_DEPENDENCIES='Installing tool dependencies', LOADING_PROPRIETARY_DATATYPES='Loading proprietary datatypes', INSTALLED='Installed', @@ -3148,20 +3151,89 @@ def can_reinstall_or_activate( self ): return self.deleted @property + def has_repository_dependencies( self ): + if self.metadata: + return 'repository_dependencies' in self.metadata + return False + @property def includes_tools( self ): - return self.metadata and 'tools' in self.metadata + if self.metadata: + return 'tools' in self.metadata + return False @property def includes_tool_dependencies( self ): - return self.metadata and 'tool_dependencies' in self.metadata + if self.metadata: + return 'tool_dependencies' in self.metadata + return False @property def includes_workflows( self ): - return self.metadata and 'workflows' in self.metadata + if self.metadata: + return 'workflows' in self.metadata + return False @property def in_error_state( self ): return self.status == self.installation_status.ERROR @property def has_readme_files( self ): - return self.metadata and 'readme_files' in self.metadata + if self.metadata: + return 'readme_files' in self.metadata + return False + @property + def repository_dependencies( self ): + required_repositories = [] + for rrda in self.required_repositories: + repository_dependency = rrda.repository_dependency + required_repository = repository_dependency.repository + required_repositories.append( required_repository ) + return required_repositories + @property + def installed_repository_dependencies( self ): + """Return the repository's repository dependencies that are currently installed.""" + installed_required_repositories = [] + for required_repository in self.repository_dependencies: + if required_repository.status == self.installation_status.INSTALLED: + installed_required_repositories.append( required_repository ) + return installed_required_repositories + @property + def missing_repository_dependencies( self ): + """Return the repository's repository dependencies that are not currently installed, and may not ever have been installed.""" + missing_required_repositories = [] + for required_repository in self.repository_dependencies: + if required_repository.status not in [ self.installation_status.INSTALLED ]: + missing_required_repositories.append( required_repository ) + return missing_required_repositories + @property + def repository_dependencies_being_installed( self ): + required_repositories_being_installed = [] + for required_repository in self.repository_dependencies: + if tool_dependency.status == ToolDependency.installation_status.INSTALLING: + required_repositories_being_installed.append( required_repository ) + return required_repositories_being_installed + @property + def repository_dependencies_missing_or_being_installed( self ): + required_repositories_missing_or_being_installed = [] + for required_repository in self.repository_dependencies: + if required_repository.status in [ self.installation_status.ERROR, + self.installation_status.INSTALLING, + self.installation_status.NEVER_INSTALLED, + self.installation_status.UNINSTALLED ]: + required_repositories_missing_or_being_installed.append( required_repository ) + return required_repositories_missing_or_being_installed + @property + def repository_dependencies_with_installation_errors( self ): + required_repositories_with_installation_errors = [] + for required_repository in self.repository_dependencies: + if required_repository.status == self.installation_status.ERROR: + required_repositories_with_installation_errors.append( required_repository ) + return required_repositories_with_installation_errors + @property + def uninstalled_repository_dependencies( self ): + """Return the repository's repository dependencies that have been uninstalled.""" + uninstalled_required_repositories = [] + for required_repository in self.repository_dependencies: + if required_repository.status == self.installation_status.UNINSTALLED: + uninstalled_required_repositories.append( required_repository ) + return uninstalled_required_repositories @property def installed_tool_dependencies( self ): """Return the repository's tool dependencies that are currently installed.""" @@ -3211,6 +3283,15 @@ uninstalled_tool_dependencies.append( tool_dependency ) return uninstalled_tool_dependencies +class RepositoryRepositoryDependencyAssociation( object ): + def __init__( self, tool_shed_repository_id=None, repository_dependency_id=None ): + self.tool_shed_repository_id = tool_shed_repository_id + self.repository_dependency_id = repository_dependency_id + +class RepositoryDependency( object ): + def __init__( self, tool_shed_repository_id=None ): + self.tool_shed_repository_id = tool_shed_repository_id + class ToolDependency( object ): installation_status = Bunch( NEVER_INSTALLED='Never installed', INSTALLING='Installing', diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -409,6 +409,19 @@ Column( "status", TrimmedString( 255 ) ), Column( "error_message", TEXT ) ) +RepositoryRepositoryDependencyAssociation.table = Table( 'repository_repository_dependency_association', metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True ), + Column( "repository_dependency_id", Integer, ForeignKey( "repository_dependency.id" ), index=True ) ) + +RepositoryDependency.table = Table( "repository_dependency", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True, nullable=False ) ) + ToolDependency.table = Table( "tool_dependency", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -1744,7 +1757,19 @@ tool_dependencies=relation( ToolDependency, primaryjoin=( ToolShedRepository.table.c.id == ToolDependency.table.c.tool_shed_repository_id ), order_by=ToolDependency.table.c.name, - backref='tool_shed_repository' ) ) ) + backref='tool_shed_repository' ), + required_repositories=relation( RepositoryRepositoryDependencyAssociation, + primaryjoin=( ToolShedRepository.table.c.id == RepositoryRepositoryDependencyAssociation.table.c.tool_shed_repository_id ) ) ) ) + +assign_mapper( context, RepositoryRepositoryDependencyAssociation, RepositoryRepositoryDependencyAssociation.table, + properties=dict( repository=relation( ToolShedRepository, + primaryjoin=( RepositoryRepositoryDependencyAssociation.table.c.tool_shed_repository_id == ToolShedRepository.table.c.id ) ), + repository_dependency=relation( RepositoryDependency, + primaryjoin=( RepositoryRepositoryDependencyAssociation.table.c.repository_dependency_id == RepositoryDependency.table.c.id ) ) ) ) + +assign_mapper( context, RepositoryDependency, RepositoryDependency.table, + properties=dict( repository=relation( ToolShedRepository, + primaryjoin=( RepositoryDependency.table.c.tool_shed_repository_id == ToolShedRepository.table.c.id ) ) ) ) assign_mapper( context, ToolDependency, ToolDependency.table ) @@ -1925,7 +1950,7 @@ # Let this go, it could possibly work with db's we don't support log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) -def init( file_path, url, engine_options={}, create_tables=False, database_query_profiling_proxy=False, object_store=None ): +def init( file_path, url, engine_options={}, create_tables=False, database_query_profiling_proxy=False, object_store=None, trace_logger=None ): """Connect mappings to the database""" # Connect dataset to the file path Dataset.file_path = file_path @@ -1937,6 +1962,10 @@ if database_query_profiling_proxy: import galaxy.model.orm.logging_connection_proxy as logging_connection_proxy proxy = logging_connection_proxy.LoggingProxy() + # If metlog is enabled, do micrologging + elif trace_logger: + import galaxy.model.orm.logging_connection_proxy as logging_connection_proxy + proxy = logging_connection_proxy.TraceLoggerProxy( trace_logger ) else: proxy = None # Create the database engine diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 lib/galaxy/model/migrate/versions/0109_add_repository_dependency_tables.py --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0109_add_repository_dependency_tables.py @@ -0,0 +1,58 @@ +""" +Migration script to add the repository_dependency and repository_repository_dependency_association tables. +""" +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * +import sys, logging +from galaxy.model.custom_types import * +from sqlalchemy.exc import * +import datetime +now = datetime.datetime.utcnow + +log = logging.getLogger( __name__ ) +log.setLevel( logging.DEBUG ) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) + +RepositoryDependency_table = Table( "repository_dependency", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True, nullable=False ) ) + +RepositoryRepositoryDependencyAssociation_table = Table( "repository_repository_dependency_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True ), + Column( "repository_dependency_id", Integer, ForeignKey( "repository_dependency.id" ), index=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + try: + RepositoryDependency_table.create() + except Exception, e: + log.debug( "Creating repository_dependency table failed: %s" % str( e ) ) + try: + RepositoryRepositoryDependencyAssociation_table.create() + except Exception, e: + log.debug( "Creating repository_repository_dependency_association table failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + try: + RepositoryRepositoryDependencyAssociation_table.drop() + except Exception, e: + log.debug( "Dropping repository_repository_dependency_association table failed: %s" % str( e ) ) + try: + RepositoryDependency_table.drop() + except Exception, e: + log.debug( "Dropping repository_dependency table failed: %s" % str( e ) ) diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 lib/galaxy/model/orm/logging_connection_proxy.py --- a/lib/galaxy/model/orm/logging_connection_proxy.py +++ b/lib/galaxy/model/orm/logging_connection_proxy.py @@ -18,13 +18,31 @@ rval = [] for frame, fname, line, funcname, _, _ in inspect.stack()[2:]: rval.append( "%s:%s@%d" % ( stripwd( fname ), funcname, line ) ) - return " > ".join( rval ) + return rval class LoggingProxy(ConnectionProxy): + """ + Logs SQL statements using standard logging module + """ def cursor_execute(self, execute, cursor, statement, parameters, context, executemany): start = time.clock() rval = execute(cursor, statement, parameters, context) duration = time.clock() - start log.debug( "statement: %r parameters: %r executemany: %r duration: %r stack: %r", - statement, parameters, executemany, duration, pretty_stack() ) + statement, parameters, executemany, duration, " > ".join( pretty_stack() ) ) return rval + +class TraceLoggerProxy(ConnectionProxy): + """ + Logs SQL statements using a metlog client + """ + def __init__( self, trace_logger ): + self.trace_logger = trace_logger + def cursor_execute(self, execute, cursor, statement, parameters, context, executemany): + start = time.clock() + rval = execute(cursor, statement, parameters, context) + duration = time.clock() - start + self.trace_logger.log( "sqlalchemy_query", + message="Query executed", statement=statement, parameters=parameters, + executemany=executemany, duration=duration ) + return rval \ No newline at end of file diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 lib/galaxy/tool_shed/__init__.py --- a/lib/galaxy/tool_shed/__init__.py +++ b/lib/galaxy/tool_shed/__init__.py @@ -3,7 +3,8 @@ """ import os import galaxy.util.shed_util -from galaxy.model.orm import * +import galaxy.util.shed_util_common +from galaxy.model.orm import and_ from galaxy import eggs import pkg_resources @@ -27,7 +28,7 @@ ElementInclude.include( root ) tool_path = root.get( 'tool_path', None ) if tool_path: - tool_shed = galaxy.util.shed_util.clean_tool_shed_url( tool_shed_repository.tool_shed ) + tool_shed = galaxy.util.shed_util_common.clean_tool_shed_url( tool_shed_repository.tool_shed ) relative_path = os.path.join( tool_path, tool_shed, 'repos', diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 lib/galaxy/tool_shed/common_util.py --- a/lib/galaxy/tool_shed/common_util.py +++ b/lib/galaxy/tool_shed/common_util.py @@ -1,7 +1,7 @@ import os, urllib2 from galaxy import util from galaxy.util.odict import odict -from galaxy.tool_shed.encoding_util import * +from galaxy.tool_shed import encoding_util REPOSITORY_OWNER = 'devteam' @@ -36,7 +36,7 @@ print "The URL\n%s\nraised the exception:\n%s\n" % ( url, str( e ) ) if tool_shed_accessible: if text: - tool_dependencies_dict = tool_shed_decode( text ) + tool_dependencies_dict = encoding_util.tool_shed_decode( text ) for dependency_key, requirements_dict in tool_dependencies_dict.items(): tool_dependency_name = requirements_dict[ 'name' ] tool_dependency_version = requirements_dict[ 'version' ] diff -r 087a08c43cb5ac21b5c3f73240dbcde1355fa089 -r caaab03824478384c256fc6b5678bb39dbf8f9f2 lib/galaxy/tool_shed/encoding_util.py --- a/lib/galaxy/tool_shed/encoding_util.py +++ b/lib/galaxy/tool_shed/encoding_util.py @@ -1,5 +1,5 @@ -import binascii -from galaxy.util.hash_util import * +import binascii, logging +from galaxy.util.hash_util import hmac_new from galaxy.util.json import json_fix from galaxy import eggs @@ -8,7 +8,10 @@ pkg_resources.require( "simplejson" ) import simplejson +log = logging.getLogger( __name__ ) + encoding_sep = '__esep__' +encoding_sep2 = '__esepii__' def tool_shed_decode( value ): # Extract and verify hash @@ -21,12 +24,12 @@ try: values = simplejson.loads( value ) except Exception, e: - log.debug( "Decoding json value from tool shed threw exception: %s" % str( e ) ) + log.debug( "Decoding json value from tool shed for value '%s' threw exception: %s" % ( str( value ), str( e ) ) ) if values is not None: try: return json_fix( values ) except Exception, e: - log.debug( "Fixing decoded json value from tool shed threw exception: %s" % str( e ) ) + log.debug( "Fixing decoded json values '%s' from tool shed threw exception: %s" % ( str( values ), str( e ) ) ) fixed_values = values if values is None: values = value This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.