2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/47171159e9c0/ Changeset: 47171159e9c0 User: jmchilton Date: 2014-10-07 14:23:41+00:00 Summary: Python 3 fix from planemo. Affected #: 1 file diff -r 9cad38925c8da6221a1ce9817d382421c6627ecd -r 47171159e9c05fba08a6ed5e6f2883c3ef8e1ead lib/galaxy/util/submodules.py --- a/lib/galaxy/util/submodules.py +++ b/lib/galaxy/util/submodules.py @@ -14,7 +14,7 @@ __import__(full_submodule) submodule = getattr(module, submodule_name) submodules.append(submodule) - except BaseException, exception: + except BaseException as exception: exception_str = str(exception) message = "%s dynamic module could not be loaded: %s" % (full_submodule, exception_str) log.debug(message) https://bitbucket.org/galaxy/galaxy-central/commits/6b7782f17e84/ Changeset: 6b7782f17e84 User: jmchilton Date: 2014-10-07 14:23:41+00:00 Summary: Bring in newer experimental components from downstream work on planemo. This includes some utilities for working with Dockerfiles requested by Kyle Ellrott, a linting framework for Galaxy tools and a bunch of work on brew integration - most specifically a brew dependency resolver. It is obvious why the brew dependnecy resolver needs to be included in galaxy core - but I am also including the linting stuff here in case we want to reuse it in the tool shed or in the tool form. Sam's new tool form, plus the automatic tool form reloading by me and Kyle, and the tool package downloader by Dave B - is an exciting confluence of features that should really speed up tool development - adding in an GUI tool linting report would pair nicely with these features. (GUI + API work not included here - just the outline of the tool linter). The homebrew work is tracking progress on building isolated, versioned homebrew environments here - https://github.com/jmchilton/brew-tests. Allows deployers to add homebrew elements to config/tool_dependency_resolvers_conf.xml. Optional attributes include 'cellar' - this should be the absolute path to the homebrew Cellar to target (defaults to $HOME/.linuxbrew/Cellar under linux and to /usr/local/Cellar under that other operating system Galaxy supports). 'versionless' is another attribute supported by this tag - if set to true - it will ignore the specified package version and just resolve the latest installed homebrew version of that recipe. (If used this should always come after a resolver that respects versions.) This should work in some superficial way for any brew installed recipe - but it is much more robust and useful for packages installed with the `brew vinstall` external command (found in https://github.com/jmchilton/brew-tests). For `install`ed packages - each dependency must be laided out as a requirement in Galaxy - so samtools 1.1 would require to second requirement tag for hstlib for instance. For `vinstall`ed packages the dependencies are recorded at install time and can be reproducibily recovered at runtime without requiring modifying the state of the Cellar or even needing brew installed at runtime on the worker (only the Cellar directory needs to be avaialable). Affected #: 15 files diff -r 47171159e9c05fba08a6ed5e6f2883c3ef8e1ead -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 lib/galaxy/tools/deps/brew_exts.py --- /dev/null +++ b/lib/galaxy/tools/deps/brew_exts.py @@ -0,0 +1,470 @@ +#!/usr/bin/env python + +# % brew vinstall samtools 1.0 +# % brew vinstall samtools 0.1.19 +# % brew vinstall samtools 1.1 +# % brew env samtools 1.1 +# PATH=/home/john/.linuxbrew/Cellar/htslib/1.1/bin:/home/john/.linuxbrew/Cellar/samtools/1.1/bin:$PATH +# export PATH +# LD_LIBRARY_PATH=/home/john/.linuxbrew/Cellar/htslib/1.1/lib:/home/john/.linuxbrew/Cellar/samtools/1.1/lib:$LD_LIBRARY_PATH +# export LD_LIBRARY_PATH +# % . <(brew env samtools 1.1) +# % which samtools +# /home/john/.linuxbrew/Cellar/samtools/1.1/bin/samtools +# % . <(brew env samtools 0.1.19) +# % which samtools +# /home/john/.linuxbrew/Cellar/samtools/0.1.19/bin/samtools +# % brew vuninstall samtools 1.0 +# % brew vdeps samtools 1.1 +# htslib@1.1 +# % brew vdeps samtools 0.1.19 + +from __future__ import print_function + +import argparse +import contextlib +import json +import glob +import os +import re +import sys +import subprocess + +WHITESPACE_PATTERN = re.compile("[\s]+") + +DESCRIPTION = "Script built on top of linuxbrew to operate on isolated, versioned brew installed environments." + +if sys.platform == "darwin": + DEFAULT_HOMEBREW_ROOT = "/usr/local" +else: + DEFAULT_HOMEBREW_ROOT = os.path.join(os.path.expanduser("~"), ".linuxbrew") + +NO_BREW_ERROR_MESSAGE = "Could not find brew on PATH, please place on path or pass to script with --brew argument." +CANNOT_DETERMINE_TAP_ERROR_MESSAGE = "Cannot determine tap of specified recipe - please use fully qualified recipe (e.g. homebrew/science/samtools)." +VERBOSE = False +RELAXED = False + + +class BrewContext(object): + + def __init__(self, args=None): + ensure_brew_on_path(args) + raw_config = brew_execute(["config"]) + config_lines = [l.strip().split(":", 1) for l in raw_config.split("\n") if l] + config = dict([(p[0].strip(), p[1].strip()) for p in config_lines]) + # unset if "/usr/local" -> https://github.com/Homebrew/homebrew/blob/master/Library/Homebrew/cmd/config... + homebrew_prefix = config.get("HOMEBREW_PREFIX", "/usr/local") + homebrew_cellar = config.get("HOMEBREW_CELLAR", os.path.join(homebrew_prefix, "Cellar")) + self.homebrew_prefix = homebrew_prefix + self.homebrew_cellar = homebrew_cellar + + +class RecipeContext(object): + + @staticmethod + def from_args(args, brew_context=None): + return RecipeContext(args.recipe, args.version, brew_context) + + def __init__(self, recipe, version, brew_context=None): + self.recipe = recipe + self.version = version + self.brew_context = brew_context or BrewContext() + + @property + def cellar_path(self): + return recipe_cellar_path(self.brew_context.homebrew_cellar, self.recipe, self.version) + + @property + def tap_path(self): + return os.path.join(self.brew_context.homebrew_prefix, "Library", "Taps", self.__tap_path(self.recipe)) + + def __tap_path(self, recipe): + parts = recipe.split("/") + if len(parts) == 1: + info = brew_info(self.recipe) + from_url = info["from_url"] + if not from_url: + raise Exception(CANNOT_DETERMINE_TAP_ERROR_MESSAGE) + from_url_parts = from_url.split("/") + blob_index = from_url_parts.index("blob") # comes right after username and repository + if blob_index < 2: + raise Exception(CANNOT_DETERMINE_TAP_ERROR_MESSAGE) + username = from_url_parts[blob_index - 2] + repository = from_url_parts[blob_index - 1] + else: + assert len(parts) == 3 + parts = recipe.split("/") + username = parts[0] + repository = "homebrew-%s" % parts[1] + + path = os.path.join(username, repository) + return path + + +def main(): + global VERBOSE + global RELAXED + parser = argparse.ArgumentParser(description=DESCRIPTION) + parser.add_argument("--brew", help="Path to linuxbrew 'brew' executable to target") + actions = ["vinstall", "vuninstall", "vdeps", "vinfo", "env"] + action = __action(sys) + if not action: + parser.add_argument('action', metavar='action', help="Versioned action to perform.", choices=actions) + parser.add_argument('recipe', metavar='recipe', help="Recipe for action - should be absolute (e.g. homebrew/science/samtools).") + parser.add_argument('version', metavar='version', help="Version for action (e.g. 0.1.19).") + parser.add_argument('--relaxed', action='store_true', help="Relaxed processing - for instance allow use of env on non-vinstall-ed recipes.") + parser.add_argument('--verbose', action='store_true', help="Verbose output") + args = parser.parse_args() + if args.verbose: + VERBOSE = True + if args.relaxed: + RELAXED = True + if not action: + action = args.action + brew_context = BrewContext(args) + recipe_context = RecipeContext.from_args(args, brew_context) + if action == "vinstall": + versioned_install(recipe_context, args.recipe, args.version) + elif action == "vuninstall": + brew_execute(["switch", args.recipe, args.version]) + brew_execute(["uninstall", args.recipe]) + elif action == "vdeps": + print_versioned_deps(recipe_context, args.recipe, args.version) + elif action == "env": + env_statements = build_env_statements_from_recipe_context(recipe_context) + print(env_statements) + elif action == "vinfo": + with brew_head_at_version(recipe_context, args.recipe, args.version): + print(brew_info(args.recipe)) + else: + raise NotImplementedError() + + +class CommandLineException(Exception): + + def __init__(self, command, stdout, stderr): + self.command = command + self.stdout = stdout + self.stderr = stderr + self.message = ("Failed to execute command-line %s, stderr was:\n" + "-------->>begin stderr<<--------\n" + "%s\n" + "-------->>end stderr<<--------\n" + "-------->>begin stdout<<--------\n" + "%s\n" + "-------->>end stdout<<--------\n" + ) % (command, stderr, stdout) + + def __str__(self): + return self.message + + +def versioned_install(recipe_context, package=None, version=None): + if package is None: + package = recipe_context.recipe + version = recipe_context.version + + attempt_unlink(package) + with brew_head_at_version(recipe_context, package, version): + deps = brew_deps(package) + deps_metadata = [] + dep_to_version = {} + for dep in deps: + version_info = brew_versions_info(dep, recipe_context.tap_path)[0] + dep_version = version_info[0] + dep_to_version[dep] = dep_version + versioned = version_info[2] + if versioned: + dep_to_version[dep] = dep_version + versioned_install(recipe_context, dep, dep_version) + else: + # Install latest. + dep_to_version[dep] = None + unversioned_install(dep) + try: + for dep in deps: + dep_version = dep_to_version[dep] + if dep_version: + brew_execute(["switch", dep, dep_version]) + else: + brew_execute(["link", dep]) + # dep_version obtained from brew versions doesn't + # include revision. This linked_keg attribute does. + keg_verion = brew_info(dep)["linked_keg"] + dep_metadata = { + 'name': dep, + 'version': keg_verion, + 'versioned': versioned + } + deps_metadata.append(dep_metadata) + + brew_execute(["install", package]) + deps = brew_execute(["deps", package]) + deps = [d.strip() for d in deps.split("\n") if d] + metadata = { + 'deps': deps_metadata + } + cellar_root = recipe_context.brew_context.homebrew_cellar + cellar_path = recipe_cellar_path( cellar_root, package, version ) + v_metadata_path = os.path.join(cellar_path, "INSTALL_RECEIPT_VERSIONED.json") + with open(v_metadata_path, "w") as f: + json.dump(metadata, f) + + finally: + attempt_unlink_all(package, deps) + + +def commit_for_version(recipe_context, package, version): + tap_path = recipe_context.tap_path + commit = None + with brew_head_at_commit("master", tap_path): + version_to_commit = brew_versions_info(package, tap_path) + if version is None: + version = version_to_commit[0][0] + commit = version_to_commit[0][1] + else: + for mapping in version_to_commit: + if mapping[0] == version: + commit = mapping[1] + if commit is None: + raise Exception("Failed to find commit for version %s" % version) + return commit + + +def print_versioned_deps(recipe_context, recipe, version): + deps = load_versioned_deps(recipe_context.cellar_path) + for dep in deps: + val = dep['name'] + if dep['versioned']: + val += "@%s" % dep['version'] + print(val) + + +def load_versioned_deps(cellar_path, relaxed=None): + if relaxed is None: + relaxed = RELAXED + v_metadata_path = os.path.join(cellar_path, "INSTALL_RECEIPT_VERSIONED.json") + if not os.path.isfile(v_metadata_path): + if RELAXED: + return [] + else: + raise IOError("Could not locate versioned receipt file: {}".format(v_metadata_path)) + with open(v_metadata_path, "r") as f: + metadata = json.load(f) + return metadata['deps'] + + +def unversioned_install(package): + try: + deps = brew_deps(package) + for dep in deps: + brew_execute(["link", dep]) + brew_execute(["install", package]) + finally: + attempt_unlink_all(package, deps) + + +def attempt_unlink_all(package, deps): + for dep in deps: + attempt_unlink(dep) + attempt_unlink(package) + + +def attempt_unlink(package): + try: + brew_execute(["unlink", package]) + except Exception: + # TODO: warn + pass + + +def brew_execute(args): + os.environ["HOMEBREW_NO_EMOJI"] = "1" # simplify brew parsing. + cmds = ["brew"] + args + return execute(cmds) + + +def build_env_statements_from_recipe_context(recipe_context, **kwds): + cellar_root = recipe_context.brew_context.homebrew_cellar + env_statements = build_env_statements(cellar_root, recipe_context.cellar_path, **kwds) + return env_statements + + +def build_env_statements(cellar_root, cellar_path, relaxed=None): + deps = load_versioned_deps(cellar_path, relaxed=relaxed) + + path_appends = [] + ld_path_appends = [] + + def handle_keg(cellar_path): + bin_path = os.path.join(cellar_path, "bin") + if os.path.isdir(bin_path): + path_appends.append(bin_path) + lib_path = os.path.join(cellar_path, "lib") + if os.path.isdir(lib_path): + ld_path_appends.append(lib_path) + + for dep in deps: + package = dep['name'] + version = dep['version'] + dep_cellar_path = recipe_cellar_path( cellar_root, package, version ) + handle_keg( dep_cellar_path ) + + handle_keg( cellar_path ) + env_statements = [] + if path_appends: + env_statements.append("PATH=" + ":".join(path_appends) + ":$PATH") + env_statements.append("export PATH") + if ld_path_appends: + env_statements.append("LD_LIBRARY_PATH=" + ":".join(ld_path_appends) + ":$LD_LIBRARY_PATH") + env_statements.append("export LD_LIBRARY_PATH") + return "\n".join(env_statements) + + +@contextlib.contextmanager +def brew_head_at_version(recipe_context, package, version): + commit = commit_for_version(recipe_context, package, version) + tap_path = recipe_context.tap_path + with brew_head_at_commit(commit, tap_path): + yield + + +@contextlib.contextmanager +def brew_head_at_commit(commit, tap_path): + try: + os.chdir(tap_path) + current_commit = git_execute(["rev-parse", "HEAD"]).strip() + try: + git_execute(["checkout", commit]) + yield + finally: + git_execute(["checkout", current_commit]) + finally: + # TODO: restore chdir - or better yet just don't chdir + # shouldn't be needed. + pass + + +def git_execute(args): + cmds = ["git"] + args + return execute(cmds) + + +def execute(cmds): + p = subprocess.Popen(cmds, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + #log = p.stdout.read() + global VERBOSE + stdout, stderr = p.communicate() + if p.returncode != 0: + raise CommandLineException(" ".join(cmds), stdout, stderr) + if VERBOSE: + print(stdout) + return stdout + + +def brew_deps(package): + stdout = brew_execute(["deps", package]) + return [p.strip() for p in stdout.split("\n") if p] + + +def brew_info(recipe): + info_json = brew_execute(["info", "--json=v1", recipe]) + info = json.loads(info_json)[0] + info.update(extended_brew_info(recipe)) + return info + + +def extended_brew_info(recipe): + # Extract more info from non-json variant. JSON variant should + # include this in a backward compatible way (TODO: Open PR). + raw_info = brew_execute(["info", recipe]) + extra_info = dict( + from_url=None, + build_dependencies=[], + required_dependencies=[], + recommended_dependencies=[], + optional_dependencies=[], + ) + + for line in raw_info.split("\n"): + if line.startswith("From: "): + extra_info["from_url"] = line[len("From: "):].strip() + for dep_type in ["Build", "Required", "Recommended", "Optional"]: + if line.startswith("%s: " % dep_type): + key = "%s_dependencies" % dep_type.lower() + raw_val = line[len("%s: " % dep_type):] + extra_info[key].extend(raw_val.split(", ")) + return extra_info + + +def brew_versions_info(package, tap_path): + + def versioned(recipe_path): + if not os.path.isabs(recipe_path): + recipe_path = os.path.join(os.getcwd(), recipe_path) + # Dependencies in the same repository should be versioned, + # core dependencies (presumably in base homebrew) are not + # versioned. + return tap_path in recipe_path + + # TODO: Also use tags. + stdout = brew_execute(["versions", package]) + version_parts = [l for l in stdout.split("\n") if l and "git checkout" in l] + version_parts = map(lambda l: WHITESPACE_PATTERN.split(l), version_parts) + info = [(p[0], p[3], versioned(p[4])) for p in version_parts] + return info + + +def __action(sys): + script_name = os.path.basename(sys.argv[0]) + if script_name.startswith("brew-"): + return script_name[len("brew-"):] + else: + return None + + +def recipe_cellar_path(cellar_path, recipe, version): + recipe_base = recipe.split("/")[-1] + recipe_base_path = os.path.join(cellar_path, recipe_base, version) + revision_paths = glob.glob(recipe_base_path + "_*") + if revision_paths: + revisions = map(lambda x: int(x.rsplit("_", 1)[-1]), revision_paths) + max_revision = max(revisions) + recipe_path = "%s_%d" % (recipe_base_path, max_revision) + else: + recipe_path = recipe_base_path + return recipe_path + + +def ensure_brew_on_path(args): + brew_on_path = which("brew") + if brew_on_path: + brew_on_path = os.path.abspath(brew_on_path) + + def ensure_on_path(brew): + if brew != brew_on_path: + os.environ["PATH"] = "%s:%s" % (os.path.dirname(brew), os.environ["PATH"]) + + default_brew_path = os.path.join(DEFAULT_HOMEBREW_ROOT, "bin", "brew") + if args and args.brew: + user_brew_path = os.path.abspath(args.brew) + ensure_on_path(user_brew_path) + elif brew_on_path: + return brew_on_path + elif os.path.exists(default_brew_path): + ensure_on_path(default_brew_path) + else: + raise Exception(NO_BREW_ERROR_MESSAGE) + + +def which(file): + # http://stackoverflow.com/questions/5226958/which-equivalent-function-in-pyth... + for path in os.environ["PATH"].split(":"): + if os.path.exists(path + "/" + file): + return path + "/" + file + + return None + + +if __name__ == "__main__": + main() diff -r 47171159e9c05fba08a6ed5e6f2883c3ef8e1ead -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 lib/galaxy/tools/deps/brew_util.py --- /dev/null +++ b/lib/galaxy/tools/deps/brew_util.py @@ -0,0 +1,40 @@ +""" brew_exts defines generic extensions to Homebrew this file +builds on those abstraction and provides Galaxy specific functionality +not useful to the brew external commands. +""" +from ..deps import brew_exts + +DEFAULT_TAP = "homebrew/science" + + +class HomebrewRecipe(object): + + def __init__(self, recipe, version, tap): + self.recipe = recipe + self.version = version + self.tap = tap + + +def requirements_to_recipes(requirements): + return filter(None, map(requirement_to_recipe, requirements)) + + +def requirement_to_recipe(requirement): + if requirement.type != "package": + return None + # TOOD: Allow requirements to annotate optionalbrew specific + # adaptions. + recipe_name = requirement.name + recipe_version = requirement.version + return HomebrewRecipe(recipe_name, recipe_version, tap=DEFAULT_TAP) + + +def requirements_to_recipe_contexts(requirements, brew_context): + def to_recipe_context(homebrew_recipe): + return brew_exts.RecipeContext( + homebrew_recipe.recipe, + homebrew_recipe.version, + brew_context + ) + return map(to_recipe_context, requirements_to_recipes(requirements)) + diff -r 47171159e9c05fba08a6ed5e6f2883c3ef8e1ead -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 lib/galaxy/tools/deps/commands.py --- /dev/null +++ b/lib/galaxy/tools/deps/commands.py @@ -0,0 +1,54 @@ +import os +import subprocess + + +def shell(cmds, env=None): + popen_kwds = dict( + shell=True, + ) + if env: + new_env = os.environ.copy() + new_env.update(env) + popen_kwds["env"] = new_env + p = subprocess.Popen(cmds, **popen_kwds) + return p.wait() + + +def execute(cmds): + return __wait(cmds, shell=False) + + +def which(file): + # http://stackoverflow.com/questions/5226958/which-equivalent-function-in-pyth... + for path in os.environ["PATH"].split(":"): + if os.path.exists(path + "/" + file): + return path + "/" + file + + return None + + +def __wait(cmds, **popen_kwds): + p = subprocess.Popen(cmds, **popen_kwds) + stdout, stderr = p.communicate() + if p.returncode != 0: + raise CommandLineException(" ".join(cmds), stdout, stderr) + return stdout + + +class CommandLineException(Exception): + + def __init__(self, command, stdout, stderr): + self.command = command + self.stdout = stdout + self.stderr = stderr + self.message = ("Failed to execute command-line %s, stderr was:\n" + "-------->>begin stderr<<--------\n" + "%s\n" + "-------->>end stderr<<--------\n" + "-------->>begin stdout<<--------\n" + "%s\n" + "-------->>end stdout<<--------\n" + ) % (command, stderr, stdout) + + def __str__(self): + return self.message diff -r 47171159e9c05fba08a6ed5e6f2883c3ef8e1ead -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 lib/galaxy/tools/deps/docker_util.py --- a/lib/galaxy/tools/deps/docker_util.py +++ b/lib/galaxy/tools/deps/docker_util.py @@ -1,3 +1,4 @@ +import os DEFAULT_DOCKER_COMMAND = "docker" DEFAULT_SUDO = True @@ -51,6 +52,34 @@ return ":".join([self.from_path, self.to_path, self.how]) +def build_command( + image, + docker_build_path, + docker_cmd=DEFAULT_DOCKER_COMMAND, + sudo=DEFAULT_SUDO, + sudo_cmd=DEFAULT_SUDO_COMMAND, + host=DEFAULT_HOST, +): + if os.path.isfile(docker_build_path): + docker_build_path = os.path.dirname(os.path.abspath(docker_build_path)) + build_command_parts = __docker_prefix(docker_cmd, sudo, sudo_cmd, host) + build_command_parts.extend(["build", "-t", image, docker_build_path]) + return build_command_parts + + +def build_save_image_command( + image, + destination, + docker_cmd=DEFAULT_DOCKER_COMMAND, + sudo=DEFAULT_SUDO, + sudo_cmd=DEFAULT_SUDO_COMMAND, + host=DEFAULT_HOST, +): + build_command_parts = __docker_prefix(docker_cmd, sudo, sudo_cmd, host) + build_command_parts.extend(["save", "-o", destination, image]) + return build_command_parts + + def build_docker_cache_command( image, docker_cmd=DEFAULT_DOCKER_COMMAND, @@ -72,6 +101,7 @@ def build_docker_run_command( container_command, image, + interactive=False, tag=None, volumes=[], volumes_from=DEFAULT_VOLUMES_FROM, @@ -88,6 +118,8 @@ ): command_parts = __docker_prefix(docker_cmd, sudo, sudo_cmd, host) command_parts.append("run") + if interactive: + command_parts.append("-i") for env_directive in env_directives: command_parts.extend(["-e", env_directive]) for volume in volumes: diff -r 47171159e9c05fba08a6ed5e6f2883c3ef8e1ead -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 lib/galaxy/tools/deps/dockerfiles.py --- /dev/null +++ b/lib/galaxy/tools/deps/dockerfiles.py @@ -0,0 +1,54 @@ +import os + +from ..deps import commands +from ..deps import docker_util +from ..deps.requirements import parse_requirements_from_xml +from ...tools import loader_directory + +import logging +log = logging.getLogger(__name__) + + +def docker_host_args(**kwds): + return dict( + docker_cmd=kwds["docker_cmd"], + sudo=kwds["docker_sudo"], + sudo_cmd=kwds["docker_sudo_cmd"], + host=kwds["docker_host"] + ) + + +def dockerfile_build(path, dockerfile=None, error=log.error, **kwds): + expected_container_names = set() + for (tool_path, tool_xml) in loader_directory.load_tool_elements_from_path(path): + requirements, containers = parse_requirements_from_xml(tool_xml) + for container in containers: + if container.type == "docker": + expected_container_names.add(container.identifier) + break + + if len(expected_container_names) == 0: + error("Could not find any docker identifiers to generate.") + + if len(expected_container_names) > 1: + error("Multiple different docker identifiers found for selected tools [%s]", expected_container_names) + + image_identifier = expected_container_names.pop() + if dockerfile is None: + dockerfile = "Dockerfile" + + docker_command_parts = docker_util.build_command( + image_identifier, + dockerfile, + **docker_host_args(**kwds) + ) + commands.execute(docker_command_parts) + docker_image_cache = kwds['docker_image_cache'] + if docker_image_cache: + destination = os.path.join(docker_image_cache, image_identifier + ".tar") + save_image_command_parts = docker_util.build_save_image_command( + image_identifier, + destination, + **docker_host_args(**kwds) + ) + commands.execute(save_image_command_parts) diff -r 47171159e9c05fba08a6ed5e6f2883c3ef8e1ead -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 lib/galaxy/tools/deps/resolvers/homebrew.py --- /dev/null +++ b/lib/galaxy/tools/deps/resolvers/homebrew.py @@ -0,0 +1,96 @@ +""" +This file implements a brew resolver for Galaxy requirements. In order for Galaxy +to pick up on recursively defined and versioned brew dependencies recipes should +be installed using the experimental `brew-vinstall` external command. + +More information here: + +https://github.com/jmchilton/brew-tests +https://github.com/Homebrew/homebrew-science/issues/1191 + +This is still an experimental module and there will almost certainly be backward +incompatible changes coming. +""" + +import os + +from ..brew_exts import DEFAULT_HOMEBREW_ROOT, recipe_cellar_path, build_env_statements +from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY, Dependency + +# TODO: Implement prefer version linked... +PREFER_VERSION_LINKED = 'linked' +PREFER_VERSION_LATEST = 'latest' +UNKNOWN_PREFER_VERSION_MESSAGE_TEMPLATE = "HomebrewDependencyResolver prefer_version must be latest %s" +UNKNOWN_PREFER_VERSION_MESSAGE = UNKNOWN_PREFER_VERSION_MESSAGE_TEMPLATE % (PREFER_VERSION_LATEST) +DEFAULT_PREFER_VERSION = PREFER_VERSION_LATEST + + +class HomebrewDependencyResolver(DependencyResolver): + resolver_type = "homebrew" + + def __init__(self, dependency_manager, **kwds): + self.versionless = _string_as_bool(kwds.get('versionless', 'false')) + self.prefer_version = kwds.get('prefer_version', None) + + if self.prefer_version is None: + self.prefer_version = DEFAULT_PREFER_VERSION + + if self.versionless and self.prefer_version not in [PREFER_VERSION_LATEST]: + raise Exception(UNKNOWN_PREFER_VERSION_MESSAGE) + + cellar_root = kwds.get('cellar', None) + if cellar_root is None: + cellar_root = os.path.join(DEFAULT_HOMEBREW_ROOT, "Cellar") + + self.cellar_root = cellar_root + + def resolve(self, name, version, type, **kwds): + if type != "package": + return INDETERMINATE_DEPENDENCY + + if version is None or self.versionless: + return self._find_dep_default(name, version) + else: + return self._find_dep_versioned(name, version) + + def _find_dep_versioned(self, name, version): + recipe_path = recipe_cellar_path(self.cellar_root, name, version) + if not os.path.exists(recipe_path) or not os.path.isdir(recipe_path): + return INDETERMINATE_DEPENDENCY + + commands = build_env_statements(self.cellar_root, recipe_path, relaxed=True) + return HomebrewDependency(commands) + + def _find_dep_default(self, name, version): + installed_versions = self._installed_versions(name) + if not installed_versions: + return INDETERMINATE_DEPENDENCY + + # Just grab newest installed version - may make sense some day to find + # the linked version instead. + default_version = sorted(installed_versions, reverse=True)[0] + return self._find_dep_versioned(name, default_version) + + def _installed_versions(self, recipe): + recipe_base_path = os.path.join(self.cellar_root, recipe) + if not os.path.exists(recipe_base_path): + return [] + + names = os.listdir(recipe_base_path) + return filter(lambda n: os.path.isdir(os.path.join(recipe_base_path, n)), names) + + +class HomebrewDependency(Dependency): + + def __init__(self, commands): + self.commands = commands + + def shell_commands(self, requirement): + return self.commands.replace("\n", ";") + "\n" + + +def _string_as_bool( value ): + return str( value ).lower() == "true" + + +__all__ = [HomebrewDependencyResolver] diff -r 47171159e9c05fba08a6ed5e6f2883c3ef8e1ead -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 lib/galaxy/tools/lint.py --- /dev/null +++ b/lib/galaxy/tools/lint.py @@ -0,0 +1,89 @@ +from __future__ import print_function +import inspect +from galaxy.util import submodules + +LEVEL_ALL = "all" +LEVEL_WARN = "warn" +LEVEL_ERROR = "error" + + +def lint_xml(tool_xml, level=LEVEL_ALL, fail_level=LEVEL_WARN): + import galaxy.tools.linters + lint_context = LintContext(level=level) + linter_modules = submodules.submodules(galaxy.tools.linters) + for module in linter_modules: + for (name, value) in inspect.getmembers(module): + if callable(value) and name.startswith("lint_"): + lint_context.lint(module, name, value, tool_xml) + found_warns = lint_context.found_warns + found_errors = lint_context.found_errors + if level == LEVEL_WARN and (found_warns or found_errors): + return False + else: + return found_errors + + +class LintContext(object): + + def __init__(self, level): + self.level = level + self.found_errors = False + self.found_warns = False + + def lint(self, module, name, lint_func, tool_xml): + self.printed_linter_info = False + self.valid_messages = [] + self.info_messages = [] + self.warn_messages = [] + self.error_messages = [] + lint_func(tool_xml, self) + # TODO: colorful emoji if in click CLI. + if self.error_messages: + status = "FAIL" + elif self.warn_messages: + + status = "WARNING" + else: + status = "CHECK" + + def print_linter_info(): + if self.printed_linter_info: + return + self.printed_linter_info = True + print("Applying linter %s... %s" % (name, status)) + + for message in self.error_messages: + self.found_errors = True + print_linter_info() + print(".. ERROR: %s" % message) + + if self.level != LEVEL_ERROR: + for message in self.warn_messages: + self.found_warns = True + print_linter_info() + print(".. WARNING: %s" % message) + + if self.level == LEVEL_ALL: + for message in self.info_messages: + print_linter_info() + print(".. INFO: %s" % message) + for message in self.valid_messages: + print_linter_info() + print(".. CHECK: %s" % message) + + def __handle_message(self, message_list, message, *args): + if args: + message = message % args + message_list.append(message) + + def valid(self, message, *args): + self.__handle_message(self.valid_messages, message, *args) + + def info(self, message, *args): + self.__handle_message(self.info_messages, message, *args) + + def error(self, message, *args): + self.__handle_message(self.error_messages, message, *args) + + def warn(self, message, *args): + self.__handle_message(self.warn_messages, message, *args) diff -r 47171159e9c05fba08a6ed5e6f2883c3ef8e1ead -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 lib/galaxy/tools/linters/__init__.py --- /dev/null +++ b/lib/galaxy/tools/linters/__init__.py @@ -0,0 +1,2 @@ +""" Framework for linting tools. +""" diff -r 47171159e9c05fba08a6ed5e6f2883c3ef8e1ead -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 lib/galaxy/tools/linters/citations.py --- /dev/null +++ b/lib/galaxy/tools/linters/citations.py @@ -0,0 +1,27 @@ + + +def lint_citations(tool_xml, lint_ctx): + root = tool_xml.getroot() + citations = root.findall("citations") + if len(citations) > 1: + lint_ctx.error("More than one citation section found, behavior undefined.") + return + + if len(citations) == 0: + lint_ctx.warn("No citations found, consider adding citations to your tool.") + return + + valid_citations = 0 + for citation in citations[0]: + if citation.tag != "citation": + lint_ctx.warn("Unknown tag discovered in citations block [%s], will be ignored." % citation.tag) + if "type" in citation.attrib: + citation_type = citation.attrib.get("type") + if citation_type not in ["doi", "bibtex"]: + lint_ctx.warn("Unknown citation type discovered [%s], will be ignored.", citation_type) + else: + valid_citations += 1 + + if valid_citations > 0: + lint_ctx.valid("Found %d likely valid citations.", valid_citations) + diff -r 47171159e9c05fba08a6ed5e6f2883c3ef8e1ead -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 lib/galaxy/tools/linters/help.py --- /dev/null +++ b/lib/galaxy/tools/linters/help.py @@ -0,0 +1,15 @@ + + +def lint_help(tool_xml, lint_ctx): + root = tool_xml.getroot() + helps = root.findall("help") + if len(helps) > 1: + lint_ctx.error("More than one help section found, behavior undefined.") + return + + if len(helps) == 0: + lint_ctx.warn("No help section found, consider adding a help section to your tool.") + return + + # TODO: validate help section RST. + lint_ctx.valid("Tool contains help section.") diff -r 47171159e9c05fba08a6ed5e6f2883c3ef8e1ead -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 lib/galaxy/tools/linters/inputs.py --- /dev/null +++ b/lib/galaxy/tools/linters/inputs.py @@ -0,0 +1,38 @@ + + +def lint_inputs(tool_xml, lint_ctx): + inputs = tool_xml.findall("./inputs//param") + num_inputs = 0 + for param in inputs: + num_inputs += 1 + param_attrib = param.attrib + has_errors = False + if "type" not in param_attrib: + lint_ctx.error("Found param input with type specified.") + has_errors = True + if "name" not in param_attrib: + lint_ctx.error("Found param input with not name specified.") + has_errors = True + + if has_errors: + continue + + param_type = param_attrib["type"] + param_name = param_attrib["name"] + if param_type == "data_input": + if "format" not in param_attrib: + lint_ctx.warn("Found param input %s contains no format specified - 'data' format will be assumed.", param_name) + # TODO: Validate type, much more... + if num_inputs: + lint_ctx.info("Found %d input parameters.", num_inputs) + else: + lint_ctx.warn("Found not input parameters.") + + +def lint_repeats(tool_xml, lint_ctx): + repeats = tool_xml.findall("./inputs//repeat") + for repeat in repeats: + if "name" not in repeat.attrib: + lint_ctx.error("Repeat does not specify name attribute.") + if "title" not in repeat.attrib: + lint_ctx.error("Repeat does not specify title attribute.") diff -r 47171159e9c05fba08a6ed5e6f2883c3ef8e1ead -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 lib/galaxy/tools/linters/outputs.py --- /dev/null +++ b/lib/galaxy/tools/linters/outputs.py @@ -0,0 +1,25 @@ + + +def lint_output(tool_xml, lint_ctx): + outputs = tool_xml.findall("./outputs/data") + if not outputs: + lint_ctx.warn("Tool contains no outputs, most tools should produce outputs..") + return + + num_outputs = 0 + for output in outputs: + num_outputs += 1 + output_attrib = output.attrib + format_set = False + if "format" in output_attrib: + format_set = True + format = output_attrib["format"] + if format == "input": + lint_ctx.warn("Using format='input' on output data, format_source attribute is less ambigious and should be used instead.") + elif "format_source" in output_attrib: + format_set = True + + if not format_set: + lint_ctx.warn("Tool data output doesn't define an output format.") + + lint_ctx.info("%d output datasets found.", num_outputs) diff -r 47171159e9c05fba08a6ed5e6f2883c3ef8e1ead -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 lib/galaxy/tools/linters/tests.py --- /dev/null +++ b/lib/galaxy/tools/linters/tests.py @@ -0,0 +1,20 @@ + + +# Misspelled so as not be picked up by nosetests. +def lint_tsts(tool_xml, lint_ctx): + tests = tool_xml.findall("./tests/test") + if not tests: + lint_ctx.warn("No tests found, most tools should define test cases.") + + num_valid_tests = 0 + for test in tests: + outputs = test.findall("output") + if not outputs: + lint_ctx.warn("No outputs defined for tests, this test is likely invalid.") + else: + num_valid_tests += 1 + + if num_valid_tests: + lint_ctx.valid("%d test(s) found.", num_valid_tests) + else: + lint_ctx.warn("No valid test(s) found.") diff -r 47171159e9c05fba08a6ed5e6f2883c3ef8e1ead -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 lib/galaxy/tools/linters/top_level.py --- /dev/null +++ b/lib/galaxy/tools/linters/top_level.py @@ -0,0 +1,17 @@ + +def lint_top_level(tree, lint_ctx): + root = tree.getroot() + if "version" not in root.attrib: + lint_ctx.error("Tool does not define a version attribute.") + else: + lint_ctx.valid("Tool defines a version.") + + if "name" not in root.attrib: + lint_ctx.error("Tool does not define a name attribute.") + else: + lint_ctx.valid("Tool defines a name.") + + if "id" not in root.attrib: + lint_ctx.error("Tool does not define an id attribute.") + else: + lint_ctx.valid("Tool defines an id name.") diff -r 47171159e9c05fba08a6ed5e6f2883c3ef8e1ead -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 lib/galaxy/tools/loader_directory.py --- /dev/null +++ b/lib/galaxy/tools/loader_directory.py @@ -0,0 +1,31 @@ +import glob +import os +from ..tools import loader + +PATH_DOES_NOT_EXIST_ERROR = "Could not load tools from path [%s] - this path does not exist." + + +def load_tool_elements_from_path(path): + tool_elements = [] + for file in __find_tool_files(path): + if __looks_like_a_tool(file): + tool_elements.append((file, loader.load_tool(file))) + return tool_elements + + +def __looks_like_a_tool(path): + with open(path) as f: + for i in range(10): + line = f.next() + if "<tool" in line: + return True + return False + + +def __find_tool_files(path): + if not os.path.exists(path): + raise Exception(PATH_DOES_NOT_EXIST_ERROR) + if not os.path.isdir(path): + return [os.path.abspath(path)] + else: + return map(os.path.abspath, glob.glob(path + "/**.xml")) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.