commit/galaxy-central: greg: Add support for a new <set_environment> tag set in a tool shed repository's tool_dependencies.xml file. This new tag set is defined outside of <package> tag sets, and allows for setting environment variables to point to tool dependency files (scripts, binaries, etc) that are included in the installed repository.
1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/6aadac8026cb/ changeset: 6aadac8026cb user: greg date: 2012-09-13 17:21:51 summary: Add support for a new <set_environment> tag set in a tool shed repository's tool_dependencies.xml file. This new tag set is defined outside of <package> tag sets, and allows for setting environment variables to point to tool dependency files (scripts, binaries, etc) that are included in the installed repository. affected #: 12 files diff -r 1eba2def52653577163901c765fd4686c9d42dfc -r 6aadac8026cb55844a0e8e49360ad30a9f6d0a25 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2996,12 +2996,20 @@ self.status = status self.error_message = error_message def installation_directory( self, app ): - return os.path.join( app.config.tool_dependency_dir, - self.name, - self.version, - self.tool_shed_repository.owner, - self.tool_shed_repository.name, - self.tool_shed_repository.installed_changeset_revision ) + if self.type == 'package': + return os.path.join( app.config.tool_dependency_dir, + self.name, + self.version, + self.tool_shed_repository.owner, + self.tool_shed_repository.name, + self.tool_shed_repository.installed_changeset_revision ) + if self.type == 'set_environment': + return os.path.join( app.config.tool_dependency_dir, + 'environment_settings', + self.name, + self.tool_shed_repository.owner, + self.tool_shed_repository.name, + self.tool_shed_repository.installed_changeset_revision ) class ToolVersion( object ): def __init__( self, id=None, create_time=None, tool_id=None, tool_shed_repository=None ): diff -r 1eba2def52653577163901c765fd4686c9d42dfc -r 6aadac8026cb55844a0e8e49360ad30a9f6d0a25 lib/galaxy/tool_shed/tool_dependencies/common_util.py --- a/lib/galaxy/tool_shed/tool_dependencies/common_util.py +++ b/lib/galaxy/tool_shed/tool_dependencies/common_util.py @@ -1,6 +1,47 @@ import os, shutil, tarfile, urllib2 from galaxy.datatypes.checkers import * +def create_env_var_dict( elem, tool_dependency_install_dir=None, tool_shed_repository_install_dir=None ): + env_var_name = elem.get( 'name', 'PATH' ) + env_var_action = elem.get( 'action', 'prepend_to' ) + env_var_text = None + if elem.text and elem.text.find( 'REPOSITORY_INSTALL_DIR' ) >= 0: + if tool_shed_repository_install_dir: + env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR', tool_shed_repository_install_dir ) + return dict( name=env_var_name, action=env_var_action, value=env_var_text ) + else: + env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR', tool_dependency_install_dir ) + return dict( name=env_var_name, action=env_var_action, value=env_var_text ) + if elem.text and elem.text.find( 'INSTALL_DIR' ) >= 0: + if tool_dependency_install_dir: + env_var_text = elem.text.replace( '$INSTALL_DIR', tool_dependency_install_dir ) + return dict( name=env_var_name, action=env_var_action, value=env_var_text ) + else: + env_var_text = elem.text.replace( '$INSTALL_DIR', tool_shed_repository_install_dir ) + return dict( name=env_var_name, action=env_var_action, value=env_var_text ) + return None +def create_or_update_env_shell_file( install_dir, env_var_dict ): + env_var_name = env_var_dict[ 'name' ] + env_var_action = env_var_dict[ 'action' ] + env_var_value = env_var_dict[ 'value' ] + if env_var_action == 'prepend_to': + changed_value = '%s:$%s' % ( env_var_value, env_var_name ) + elif env_var_action == 'set_to': + changed_value = '%s' % env_var_value + elif env_var_action == 'append_to': + changed_value = '$%s:%s' % ( env_var_name, env_var_value ) + env_shell_file_path = '%s/env.sh' % install_dir + if os.path.exists( env_shell_file_path ): + write_action = '>>' + else: + write_action = '>' + cmd = "echo '%s=%s; export %s' %s %s;chmod +x %s" % ( env_var_name, + changed_value, + env_var_name, + write_action, + env_shell_file_path, + env_shell_file_path ) + return cmd def extract_tar( file_name, file_path ): if isgzip( file_name ) or isbz2( file_name ): # Open for reading with transparent compression. diff -r 1eba2def52653577163901c765fd4686c9d42dfc -r 6aadac8026cb55844a0e8e49360ad30a9f6d0a25 lib/galaxy/tool_shed/tool_dependencies/fabric_util.py --- a/lib/galaxy/tool_shed/tool_dependencies/fabric_util.py +++ b/lib/galaxy/tool_shed/tool_dependencies/fabric_util.py @@ -34,10 +34,8 @@ yield work_dir if os.path.exists( work_dir ): local( 'rm -rf %s' % work_dir ) -def handle_post_build_processing( app, tool_dependency, install_dir, env_dependency_path, package_name=None ): - # TODO: This method is deprecated and should be eliminated when the implementation for handling proprietary fabric scripts is implemented. +def handle_command( app, tool_dependency, install_dir, cmd ): sa_session = app.model.context.current - cmd = "echo 'PATH=%s:$PATH; export PATH' > %s/env.sh;chmod +x %s/env.sh" % ( env_dependency_path, install_dir, install_dir ) output = local( cmd, capture=True ) log_results( cmd, output, os.path.join( install_dir, INSTALLATION_LOG ) ) if output.return_code: @@ -45,6 +43,7 @@ tool_dependency.error_message = str( output.stderr ) sa_session.add( tool_dependency ) sa_session.flush() + return output.return_code def install_and_build_package( app, tool_dependency, actions_dict ): """Install a Galaxy tool dependency package either via a url or a mercurial or git clone command.""" sa_session = app.model.context.current @@ -71,14 +70,8 @@ dir = work_dir elif action_type == 'shell_command': # <action type="shell_command">git clone --recursive git://github.com/ekg/freebayes.git</action> - clone_cmd = action_dict[ 'command' ] - output = local( clone_cmd, capture=True ) - log_results( clone_cmd, output, os.path.join( install_dir, INSTALLATION_LOG ) ) - if output.return_code: - tool_dependency.status = app.model.ToolDependency.installation_status.ERROR - tool_dependency.error_message = str( output.stderr ) - sa_session.add( tool_dependency ) - sa_session.flush() + return_code = handle_command( app, tool_dependency, install_dir, action_dict[ 'command' ] ) + if return_code: return dir = package_name if not os.path.exists( dir ): @@ -102,44 +95,14 @@ # Currently the only action supported in this category is "environment_variable". env_var_dicts = action_dict[ 'environment_variable' ] for env_var_dict in env_var_dicts: - env_var_name = env_var_dict[ 'name' ] - env_var_action = env_var_dict[ 'action' ] - env_var_value = env_var_dict[ 'value' ] - if env_var_action == 'prepend_to': - changed_value = '%s:$%s' % ( env_var_value, env_var_name ) - elif env_var_action == 'set_to': - changed_value = '%s' % env_var_value - elif env_var_action == 'append_to': - changed_value = '$%s:%s' % ( env_var_name, env_var_value ) - env_shell_file_path = '%s/env.sh' % install_dir - if os.path.exists( env_shell_file_path ): - write_action = '>>' - else: - write_action = '>' - cmd = "echo '%s=%s; export %s' %s %s;chmod +x %s" % ( env_var_name, - changed_value, - env_var_name, - write_action, - env_shell_file_path, - env_shell_file_path ) - output = local( cmd, capture=True ) - log_results( cmd, output, os.path.join( install_dir, INSTALLATION_LOG ) ) - if output.return_code: - tool_dependency.status = app.model.ToolDependency.installation_status.ERROR - tool_dependency.error_message = str( output.stderr ) - sa_session.add( tool_dependency ) - sa_session.flush() + cmd = common_util.create_or_update_env_shell_file( install_dir, env_var_dict ) + return_code = handle_command( app, tool_dependency, install_dir, cmd ) + if return_code: return elif action_type == 'shell_command': - action = action_dict[ 'command' ] with settings( warn_only=True ): - output = local( action, capture=True ) - log_results( action, output, os.path.join( install_dir, INSTALLATION_LOG ) ) - if output.return_code: - tool_dependency.status = app.model.ToolDependency.installation_status.ERROR - tool_dependency.error_message = str( output.stderr ) - sa_session.add( tool_dependency ) - sa_session.flush() + return_code = handle_command( app, tool_dependency, install_dir, action_dict[ 'command' ] ) + if return_code: return def log_results( command, fabric_AttributeString, file_path ): """ diff -r 1eba2def52653577163901c765fd4686c9d42dfc -r 6aadac8026cb55844a0e8e49360ad30a9f6d0a25 lib/galaxy/tool_shed/tool_dependencies/install_util.py --- a/lib/galaxy/tool_shed/tool_dependencies/install_util.py +++ b/lib/galaxy/tool_shed/tool_dependencies/install_util.py @@ -15,7 +15,10 @@ # Called from Galaxy (never the tool shed) when a new repository is being installed or when an uninstalled repository is being reinstalled. sa_session = app.model.context.current # First see if an appropriate tool_dependency record exists for the received tool_shed_repository. - tool_dependency = get_tool_dependency_by_name_version_type_repository( app, tool_shed_repository, name, version, type ) + if version: + tool_dependency = get_tool_dependency_by_name_version_type_repository( app, tool_shed_repository, name, version, type ) + else: + tool_dependency = get_tool_dependency_by_name_type_repository( app, tool_shed_repository, name, type ) if tool_dependency: if set_status: tool_dependency.status = status @@ -25,6 +28,13 @@ sa_session.add( tool_dependency ) sa_session.flush() return tool_dependency +def get_tool_dependency_by_name_type_repository( app, repository, name, type ): + sa_session = app.model.context.current + return sa_session.query( app.model.ToolDependency ) \ + .filter( and_( app.model.ToolDependency.table.c.tool_shed_repository_id == repository.id, + app.model.ToolDependency.table.c.name == name, + app.model.ToolDependency.table.c.type == type ) ) \ + .first() def get_tool_dependency_by_name_version_type_repository( app, repository, name, version, type ): sa_session = app.model.context.current return sa_session.query( app.model.ToolDependency ) \ @@ -33,13 +43,23 @@ app.model.ToolDependency.table.c.version == version, app.model.ToolDependency.table.c.type == type ) ) \ .first() -def get_tool_dependency_install_dir( app, repository, package_name, package_version ): - return os.path.abspath( os.path.join( app.config.tool_dependency_dir, - package_name, - package_version, - repository.owner, - repository.name, - repository.installed_changeset_revision ) ) +def get_tool_dependency_install_dir( app, repository, type, name, version ): + if type == 'package': + return os.path.abspath( os.path.join( app.config.tool_dependency_dir, + name, + version, + repository.owner, + repository.name, + repository.installed_changeset_revision ) ) + if type == 'set_environment': + return os.path.abspath( os.path.join( app.config.tool_dependency_dir, + 'environment_settings', + name, + repository.owner, + repository.name, + repository.installed_changeset_revision ) ) +def get_tool_shed_repository_install_dir( app, tool_shed_repository ): + return os.path.abspath( tool_shed_repository.repo_files_directory( app ) ) def install_package( app, elem, tool_shed_repository, tool_dependencies=None ): # The value of tool_dependencies is a partial or full list of ToolDependency records associated with the tool_shed_repository. sa_session = app.model.context.current @@ -49,7 +69,11 @@ package_version = elem.get( 'version', None ) if package_name and package_version: if tool_dependencies: - install_dir = get_tool_dependency_install_dir( app, tool_shed_repository, package_name, package_version ) + install_dir = get_tool_dependency_install_dir( app, + repository=tool_shed_repository, + type='package', + name=package_name, + version=package_version ) if not os.path.exists( install_dir ): for package_elem in elem: if package_elem.tag == 'install': @@ -140,11 +164,9 @@ env_var_dicts = [] for env_elem in action_elem: if env_elem.tag == 'environment_variable': - env_var_name = env_elem.get( 'name', 'PATH' ) - env_var_action = env_elem.get( 'action', 'prepend_to' ) - env_var_text = env_elem.text.replace( '$INSTALL_DIR', install_dir ) - if env_var_text: - env_var_dicts.append( dict( name=env_var_name, action=env_var_action, value=env_var_text ) ) + env_var_dict = create_env_var_dict( env_elem, tool_dependency_install_dir=install_dir ) + if env_var_dict: + env_var_dicts.append( env_var_dict ) if env_var_dicts: action_dict[ env_elem.tag ] = env_var_dicts else: @@ -206,11 +228,7 @@ return "Exception executing fabric script %s: %s. " % ( str( proprietary_fabfile_path ), str( e ) ) if returncode: return message - message = handle_post_build_processing( app, tool_dependency, install_dir, env_dependency_path, package_name=package_name ) - if message: - return message - else: - return '' + handle_environment_settings( app, tool_dependency, install_dir, cmd ) def run_subprocess( app, cmd ): env = os.environ PYTHONPATH = env.get( 'PYTHONPATH', '' ) @@ -229,3 +247,47 @@ message = '%s\n' % str( tmp_stderr.read() ) tmp_stderr.close() return returncode, message +def set_environment( app, elem, tool_shed_repository ): + """ + Create a ToolDependency to set an environment variable. This is different from the process used to set an environment variable that is associated + with a package. An example entry in a tool_dependencies.xml file is: + <set_environment version="1.0"> + <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> + </set_environment> + """ + sa_session = app.model.context.current + tool_dependency = None + env_var_version = elem.get( 'version', '1.0' ) + for env_var_elem in elem: + # The value of env_var_name must match the text value of at least 1 <requirement> tag in the tool config's <requirements> tag set whose + # "type" attribute is "set_environment" (e.g., <requirement type="set_environment">R_SCRIPT_PATH</requirement>). + env_var_name = env_var_elem.get( 'name', None ) + env_var_action = env_var_elem.get( 'action', None ) + if env_var_name and env_var_action: + install_dir = get_tool_dependency_install_dir( app, + repository=tool_shed_repository, + type='set_environment', + name=env_var_name, + version=None ) + tool_shed_repository_install_dir = get_tool_shed_repository_install_dir( app, tool_shed_repository ) + env_var_dict = create_env_var_dict( env_var_elem, tool_shed_repository_install_dir=tool_shed_repository_install_dir ) + if env_var_dict: + if not os.path.exists( install_dir ): + os.makedirs( install_dir ) + tool_dependency = create_or_update_tool_dependency( app, + tool_shed_repository, + name=env_var_name, + version=None, + type='set_environment', + status=app.model.ToolDependency.installation_status.INSTALLING, + set_status=True ) + cmd = create_or_update_env_shell_file( install_dir, env_var_dict ) + if env_var_version == '1.0': + # Handle setting environment variables using a fabric method. + handle_command( app, tool_dependency, install_dir, cmd ) + sa_session.refresh( tool_dependency ) + if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: + tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED + sa_session.add( tool_dependency ) + sa_session.flush() + print 'Environment variable ', env_var_name, 'set in', install_dir diff -r 1eba2def52653577163901c765fd4686c9d42dfc -r 6aadac8026cb55844a0e8e49360ad30a9f6d0a25 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2402,31 +2402,28 @@ command_line = self.interpreter + " " + command_line return command_line def build_dependency_shell_commands( self ): - """ - Return a list of commands to be run to populate the current - environment to include this tools requirements. - """ + """Return a list of commands to be run to populate the current environment to include this tools requirements.""" commands = [] if self.tool_shed_repository: installed_tool_dependencies = self.tool_shed_repository.installed_tool_dependencies else: installed_tool_dependencies = None for requirement in self.requirements: - # TODO: currently only supporting requirements of type package, - # need to implement some mechanism for mapping other types - # back to packages log.debug( "Building dependency shell command for dependency '%s'", requirement.name ) - if requirement.type == 'package': + script_file = None + base_path = None + version = None + if requirement.type in [ 'package', 'set_environment' ]: script_file, base_path, version = self.app.toolbox.dependency_manager.find_dep( name=requirement.name, version=requirement.version, type=requirement.type, installed_tool_dependencies=installed_tool_dependencies ) - if script_file is None and base_path is None: - log.warn( "Failed to resolve dependency on '%s', ignoring", requirement.name ) - elif script_file is None: - commands.append( 'PACKAGE_BASE=%s; export PACKAGE_BASE; PATH="%s/bin:$PATH"; export PATH' % ( base_path, base_path ) ) - else: - commands.append( 'PACKAGE_BASE=%s; export PACKAGE_BASE; . %s' % ( base_path, script_file ) ) + if script_file is None and base_path is None: + log.warn( "Failed to resolve dependency on '%s', ignoring", requirement.name ) + elif requirement.type == 'package' and script_file is None: + commands.append( 'PACKAGE_BASE=%s; export PACKAGE_BASE; PATH="%s/bin:$PATH"; export PATH' % ( base_path, base_path ) ) + else: + commands.append( 'PACKAGE_BASE=%s; export PACKAGE_BASE; . %s' % ( base_path, script_file ) ) return commands def build_redirect_url_params( self, param_dict ): """ diff -r 1eba2def52653577163901c765fd4686c9d42dfc -r 6aadac8026cb55844a0e8e49360ad30a9f6d0a25 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -9,14 +9,12 @@ class DependencyManager( object ): """ - A DependencyManager attempts to resolve named and versioned dependencies - by searching for them under a list of directories. Directories should be + A DependencyManager attempts to resolve named and versioned dependencies by searching for them under a list of directories. Directories should be of the form: $BASE/name/version/... - and should each contain a file 'env.sh' which can be sourced to make the - dependency available in the current shell environment. + and should each contain a file 'env.sh' which can be sourced to make the dependency available in the current shell environment. """ def __init__( self, base_paths=[] ): """ @@ -32,31 +30,18 @@ self.base_paths.append( os.path.abspath( base_path ) ) def find_dep( self, name, version=None, type='package', installed_tool_dependencies=None ): """ - Attempt to find a dependency named `name` at version `version`. If - version is None, return the "default" version as determined using a - symbolic link (if found). Returns a triple of: - env_script, base_path, real_version + Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a + symbolic link (if found). Returns a triple of: env_script, base_path, real_version """ if version is None: - return self._find_dep_default( name ) + return self._find_dep_default( name, type=type, installed_tool_dependencies=installed_tool_dependencies ) else: - return self._find_dep_versioned( name, version, installed_tool_dependencies=installed_tool_dependencies ) + return self._find_dep_versioned( name, version, type=type, installed_tool_dependencies=installed_tool_dependencies ) def _find_dep_versioned( self, name, version, type='package', installed_tool_dependencies=None ): - installed_dependency = None - if installed_tool_dependencies: - for installed_tool_dependency in installed_tool_dependencies: - if installed_tool_dependency.name==name and installed_tool_dependency.version==version and installed_tool_dependency.type==type: - installed_dependency = installed_tool_dependency - break + installed_tool_dependency = self._get_installed_dependency( installed_tool_dependencies, name, type, version=version ) for base_path in self.base_paths: - if installed_dependency: - tool_shed_repository = installed_dependency.tool_shed_repository - path = os.path.join( base_path, - name, - version, - tool_shed_repository.owner, - tool_shed_repository.name, - tool_shed_repository.installed_changeset_revision ) + if installed_tool_dependency: + path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) else: path = os.path.join( base_path, name, version ) script = os.path.join( path, 'env.sh' ) @@ -66,8 +51,14 @@ return None, path, version else: return None, None, None - def _find_dep_default( self, name, type='package' ): - version = None + def _find_dep_default( self, name, type='package', installed_tool_dependencies=None ): + if type == 'set_environment' and installed_tool_dependencies: + installed_tool_dependency = self._get_installed_dependency( installed_tool_dependencies, name, type, version=None ) + if installed_tool_dependency: + script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) + if script and path: + # Environment settings do not use versions. + return script, path, None for base_path in self.base_paths: path = os.path.join( base_path, name, 'default' ) if os.path.islink( path ): @@ -81,3 +72,33 @@ return None, real_path, real_version else: return None, None, None + def _get_installed_dependency( self, installed_tool_dependencies, name, type, version=None ): + for installed_tool_dependency in installed_tool_dependencies: + if version: + if installed_tool_dependency.name==name and installed_tool_dependency.type==type and installed_tool_dependency.version==version: + return installed_tool_dependency + else: + if installed_tool_dependency.name==name and installed_tool_dependency.type==type: + return installed_tool_dependency + return None + def _get_package_installed_dependency_path( self, installed_tool_dependency, base_path, name, version ): + tool_shed_repository = installed_tool_dependency.tool_shed_repository + return os.path.join( base_path, + name, + version, + tool_shed_repository.owner, + tool_shed_repository.name, + tool_shed_repository.installed_changeset_revision ) + def _get_set_environment_installed_dependency_script_path( self, installed_tool_dependency, name ): + tool_shed_repository = installed_tool_dependency.tool_shed_repository + for base_path in self.base_paths: + path = os.path.abspath( os.path.join( base_path, + 'environment_settings', + name, + tool_shed_repository.owner, + tool_shed_repository.name, + tool_shed_repository.installed_changeset_revision ) ) + if os.path.exists( path ): + script = os.path.join( path, 'env.sh' ) + return script, path, None + return None, None, None diff -r 1eba2def52653577163901c765fd4686c9d42dfc -r 6aadac8026cb55844a0e8e49360ad30a9f6d0a25 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -7,7 +7,7 @@ from galaxy.datatypes.checkers import * from galaxy.util.json import * from galaxy.tools.search import ToolBoxSearch -from galaxy.tool_shed.tool_dependencies.install_util import create_or_update_tool_dependency, install_package +from galaxy.tool_shed.tool_dependencies.install_util import create_or_update_tool_dependency, install_package, set_environment from galaxy.tool_shed.encoding_util import * from galaxy.model.orm import * @@ -255,26 +255,47 @@ """ can_generate_dependency_metadata = False for elem in root: - can_generate_dependency_metadata = False - tool_dependency_name = elem.get( 'name', None ) + tool_dependency_type = elem.tag tool_dependency_version = elem.get( 'version', None ) - tool_dependency_type = elem.tag - if tool_dependency_name and tool_dependency_version and tool_dependency_type: - for tool_dict in metadata_dict[ 'tools' ]: - requirements = tool_dict.get( 'requirements', [] ) - for requirement_dict in requirements: - req_name = requirement_dict.get( 'name', None ) - req_version = requirement_dict.get( 'version', None ) - req_type = requirement_dict.get( 'type', None ) - if req_name==tool_dependency_name and req_version==tool_dependency_version and req_type==tool_dependency_type: - can_generate_dependency_metadata = True + if tool_dependency_type == 'package': + can_generate_dependency_metadata = False + tool_dependency_name = elem.get( 'name', None ) + if tool_dependency_name and tool_dependency_version: + for tool_dict in metadata_dict[ 'tools' ]: + requirements = tool_dict.get( 'requirements', [] ) + for requirement_dict in requirements: + req_name = requirement_dict.get( 'name', None ) + req_version = requirement_dict.get( 'version', None ) + req_type = requirement_dict.get( 'type', None ) + if req_name==tool_dependency_name and req_version==tool_dependency_version and req_type==tool_dependency_type: + can_generate_dependency_metadata = True + break + if requirements and not can_generate_dependency_metadata: + # We've discovered at least 1 combination of name, version and type that is not defined in the <requirement> + # tag for any tool in the repository. break - if requirements and not can_generate_dependency_metadata: - # We've discovered at least 1 combination of name, version and type that is not defined in the <requirement> - # tag for any tool in the repository. + if not can_generate_dependency_metadata: break - if not can_generate_dependency_metadata: - break + elif tool_dependency_type == 'set_environment': + # Here elem is something like: <set_environment version="1.0"> + for env_var_elem in elem: + can_generate_dependency_metadata = False + # <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> + env_var_name = env_var_elem.get( 'name', None ) + if env_var_name: + for tool_dict in metadata_dict[ 'tools' ]: + requirements = tool_dict.get( 'requirements', [] ) + for requirement_dict in requirements: + # {"name": "R_SCRIPT_PATH", "type": "set_environment", "version": null} + req_name = requirement_dict.get( 'name', None ) + req_type = requirement_dict.get( 'type', None ) + if req_name==env_var_name and req_type==tool_dependency_type: + can_generate_dependency_metadata = True + break + if requirements and not can_generate_dependency_metadata: + # We've discovered at least 1 combination of name, version and type that is not defined in the <requirement> + # tag for any tool in the repository. + break return can_generate_dependency_metadata def check_tool_input_params( app, repo_dir, tool_config_name, tool, sample_files, webapp='galaxy' ): """ @@ -466,23 +487,43 @@ tool_dependency_objects = [] # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = get_config_from_disk( 'tool_dependencies.xml', relative_install_dir ) - tree = ElementTree.parse( tool_dependencies_config ) + try: + tree = ElementTree.parse( tool_dependencies_config ) + except Exception, e: + log.debug( "Exception attempting to parse tool_dependencies.xml: %s" %str( e ) ) + return tool_dependency_objects root = tree.getroot() ElementInclude.include( root ) fabric_version_checked = False for elem in root: - if elem.tag == 'package': - package_name = elem.get( 'name', None ) - package_version = elem.get( 'version', None ) - if package_name and package_version: + tool_dependency_type = elem.tag + if tool_dependency_type == 'package': + name = elem.get( 'name', None ) + version = elem.get( 'version', None ) + if name and version: tool_dependency = create_or_update_tool_dependency( app, tool_shed_repository, - name=package_name, - version=package_version, - type='package', + name=name, + version=version, + type=tool_dependency_type, status=app.model.ToolDependency.installation_status.NEVER_INSTALLED, set_status=set_status ) tool_dependency_objects.append( tool_dependency ) + + elif tool_dependency_type == 'set_environment': + for env_elem in elem: + # <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> + name = env_elem.get( 'name', None ) + action = env_elem.get( 'action', None ) + if name and action: + tool_dependency = create_or_update_tool_dependency( app, + tool_shed_repository, + name=name, + version=None, + type=tool_dependency_type, + status=app.model.ToolDependency.installation_status.NEVER_INSTALLED, + set_status=set_status ) + tool_dependency_objects.append( tool_dependency ) return tool_dependency_objects def generate_clone_url( trans, repository ): """Generate the URL for cloning a repository.""" @@ -528,12 +569,30 @@ if datatypes: metadata_dict[ 'datatypes' ] = datatypes return metadata_dict +def generate_environment_dependency_metadata( elem, tool_dependencies_dict ): + """The value of env_var_name must match the value of the "set_environment" type in the tool config's <requirements> tag set.""" + requirements_dict = {} + for env_elem in elem: + env_name = env_elem.get( 'name', None ) + if env_name: + requirements_dict [ 'name' ] = env_name + requirements_dict [ 'type' ] = 'environment variable' + if requirements_dict: + if 'set_environment' in tool_dependencies_dict: + tool_dependencies_dict[ 'set_environment' ].append( requirements_dict ) + else: + tool_dependencies_dict[ 'set_environment' ] = [ requirements_dict ] + return tool_dependencies_dict def generate_tool_dependency_metadata( tool_dependencies_config, metadata_dict ): """ If the combination of name, version and type of each element is defined in the <requirement> tag for at least one tool in the repository, then update the received metadata_dict with information from the parsed tool_dependencies_config. """ - tree = ElementTree.parse( tool_dependencies_config ) + try: + tree = ElementTree.parse( tool_dependencies_config ) + except Exception, e: + log.debug( "Exception attempting to parse tool_dependencies.xml: %s" %str( e ) ) + return metadata_dict root = tree.getroot() ElementInclude.include( root ) tool_dependencies_dict = {} @@ -541,9 +600,13 @@ for elem in root: if elem.tag == 'package': tool_dependencies_dict = generate_package_dependency_metadata( elem, tool_dependencies_dict ) + elif elem.tag == 'set_environment': + tool_dependencies_dict = generate_environment_dependency_metadata( elem, tool_dependencies_dict ) # Handle tool dependency installation via other means here (future). if tool_dependencies_dict: metadata_dict[ 'tool_dependencies' ] = tool_dependencies_dict + if tool_dependencies_dict: + metadata_dict[ 'tool_dependencies' ] = tool_dependencies_dict return metadata_dict def generate_metadata_for_changeset_revision( app, repository_clone_url, relative_install_dir=None, repository_files_dir=None, resetting_all_metadata_on_repository=False, webapp='galaxy' ): @@ -712,7 +775,7 @@ outputs = [] for output in ttb.outputs: name, file_name, extra = output - outputs.append( ( name, strip_path( file_name ) ) ) + outputs.append( ( name, strip_path( file_name ) if file_name else None ) ) test_dict = dict( name=ttb.name, required_files=required_files, inputs=inputs, @@ -1337,27 +1400,27 @@ ElementInclude.include( root ) fabric_version_checked = False for elem in root: - # Only install the package if it is not already installed. - can_install = False if elem.tag == 'package': + # Only install the tool_dependency if it is not already installed. + can_install = False package_name = elem.get( 'name', None ) package_version = elem.get( 'version', None ) if package_name and package_version: - # The value of tool_dependencies will be None only when this method is called by the InstallManager. In that case, tool - # dependency installation is not ajaxian, so the ToolDependency objects do not yet exist. - if tool_dependencies: - for tool_dependency in tool_dependencies: - if tool_dependency.name==package_name and tool_dependency.version==package_version: - can_install = tool_dependency.status in [ app.model.ToolDependency.installation_status.NEVER_INSTALLED, - app.model.ToolDependency.installation_status.UNINSTALLED ] - break - else: - can_install = False + for tool_dependency in tool_dependencies: + if tool_dependency.name==package_name and tool_dependency.version==package_version: + can_install = tool_dependency.status in [ app.model.ToolDependency.installation_status.NEVER_INSTALLED, + app.model.ToolDependency.installation_status.UNINSTALLED ] + break if can_install: tool_dependency = install_package( app, elem, tool_shed_repository, tool_dependencies=tool_dependencies ) if tool_dependency and tool_dependency.status in [ app.model.ToolDependency.installation_status.INSTALLED, app.model.ToolDependency.installation_status.ERROR ]: installed_tool_dependencies.append( tool_dependency ) + elif elem.tag == 'set_environment': + tool_dependency = set_environment( app, elem, tool_shed_repository ) + if tool_dependency and tool_dependency.status in [ app.model.ToolDependency.installation_status.INSTALLED, + app.model.ToolDependency.installation_status.ERROR ]: + installed_tool_dependencies.append( tool_dependency ) return installed_tool_dependencies def handle_tool_versions( app, tool_version_dicts, tool_shed_repository ): """ @@ -1662,7 +1725,12 @@ def reversed_upper_bounded_changelog( repo, included_upper_bounds_changeset_revision ): return reversed_lower_upper_bounded_changelog( repo, INITIAL_CHANGELOG_HASH, included_upper_bounds_changeset_revision ) def strip_path( fpath ): - file_path, file_name = os.path.split( fpath ) + if not fpath: + return fpath + try: + file_path, file_name = os.path.split( fpath ) + except: + file_name = fpath return file_name def to_html_escaped( text ): """Translates the characters in text to html values""" diff -r 1eba2def52653577163901c765fd4686c9d42dfc -r 6aadac8026cb55844a0e8e49360ad30a9f6d0a25 lib/galaxy/web/controllers/admin_toolshed.py --- a/lib/galaxy/web/controllers/admin_toolshed.py +++ b/lib/galaxy/web/controllers/admin_toolshed.py @@ -2,7 +2,6 @@ from galaxy.web.controllers.admin import * from galaxy.util.json import from_json_string, to_json_string from galaxy.util.shed_util import * -from galaxy.tool_shed.tool_dependencies.install_util import get_tool_dependency_install_dir from galaxy.tool_shed.encoding_util import * from galaxy import eggs, tools @@ -1364,10 +1363,11 @@ % ( repository.name, original_section_name ) message += "Uncheck the <b>No changes</b> check box and select a different tool panel section to load the tools in a " message += "different section in the tool panel." + status = 'warning' else: message = "The tools contained in your <b>%s</b> repository were last loaded into the tool panel outside of any sections. " % repository.name message += "Uncheck the <b>No changes</b> check box and select a tool panel section to load the tools into that section." - status = 'done' + status = 'warning' includes_tool_dependencies = 'tool_dependencies' in metadata install_tool_dependencies_check_box = CheckboxField( 'install_tool_dependencies', checked=True ) return trans.fill_template( '/admin/tool_shed_repository/reselect_tool_panel_section.mako', diff -r 1eba2def52653577163901c765fd4686c9d42dfc -r 6aadac8026cb55844a0e8e49360ad30a9f6d0a25 lib/galaxy/webapps/community/controllers/repository.py --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -2263,25 +2263,27 @@ guid = None revision_label = get_revision_label( trans, repository, changeset_revision ) repository_metadata = get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) - metadata = repository_metadata.metadata - if 'tools' in metadata: - for tool_metadata_dict in metadata[ 'tools' ]: - if tool_metadata_dict[ 'id' ] == tool_id: - relative_path_to_tool_config = tool_metadata_dict[ 'tool_config' ] - guid = tool_metadata_dict[ 'guid' ] - full_path = os.path.abspath( relative_path_to_tool_config ) - can_use_disk_file = can_use_tool_config_disk_file( trans, repository, repo, full_path, changeset_revision ) - if can_use_disk_file: - tool, valid, message = load_tool_from_config( trans.app, full_path ) - else: - # We're attempting to load a tool using a config that no longer exists on disk. - work_dir = tempfile.mkdtemp() - manifest_ctx, ctx_file = get_ctx_file_path_from_manifest( relative_path_to_tool_config, repo, changeset_revision ) - if manifest_ctx and ctx_file: - tool, message = load_tool_from_tmp_config( trans, repo, manifest_ctx, ctx_file, work_dir ) - break - if guid: - tool_lineage = self.get_versions_of_tool( trans, repository, repository_metadata, guid ) + if repository_metadata: + metadata = repository_metadata.metadata + if metadata: + if 'tools' in metadata: + for tool_metadata_dict in metadata[ 'tools' ]: + if tool_metadata_dict[ 'id' ] == tool_id: + relative_path_to_tool_config = tool_metadata_dict[ 'tool_config' ] + guid = tool_metadata_dict[ 'guid' ] + full_path = os.path.abspath( relative_path_to_tool_config ) + can_use_disk_file = can_use_tool_config_disk_file( trans, repository, repo, full_path, changeset_revision ) + if can_use_disk_file: + tool, valid, message = load_tool_from_config( trans.app, full_path ) + else: + # We're attempting to load a tool using a config that no longer exists on disk. + work_dir = tempfile.mkdtemp() + manifest_ctx, ctx_file = get_ctx_file_path_from_manifest( relative_path_to_tool_config, repo, changeset_revision ) + if manifest_ctx and ctx_file: + tool, message = load_tool_from_tmp_config( trans, repo, manifest_ctx, ctx_file, work_dir ) + break + if guid: + tool_lineage = self.get_versions_of_tool( trans, repository, repository_metadata, guid ) is_malicious = changeset_is_malicious( trans, repository_id, repository.tip ) changeset_revision_select_field = build_changeset_revision_select_field( trans, repository, diff -r 1eba2def52653577163901c765fd4686c9d42dfc -r 6aadac8026cb55844a0e8e49360ad30a9f6d0a25 templates/admin/tool_shed_repository/common.mako --- a/templates/admin/tool_shed_repository/common.mako +++ b/templates/admin/tool_shed_repository/common.mako @@ -72,26 +72,20 @@ <div class="form-row"><div class="toolParamHelp" style="clear: both;"><p> - These tool dependencies can be automatically installed with the repository. Installing them provides significant benefits and + These tool dependencies can be automatically handled with the installed repository, providing significant benefits, and Galaxy includes various features to manage them. </p> - <p> - Each of these dependencies may require their own build requirements (e.g., CMake, g++, etc). Galaxy will not attempt to install - these build requirements, so tool dependency installation may partially fail if any are missing from your environment, but the - repository and all of it's contents will be installed. You can install the missing build requirements and have Galaxy attempt - to install the tool dependencies again if tool dependency installation fails in any way. - </p></div></div><div class="form-row"> - <label>Install tool dependencies?</label> + <label>Handle tool dependencies?</label><% disabled = trans.app.config.tool_dependency_dir is None %> ${install_tool_dependencies_check_box.get_html( disabled=disabled )} <div class="toolParamHelp" style="clear: both;"> %if disabled: - Set the tool_dependency_dir configuration value in your universe_wsgi.ini to automatically install tool dependencies. + Set the tool_dependency_dir configuration value in your universe_wsgi.ini to automatically handle tool dependencies. %else: - Un-check to skip automatic installation of these tool dependencies. + Un-check to skip automatic handling of these tool dependencies. %endif </div></div> @@ -99,42 +93,102 @@ <div class="form-row"><table class="grid"><tr><td colspan="4" bgcolor="#D8D8D8"><b>Tool dependencies</b></td></tr> - <tr> - <th>Name</th> - <th>Version</th> - <th>Type</th> - <th>Install directory</th> - </tr> + <% + env_settings_heaader_row_displayed = False + package_header_row_displayed = False + %> %for repo_info_dict in repo_info_dicts: %for repository_name, repo_info_tuple in repo_info_dict.items(): <% description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, tool_dependencies = repo_info_tuple %> %if tool_dependencies: - %for dependency_key, requirements_dict in tool_dependencies.items(): - <% - name = requirements_dict[ 'name' ] - version = requirements_dict[ 'version' ] - type = requirements_dict[ 'type' ] - install_dir = os.path.join( trans.app.config.tool_dependency_dir, - name, - version, - repository_owner, - repository_name, - changeset_revision ) - tool_dependency_readme_text = requirements_dict.get( 'readme', None ) - %> - %if not os.path.exists( install_dir ): + <% + # See if tool dependencies are packages, environment settings or both. + contains_packages = False + for k in tool_dependencies.keys(): + if k != 'set_environment': + contains_packages = True + break + contains_env_settings = 'set_environment' in tool_dependencies.keys() + %> + %if contains_packages: + %if not package_header_row_displayed: + <% + info_str = "Each of these packages may require their own build requirements (e.g., CMake, g++, etc). " + info_str += "Galaxy will not attempt to install these build requirements, so tool dependency installation " + info_str += "may partially fail if any are missing from your environment, but the repository and all of it's " + info_str += "contents will be installed. You can install the missing build requirements and have Galaxy attempt " + info_str += "to install the packages again if tool dependency installation fails in any way." + %> + </tr><td bgcolor="#FFFFCC" colspan="4">${info_str}</td></tr><tr> - <td>${name}</td> - <td>${version}</td> - <td>${type}</td> - <td>${install_dir}</td> + <th>Name</th> + <th>Version</th> + <th>Type</th> + <th>Install directory</th></tr> - %if tool_dependency_readme_text: - <tr><td colspan="4" bgcolor="#FFFFCC">${name} ${version} requirements and installation information</td></tr> - <tr><td colspan="4"><pre>${tool_dependency_readme_text}</pre></td></tr> + <% package_header_row_displayed = True %> + %endif + %for dependency_key, requirements_dict in tool_dependencies.items(): + <% + name = requirements_dict[ 'name' ] + version = requirements_dict[ 'version' ] + type = requirements_dict[ 'type' ] + install_dir = os.path.join( trans.app.config.tool_dependency_dir, + name, + version, + repository_owner, + repository_name, + changeset_revision ) + tool_dependency_readme_text = requirements_dict.get( 'readme', None ) + %> + %if not os.path.exists( install_dir ): + <tr> + <td>${name}</td> + <td>${version}</td> + <td>${type}</td> + <td>${install_dir}</td> + </tr> + %if tool_dependency_readme_text: + <tr><td colspan="4" bgcolor="#FFFFCC">${name} ${version} requirements and installation information</td></tr> + <tr><td colspan="4"><pre>${tool_dependency_readme_text}</pre></td></tr> + %endif %endif + %endfor + %endif + %if contains_env_settings: + <% environment_settings = tool_dependencies[ 'set_environment' ] %> + %if not env_settings_heaader_row_displayed: + <% + info_str = "Each of these environment settings will be defined in a file named env.sh in the installation directory." + %> + </tr><td bgcolor="#FFFFCC" colspan="4">${info_str}</td></tr> + <tr> + <th>Name</th> + <th>Type</th> + <th colspan="2">Install directory for environment shell file</th> + </tr> + <% env_settings_heaader_row_displayed = True %> %endif - %endfor + %for environment_setting_dict in environment_settings: + <% + name = environment_setting_dict[ 'name' ] + type = environment_setting_dict[ 'type' ] + install_dir = os.path.join( trans.app.config.tool_dependency_dir, + 'environment_settings', + name, + repository_owner, + repository_name, + changeset_revision ) + %> + %if not os.path.exists( install_dir ): + <tr> + <td>${name}</td> + <td>${type}</td> + <td colspan="2">${install_dir}</td> + </tr> + %endif + %endfor + %endif %endif %endfor %endfor diff -r 1eba2def52653577163901c765fd4686c9d42dfc -r 6aadac8026cb55844a0e8e49360ad30a9f6d0a25 templates/admin/tool_shed_repository/uninstall_tool_dependencies.mako --- a/templates/admin/tool_shed_repository/uninstall_tool_dependencies.mako +++ b/templates/admin/tool_shed_repository/uninstall_tool_dependencies.mako @@ -22,12 +22,20 @@ %for tool_dependency in tool_dependencies: <input type="hidden" name="tool_dependency_ids" value="${trans.security.encode_id( tool_dependency.id )}"/><% - install_dir = os.path.join( trans.app.config.tool_dependency_dir, - tool_dependency.name, - tool_dependency.version, - tool_dependency.tool_shed_repository.owner, - tool_dependency.tool_shed_repository.name, - tool_dependency.tool_shed_repository.installed_changeset_revision ) + if tool_dependency.type == 'package': + install_dir = os.path.join( trans.app.config.tool_dependency_dir, + tool_dependency.name, + tool_dependency.version, + tool_dependency.tool_shed_repository.owner, + tool_dependency.tool_shed_repository.name, + tool_dependency.tool_shed_repository.installed_changeset_revision ) + elif tool_dependency.type == 'set_environment': + install_dir = os.path.join( trans.app.config.tool_dependency_dir, + 'environment_settings', + tool_dependency.name, + tool_dependency.tool_shed_repository.owner, + tool_dependency.tool_shed_repository.name, + tool_dependency.tool_shed_repository.installed_changeset_revision ) %> %if os.path.exists( install_dir ): <tr> diff -r 1eba2def52653577163901c765fd4686c9d42dfc -r 6aadac8026cb55844a0e8e49360ad30a9f6d0a25 templates/webapps/community/repository/common.mako --- a/templates/webapps/community/repository/common.mako +++ b/templates/webapps/community/repository/common.mako @@ -92,38 +92,76 @@ <div class="toolFormBody"> %if metadata: %if 'tool_dependencies' in metadata: - <div class="form-row"> - <table width="100%"> - <tr bgcolor="#D8D8D8" width="100%"> - <td><b>The following tool dependencies can optionally be automatically installed</i></td> - </tr> - </table> - </div> - <div style="clear: both"></div> - <div class="form-row"> - <% tool_dependencies = metadata[ 'tool_dependencies' ] %> - <table class="grid"> - <tr> - <td><b>name</b></td> - <td><b>version</b></td> - <td><b>type</b></td> - </tr> - %for dependency_key, requirements_dict in tool_dependencies.items(): - <% - name = requirements_dict[ 'name' ] - version = requirements_dict[ 'version' ] - type = requirements_dict[ 'type' ] - - %> + <% + # See if tool dependencies are packages, environment settings or both. + tool_dependencies = metadata[ 'tool_dependencies' ] + contains_packages = False + for k in tool_dependencies.keys(): + if k != 'set_environment': + contains_packages = True + break + contains_env_settings = 'set_environment' in tool_dependencies.keys() + %> + %if contains_packages: + <div class="form-row"> + <table width="100%"> + <tr bgcolor="#D8D8D8" width="100%"> + <td><b>The following tool dependencies can optionally be automatically installed</i></td> + </tr> + </table> + </div> + <div style="clear: both"></div> + <div class="form-row"> + <table class="grid"><tr> - <td>${name}</td> - <td>${version}</td> - <td>${type}</td> + <td><b>name</b></td> + <td><b>version</b></td> + <td><b>type</b></td></tr> - %endfor - </table> - </div> - <div style="clear: both"></div> + %for dependency_key, requirements_dict in tool_dependencies.items(): + %if dependency_key != 'set_environment': + <% + name = requirements_dict[ 'name' ] + version = requirements_dict[ 'version' ] + type = requirements_dict[ 'type' ] + %> + <tr> + <td>${name}</td> + <td>${version}</td> + <td>${type}</td> + </tr> + %endif + %endfor + </table> + </div> + <div style="clear: both"></div> + %endif + %if contains_env_settings: + <div class="form-row"> + <table width="100%"> + <tr bgcolor="#D8D8D8" width="100%"> + <td><b>The following environment settings can optionally be handled as part of the installation</i></td> + </tr> + </table> + </div> + <div style="clear: both"></div> + <div class="form-row"> + <table class="grid"> + <tr> + <td><b>name</b></td> + <td><b>type</b></td> + </tr> + <% environment_settings = tool_dependencies[ 'set_environment' ] %> + %for requirements_dict in environment_settings: + <tr> + <td>${requirements_dict[ 'name' ]}</td> + <td>${requirements_dict[ 'type' ]}</td> + </tr> + %endfor + </table> + </div> + <div style="clear: both"></div> + %endif %endif %if 'tools' in metadata: <div class="form-row"> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.
participants (1)
-
Bitbucket