[galaxy-commits] commit/galaxy-central: greg: Enhance and fix tool dependency installation from the tool shed to automatically source all tool dependency env.sh files defined within <setup_xxx+environment> tags. This change set also includes related code cleanup, enhanced logging and other related fixes.

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f78dcb9eb289/ Changeset: f78dcb9eb289 User: greg Date: 2013-11-19 23:05:07 Summary: Enhance and fix tool dependency installation from the tool shed to automatically source all tool dependency env.sh files defined within <setup_xxx+environment> tags. This change set also includes related code cleanup, enhanced logging and other related fixes. Affected #: 8 files diff -r d0ac928b3c1d529e32793a0fc27710e4a1585346 -r f78dcb9eb2897c452c3475d21bd601c0a91b63da lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -436,7 +436,9 @@ tool_dependencies=tool_dependencies ) for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status == trans.app.model.ToolDependency.installation_status.ERROR: - message += ' %s' % str( installed_tool_dependency.error_message ) + text = util.unicodify( installed_tool_dependency.error_message ) + if text is not None: + message += ' %s' % text tool_dependency_ids = [ trans.security.encode_id( td.id ) for td in tool_dependencies ] if message: status = 'error' @@ -681,8 +683,8 @@ action='uninstall_tool_dependencies', **kwd ) ) else: - kwd[ 'message' ] = 'All selected tool dependencies are already uninstalled.' - kwd[ 'status' ] = 'error' + message = 'No selected tool dependencies can be uninstalled, you may need to use the <b>Repair repository</b> feature.' + status = 'error' elif operation == "install": if trans.app.config.tool_dependency_dir: tool_dependencies_for_installation = [] @@ -694,13 +696,12 @@ if tool_dependencies_for_installation: self.initiate_tool_dependency_installation( trans, tool_dependencies_for_installation ) else: - kwd[ 'message' ] = 'All selected tool dependencies are already installed.' - kwd[ 'status' ] = 'error' + message = 'All selected tool dependencies are already installed.' + status = 'error' else: message = 'Set the value of your <b>tool_dependency_dir</b> setting in your Galaxy config file (universe_wsgi.ini) ' message += ' and restart your Galaxy server to install tool dependencies.' - kwd[ 'message' ] = message - kwd[ 'status' ] = 'error' + status = 'error' installed_tool_dependencies_select_field = suc.build_tool_dependencies_select_field( trans, tool_shed_repository=tool_shed_repository, name='inst_td_ids', diff -r d0ac928b3c1d529e32793a0fc27710e4a1585346 -r f78dcb9eb2897c452c3475d21bd601c0a91b63da lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -795,12 +795,11 @@ for tool_dependency in repository.missing_tool_dependencies: if tool_dependency.status in [ trans.model.ToolDependency.installation_status.ERROR, trans.model.ToolDependency.installation_status.INSTALLING ]: - tool_dependency_util.set_tool_dependency_attributes( trans, - tool_dependency, - trans.model.ToolDependency.installation_status.UNINSTALLED, - None, - remove_from_disk=True ) - trans.sa_session.refresh( tool_dependency ) + tool_dependency = tool_dependency_util.set_tool_dependency_attributes( trans.app, + tool_dependency=tool_dependency, + status=trans.model.ToolDependency.installation_status.UNINSTALLED, + error_message=None, + remove_from_disk=True ) # Install tool dependencies. suc.update_tool_shed_repository_status( trans.app, repository, diff -r d0ac928b3c1d529e32793a0fc27710e4a1585346 -r f78dcb9eb2897c452c3475d21bd601c0a91b63da lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -28,6 +28,73 @@ INSTALLATION_LOG = 'INSTALLATION.log' VIRTUALENV_URL = 'https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.9.1.tar.gz' + +class EnvFileBuilder( object ): + + def __init__( self, install_dir ): + self.install_dir = install_dir + self.return_code = 0 + + def append_line( self, skip_if_contained=True, make_executable=True, **kwd ): + env_var_dict = dict( **kwd ) + env_entry, env_file = td_common_util.create_or_update_env_shell_file( self.install_dir, env_var_dict ) + return_code = file_append( env_entry, env_file, skip_if_contained=skip_if_contained, make_executable=make_executable ) + self.return_code = self.return_code or return_code + return self.return_code + + +class InstallEnvironment( object ): + """Object describing the environment built up as part of the process of building and installing a package.""" + + def add_env_shell_file_paths( self, paths ): + for path in paths: + self.env_shell_file_paths.append( str( path ) ) + + def build_command( self, command, action_type='shell_command' ): + """ + Build command line for execution from simple command, but + configuring environment described by this object. + """ + env_cmds = self.environment_commands( action_type ) + return '\n'.join( env_cmds + [ command ] ) + + def __call__( self, install_dir ): + with settings( warn_only=True, **td_common_util.get_env_var_values( install_dir ) ): + with prefix( self.__setup_environment() ): + yield + + def environment_commands( self, action_type ): + """Build a list of commands used to construct the environment described by this object.""" + cmds = [] + for env_shell_file_path in self.env_shell_file_paths: + if os.path.exists( env_shell_file_path ): + for env_setting in open( env_shell_file_path ): + cmds.append( env_setting.strip( '\n' ) ) + else: + log.debug( 'Invalid file %s specified, ignoring %s action.' % ( str( env_shell_file_path ), str( action_type ) ) ) + return cmds + + def environment_dict( self, action_type='template_command' ): + env_vars = dict() + for env_shell_file_path in self.env_shell_file_paths: + if os.path.exists( env_shell_file_path ): + for env_setting in open( env_shell_file_path ): + env_string = env_setting.split( ';' )[ 0 ] + env_name, env_path = env_string.split( '=' ) + env_vars[ env_name ] = env_path + else: + log.debug( 'Invalid file %s specified, ignoring template_command action.' % str( env_shell_file_path ) ) + return env_vars + + def __init__( self ): + self.env_shell_file_paths = [] + + def __setup_environment( self ): + return "&&".join( [ ". %s" % file for file in self.__valid_env_shell_file_paths() ] ) + + def __valid_env_shell_file_paths( self ): + return [ file for file in self.env_shell_file_paths if os.path.exists( file ) ] + def check_fabric_version(): version = env.version if int( version.split( "." )[ 0 ] ) < 1: @@ -76,6 +143,11 @@ filtered_actions.append( action ) return filtered_actions +def handle_action_shell_file_paths( env_file_builder, action_dict ): + shell_file_paths = action_dict.get( 'action_shell_file_paths', [] ) + for shell_file_path in shell_file_paths: + env_file_builder.append_line( action="source", value=shell_file_path ) + def handle_command( app, tool_dependency, install_dir, cmd, return_output=False ): sa_session = app.model.context.current with settings( warn_only=True ): @@ -200,79 +272,6 @@ shutil.move( full_path_to_dir, venv_dir ) return True - -class EnvFileBuilder( object ): - - def __init__( self, install_dir ): - self.install_dir = install_dir - self.return_code = 0 - - def append_line( self, skip_if_contained=True, make_executable=True, **kwds ): - env_var_dict = dict(**kwds) - env_entry, env_file = td_common_util.create_or_update_env_shell_file( self.install_dir, env_var_dict ) - return_code = file_append( env_entry, env_file, skip_if_contained=skip_if_contained, make_executable=make_executable ) - self.return_code = self.return_code or return_code - return self.return_code - - -class InstallEnvironment( object ): - """ - Object describing the environment built up as part of the process of building - and installing a package. - """ - - def __init__( self ): - self.env_shell_file_paths = [] - - def build_command( self, command, action_type='shell_command' ): - """ - Build command line for execution from simple command, but - configuring environment described by this object. - """ - env_cmds = self.environment_commands(action_type) - return '\n'.join(env_cmds + [command]) - - def environment_commands(self, action_type): - """ - Build a list of commands used to construct the environment described by - this object. - """ - cmds = [] - for env_shell_file_path in self.env_shell_file_paths: - if os.path.exists( env_shell_file_path ): - for env_setting in open( env_shell_file_path ): - cmds.append( env_setting.strip( '\n' ) ) - else: - log.debug( 'Invalid file %s specified, ignoring %s action.', env_shell_file_path, action_type ) - return cmds - - def __call__( self, install_dir ): - with settings( warn_only=True, **td_common_util.get_env_var_values( install_dir ) ): - with prefix( self.__setup_environment() ): - yield - - def __setup_environment(self): - return "&&".join( [". %s" % file for file in self.__valid_env_shell_file_paths() ] ) - - def __valid_env_shell_file_paths(self): - return [ file for file in self.env_shell_file_paths if os.path.exists( file ) ] - - def environment_dict(self, action_type='template_command'): - env_vars = dict() - for env_shell_file_path in self.env_shell_file_paths: - if os.path.exists( env_shell_file_path ): - for env_setting in open( env_shell_file_path ): - env_string = env_setting.split( ';' )[ 0 ] - env_name, env_path = env_string.split( '=' ) - env_vars[ env_name ] = env_path - else: - log.debug( 'Invalid file %s specified, ignoring template_command action.', env_shell_file_path ) - return env_vars - - def add_env_shell_file_paths(self, paths): - self.env_shell_file_paths.extend(paths) - - def install_and_build_package( app, tool_dependency, actions_dict ): """Install a Galaxy tool dependency package either via a url or a mercurial or git clone command.""" sa_session = app.model.context.current @@ -335,7 +334,6 @@ dir = td_common_util.url_download( work_dir, downloaded_filename, url, extract=True ) if is_binary: log_file = os.path.join( install_dir, INSTALLATION_LOG ) - log.debug( 'log_file: %s' % log_file ) if os.path.exists( log_file ): logfile = open( log_file, 'ab' ) else: @@ -349,7 +347,7 @@ filtered_actions = actions[ 1: ] return_code = handle_command( app, tool_dependency, install_dir, action_dict[ 'command' ] ) if return_code: - return + return tool_dependency dir = package_name elif action_type == 'download_file': # <action type="download_file">http://effectors.org/download/version/TTSS_GUI-1.0.1.jar</action> @@ -374,13 +372,15 @@ # <package>https://github.com/bgruening/download_store/raw/master/DESeq2-1_0_18/BiocGenerics_0.6.0.tar.gz</package> # </action> filtered_actions = actions[ 1: ] - - if action_dict.get( 'env_shell_file_paths', False ): - install_environment.add_env_shell_file_paths( action_dict[ 'env_shell_file_paths' ] ) + env_shell_file_paths = action_dict.get( 'env_shell_file_paths', None ) + if env_shell_file_paths is None: + log.debug( 'Missing R environment. Please check your specified R installation exists.' ) + return tool_dependency else: - log.warning( 'Missing R environment. Please check your specified R installation exists.' ) - return - tarball_names = list() + install_environment.add_env_shell_file_paths( env_shell_file_paths ) + log.debug( 'Handling setup_r_environment for tool dependency %s with install_environment.env_shell_file_paths:\n%s"' % \ + ( str( tool_dependency.name ), str( install_environment.env_shell_file_paths ) ) ) + tarball_names = [] for url in action_dict[ 'r_packages' ]: filename = url.split( '/' )[ -1 ] tarball_names.append( filename ) @@ -391,20 +391,18 @@ with settings( warn_only=True ): for tarball_name in tarball_names: cmd = '''export PATH=$PATH:$R_HOME/bin && export R_LIBS=$INSTALL_DIR && - Rscript -e "install.packages(c('%s'),lib='$INSTALL_DIR', repos=NULL, dependencies=FALSE)"''' % (tarball_name) - + Rscript -e "install.packages(c('%s'),lib='$INSTALL_DIR', repos=NULL, dependencies=FALSE)"''' % ( str( tarball_name ) ) cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_dir ) ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: - return - + return tool_dependency # R libraries are installed to $INSTALL_DIR (install_dir), we now set the R_LIBS path to that directory env_file_builder = EnvFileBuilder( install_dir ) handle_action_shell_file_paths( env_file_builder, action_dict ) # Pull in R environment (runtime). env_file_builder.append_line( name="R_LIBS", action="prepend_to", value=install_dir ) return_code = env_file_builder.return_code if return_code: - return + return tool_dependency elif action_type == 'setup_ruby_environment': # setup an Ruby environment # <action type="setup_ruby_environment"> @@ -417,24 +415,27 @@ # <package>http://url-to-some-gem-file.de/protk.gem</package> # </action> filtered_actions = actions[ 1: ] - - if action_dict.get( 'env_shell_file_paths', False ): - install_environment.add_env_shell_file_paths( action_dict[ 'env_shell_file_paths' ] ) + env_shell_file_paths = action_dict.get( 'env_shell_file_paths', None ) + if env_shell_file_paths is None: + log.debug( 'Missing Ruby environment, make sure your specified Ruby installation exists.' ) + return tool_dependency else: - log.warning( 'Missing Ruby environment. Please check if your specified Ruby installation exists.' ) - return - + install_environment.add_env_shell_file_paths( env_shell_file_paths ) + log.debug( 'Handling setup_ruby_environment for tool dependency %s with install_environment.env_shell_file_paths:\n%s"' % \ + ( str( tool_dependency.name ), str( install_environment.env_shell_file_paths ) ) ) dir = os.path.curdir current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) with lcd( current_dir ): with settings( warn_only=True ): - for (gem, gem_version) in action_dict[ 'ruby_packages' ]: + ruby_package_tups = action_dict.get( 'ruby_package_tups', [] ) + for ruby_package_tup in ruby_package_tups: + gem, gem_version = ruby_package_tup if os.path.isfile( gem ): # we assume a local shipped gem file cmd = '''export PATH=$PATH:$RUBY_HOME/bin && export GEM_HOME=$INSTALL_DIR && gem install --local %s''' % ( gem ) - elif gem.find('://') != -1: - # we assume a URL to a gem file + elif gem.find( '://' ) != -1: + # We assume a URL to a gem file. url = gem gem_name = url.split( '/' )[ -1 ] td_common_util.url_download( work_dir, gem_name, url, extract=False ) @@ -453,15 +454,15 @@ cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_dir ) ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: - return - + return tool_dependency env_file_builder = EnvFileBuilder( install_dir ) - handle_action_shell_file_paths( env_file_builder, action_dict ) # Pull in ruby dependencies (runtime). + # Pull in ruby dependencies (runtime). + handle_action_shell_file_paths( env_file_builder, action_dict ) env_file_builder.append_line( name="GEM_PATH", action="prepend_to", value=install_dir ) env_file_builder.append_line( name="PATH", action="prepend_to", value=os.path.join(install_dir, 'bin') ) return_code = env_file_builder.return_code if return_code: - return + return tool_dependency elif action_type == 'setup_perl_environment': # setup an Perl environment # <action type="setup_perl_environment"> @@ -473,65 +474,59 @@ # <package>http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz</package> # </action> filtered_actions = actions[ 1: ] - - if action_dict.get( 'env_shell_file_paths', False ): - install_environment.add_env_shell_file_paths( action_dict[ 'env_shell_file_paths' ] ) + env_shell_file_paths = action_dict.get( 'env_shell_file_paths', None ) + if env_shell_file_paths is None: + log.debug( 'Missing Rerl environment, make sure your specified Rerl installation exists.' ) + return tool_dependency else: - log.warning( 'Missing Rerl environment. Please check if your specified Rerl installation exists.' ) - return - + install_environment.add_env_shell_file_paths( env_shell_file_paths ) + log.debug( 'Handling setup_perl_environment for tool dependency %s with install_environment.env_shell_file_paths:\n%s"' % \ + ( str( tool_dependency.name ), str( install_environment.env_shell_file_paths ) ) ) dir = os.path.curdir current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) with lcd( current_dir ): with settings( warn_only=True ): - - for package in action_dict[ 'perl_packages' ]: - """ - If set to a true value then MakeMaker's prompt function will always return the default without waiting for user input. - """ + perl_packages = action_dict.get( 'perl_packages', [] ) + for perl_package in perl_packages: + # If set to a true value then MakeMaker's prompt function will always + # return the default without waiting for user input. cmd = '''export PERL_MM_USE_DEFAULT=1 && ''' - - if package.find('://') != -1: - # we assume a URL to a gem file - url = package - package_name = url.split( '/' )[ -1 ] - dir = td_common_util.url_download( work_dir, package_name, url, extract=True ) - # search for Build.PL or Makefile.PL (ExtUtils::MakeMaker vs. Module::Build) - - tmp_work_dir = os.path.join( work_dir, dir) + if perl_package.find( '://' ) != -1: + # We assume a URL to a gem file. + url = perl_package + perl_package_name = url.split( '/' )[ -1 ] + dir = td_common_util.url_download( work_dir, perl_package_name, url, extract=True ) + # Search for Build.PL or Makefile.PL (ExtUtils::MakeMaker vs. Module::Build). + tmp_work_dir = os.path.join( work_dir, dir ) if os.path.exists( os.path.join( tmp_work_dir, 'Makefile.PL' ) ): - cmd += '''perl Makefile.PL INSTALL_BASE=$INSTALL_DIR && make && make install''' elif os.path.exists( os.path.join( tmp_work_dir, 'Build.PL' ) ): cmd += '''perl Build.PL --install_base $INSTALL_DIR && perl Build && perl Build install''' else: - log.warning( 'No Makefile.PL or Build.PL file found in %s. Skip installation of %s.' % ( url, package_name ) ) - return + log.debug( 'No Makefile.PL or Build.PL file found in %s. Skipping installation of %s.' % ( url, perl_package_name ) ) + return tool_dependency with lcd( tmp_work_dir ): cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_dir ) ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: - return + return tool_dependency else: - # perl package from CPAN without version number - # cpanm should be installed with the parent perl distribution, otherwise this will not work - cmd += '''cpanm --local-lib=$INSTALL_DIR %s''' % ( package ) - + # perl package from CPAN without version number. + # cpanm should be installed with the parent perl distribution, otherwise this will not work. + cmd += '''cpanm --local-lib=$INSTALL_DIR %s''' % ( perl_package ) cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_dir ) ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: - return - + return tool_dependency env_file_builder = EnvFileBuilder( install_dir ) + # Pull in perl dependencies (runtime). + handle_action_shell_file_paths( env_file_builder, action_dict ) # Recursively add dependent PERL5LIB and PATH to env.sh & anything else needed. - handle_action_shell_file_paths( env_file_builder, action_dict ) # Pull in ruby dependencies (runtime). - - env_file_builder.append_line( name="PERL5LIB", action="prepend_to", value=os.path.join(install_dir, 'lib', 'perl5') ) - env_file_builder.append_line( name="PATH", action="prepend_to", value=os.path.join(install_dir, 'bin') ) + env_file_builder.append_line( name="PERL5LIB", action="prepend_to", value=os.path.join( install_dir, 'lib', 'perl5' ) ) + env_file_builder.append_line( name="PATH", action="prepend_to", value=os.path.join( install_dir, 'bin' ) ) return_code = env_file_builder.return_code if return_code: - return - + return tool_dependency else: # We're handling a complex repository dependency where we only have a set_environment tag set. # <action type="set_environment"> @@ -577,7 +572,7 @@ env_file_builder.append_line( **env_var_dict ) return_code = env_file_builder.return_code if return_code: - return + return tool_dependency elif action_type == 'set_environment_for_install': # Currently the only action supported in this category is a list of paths to one or more tool dependency env.sh files, # the environment setting in each of which will be injected into the environment for all <action type="shell_command"> @@ -588,7 +583,7 @@ venv_src_directory = os.path.abspath( os.path.join( app.config.tool_dependency_dir, '__virtualenv_src' ) ) if not install_virtualenv( app, venv_src_directory ): log.error( 'Unable to install virtualenv' ) - return + return tool_dependency requirements = action_dict[ 'requirements' ] if os.path.exists( os.path.join( dir, requirements ) ): # requirements specified as path to a file @@ -608,26 +603,26 @@ full_setup_command = "%s; %s; %s" % ( setup_command, activate_command, install_command ) return_code = handle_command( app, tool_dependency, install_dir, full_setup_command ) if return_code: - return + return tool_dependency site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join( venv_directory, "bin", "python" ) output = handle_command( app, tool_dependency, install_dir, site_packages_command, return_output=True ) if output.return_code: - return + return tool_dependency if not os.path.exists( output.stdout ): - log.error( "virtualenv's site-packages directory '%s' does not exist", output.stdout ) - return + log.debug( "virtualenv's site-packages directory '%s' does not exist", output.stdout ) + return tool_dependency env_file_builder = EnvFileBuilder( install_dir ) env_file_builder.append_line( name="PYTHONPATH", action="prepend_to", value=output.stdout ) env_file_builder.append_line( name="PATH", action="prepend_to", value=os.path.join( venv_directory, "bin" ) ) return_code = env_file_builder.return_code if return_code: - return + return tool_dependency elif action_type == 'shell_command': with settings( warn_only=True ): cmd = install_environment.build_command( action_dict[ 'command' ] ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: - return + return tool_dependency elif action_type == 'template_command': env_vars = dict() env_vars = install_environment.environment_dict() @@ -639,7 +634,7 @@ cmd = fill_template( '#from fabric.api import env\n%s' % action_dict[ 'command' ], context=env_vars ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: - return + return tool_dependency elif action_type == 'make_install': # make; make install; allow providing make options with settings( warn_only=True ): @@ -647,7 +642,7 @@ cmd = install_environment.build_command( 'make %s && make install' % make_opts ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: - return + return tool_dependency elif action_type == 'autoconf': # Handle configure, make and make install allow providing configuration options with settings( warn_only=True ): @@ -659,7 +654,7 @@ cmd = install_environment.build_command( td_common_util.evaluate_template( pre_cmd, install_dir ) ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: - return + return tool_dependency elif action_type == 'download_file': # Download a single file to the current working directory. url = action_dict[ 'url' ] @@ -705,13 +700,7 @@ td_common_util.move_file( current_dir=work_dir, source=downloaded_filename, destination=full_path_to_dir ) - - -def handle_action_shell_file_paths( env_file_builder, action_dict ): - shell_file_paths = action_dict.get( 'action_shell_file_paths', []) - for shell_file_path in shell_file_paths: - env_file_builder.append_line( action="source", value=shell_file_path ) - + return tool_dependency def log_results( command, fabric_AttributeString, file_path ): """ diff -r d0ac928b3c1d529e32793a0fc27710e4a1585346 -r f78dcb9eb2897c452c3475d21bd601c0a91b63da lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -92,7 +92,7 @@ return text def handle_complex_repository_dependency_for_package( app, elem, package_name, package_version, tool_shed_repository ): - tool_dependency = None + handled_tool_dependencies = [] tool_shed = elem.attrib[ 'toolshed' ] required_repository_name = elem.attrib[ 'name' ] required_repository_owner = elem.attrib[ 'owner' ] @@ -156,26 +156,28 @@ required_repository_owner, required_repository_changeset_revision ) config_to_use = tmp_filename - tool_dependency, actions_dict = populate_actions_dict( app=app, - dependent_install_dir=dependent_install_dir, - required_install_dir=required_repository_package_install_dir, - tool_shed_repository=tool_shed_repository, - required_repository=required_repository, - package_name=package_name, - package_version=package_version, - tool_dependencies_config=config_to_use ) + tool_dependencies, actions_dict = populate_actions_dict( app=app, + dependent_install_dir=dependent_install_dir, + required_install_dir=required_repository_package_install_dir, + tool_shed_repository=tool_shed_repository, + required_repository=required_repository, + package_name=package_name, + package_version=package_version, + tool_dependencies_config=config_to_use ) if tmp_filename: try: os.remove( tmp_filename ) except: pass - # Install and build the package via fabric. - install_and_build_package_via_fabric( app, tool_dependency, actions_dict ) + for tool_dependency in tool_dependencies: + # Install and build the package via fabric and update the tool_dependency record accordingly.. + tool_dependency = install_and_build_package_via_fabric( app, tool_dependency, actions_dict ) + handled_tool_dependencies.append( tool_dependency ) else: message = "Unable to locate required tool shed repository named %s owned by %s with revision %s." % \ ( str( required_repository_name ), str( required_repository_owner ), str( default_required_repository_changeset_revision ) ) raise Exception( message ) - return tool_dependency + return handled_tool_dependencies def handle_set_environment_entry_for_package( app, install_dir, tool_shed_repository, package_name, package_version, elem, required_repository ): """ @@ -184,19 +186,24 @@ """ action_dict = {} actions = [] + tool_dependencies = [] for package_elem in elem: if package_elem.tag == 'install': - # Create the tool_dependency record in the database. + # Create the new tool_dependency record in the database. tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app, tool_shed_repository=tool_shed_repository, name=package_name, version=package_version, type='package', - status=app.model.ToolDependency.installation_status.INSTALLING, + status=app.model.ToolDependency.installation_status.NEVER_INSTALLED, set_status=True ) # Get the installation method version from a tag like: <install version="1.0"> package_install_version = package_elem.get( 'version', '1.0' ) if package_install_version == '1.0': + # Update the tool dependency's status. + tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, + tool_dependency=tool_dependency, + status=app.model.ToolDependency.installation_status.INSTALLING ) # Since the required tool dependency is installed for a repository dependency, we first need to inspect the <actions> tag set to find # the <action type="set_environment"> tag. env_var_dicts = [] @@ -219,9 +226,10 @@ site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join( install_dir, "venv", "bin", "python" ) output = fabric_util.handle_command( app, tool_dependency, install_dir, site_packages_command, return_output=True ) if output.return_code: - log.error( 'Dependency includes a setup_virtualenv action but venv python is broken:', output.stderr ) + log.error( 'Tool dependency %s includes a setup_virtualenv action but venv python is broken: ' % \ + ( str( tool_dependency.name ), str( output.stderr ) ) ) elif not os.path.exists( output.stdout ): - log.error( "virtualenv's site-packages directory '%s' does not exist", output.stdout ) + log.error( "virtualenv's site-packages directory '%s' does not exist", str( output.stdout ) ) else: env_var_dicts.append( dict( name="PYTHONPATH", action="prepend_to", value=output.stdout ) ) env_var_dicts.append( dict( name="PATH", action="prepend_to", value=os.path.join( install_dir, 'venv', 'bin' ) ) ) @@ -234,8 +242,8 @@ # we will replace the current "value" entries in each env_var_dict with the actual path taken from the env.sh # file generated for the installed required repository. Each env_var_dict currently looks something like this: # {'action': 'append_to', 'name': 'LD_LIBRARY_PATH', 'value': '$BOOST_ROOT_DIR/lib/'} - # We'll read the contents of the received required_repository's env.sh file and replace the 'value' entry of each env_var_dict - # with the associated value in the env.sh file. + # We'll read the contents of the received required_repository's env.sh file and replace the 'value' entry of + # each env_var_dict with the associated value in the env.sh file. new_env_var_dicts = [] env_sh_file_dir = tool_dependency_util.get_tool_dependency_install_dir( app=app, repository_name=required_repository.name, @@ -265,16 +273,25 @@ else: action_dict[ 'environment_variable' ] = env_var_dicts actions.append( ( 'set_environment', action_dict ) ) + if tool_dependency.status not in [ app.model.ToolDependency.installation_status.ERROR, + app.model.ToolDependency.installation_status.INSTALLED ]: + # Update the tool dependency's status. + tool_dependency = \ + tool_dependency_util.set_tool_dependency_attributes( app, + tool_dependency=tool_dependency, + status=app.model.ToolDependency.installation_status.INSTALLED ) + # Accumulate processed tool dependencies to return to the caller. + tool_dependencies.append( tool_dependency ) else: raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).' ) - return tool_dependency, actions - return None, actions + return tool_dependencies, actions + return tool_dependencies, actions def install_and_build_package_via_fabric( app, tool_dependency, actions_dict ): sa_session = app.model.context.current try: # There is currently only one fabric method. - fabric_util.install_and_build_package( app, tool_dependency, actions_dict ) + tool_dependency = fabric_util.install_and_build_package( app, tool_dependency, actions_dict ) except Exception, e: log.exception( 'Error installing tool dependency %s version %s.', str( tool_dependency.name ), str( tool_dependency.version ) ) # Since there was an installation error, update the tool dependency status to Error. The remove_installation_path option must @@ -284,10 +301,16 @@ tool_dependency, error_message, remove_installation_path=False ) - if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: - tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED - sa_session.add( tool_dependency ) - sa_session.flush() + if tool_dependency.status not in [ app.model.ToolDependency.installation_status.ERROR, + app.model.ToolDependency.installation_status.INSTALLED ]: + log.debug( 'Changing status for tool dependency %s from %s to %s.' % \ + ( str( tool_dependency.name ), str( tool_dependency.status ), str( app.model.ToolDependency.installation_status.INSTALLED ) ) ) + tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, + tool_dependency=tool_dependency, + status=app.model.ToolDependency.installation_status.INSTALLED, + error_message=None, + remove_from_disk=False ) + return tool_dependency def install_package( app, elem, tool_shed_repository, tool_dependencies=None ): # The value of tool_dependencies is a partial or full list of ToolDependency records associated with the tool_shed_repository. @@ -300,9 +323,15 @@ for package_elem in elem: if package_elem.tag == 'repository': # We have a complex repository dependency definition. - rd_tool_dependency = handle_complex_repository_dependency_for_package( app, package_elem, package_name, package_version, tool_shed_repository ) - if rd_tool_dependency and rd_tool_dependency.status == app.model.ToolDependency.installation_status.ERROR: - print "Error installing tool dependency for required repository: %s" % str( rd_tool_dependency.error_message ) + rd_tool_dependencies = handle_complex_repository_dependency_for_package( app, + package_elem, + package_name, + package_version, + tool_shed_repository ) + for rd_tool_dependency in rd_tool_dependencies: + if rd_tool_dependency.status == app.model.ToolDependency.installation_status.ERROR: + # We'll log the error here, but continue installing packages since some may not require this dependency. + print "Error installing tool dependency for required repository: %s" % str( rd_tool_dependency.error_message ) elif package_elem.tag == 'install': # <install version="1.0"> # Get the installation directory for tool dependencies that will be installed for the received tool_shed_repository. @@ -324,24 +353,27 @@ package_version, 'package' ) if tool_dependency.status == app.model.ToolDependency.installation_status.INSTALLING: - # The tool dependency is in an Installing state, so we don't want to do anything to it. + # The tool dependency is in an Installing state, so we don't want to do anything to it. If the tool + # dependency is being installed by someone else, we don't want to interfere with that. This assumes + # the installation by "someone else" is not hung in an Installing state, which is a weakness if that + # "someone else" never repaired it. log.debug( 'Skipping installation of tool dependency %s version %s because it has a status of %s' % \ - ( tool_dependency.name, tool_dependency.version, tool_dependency.status ) ) + ( str( tool_dependency.name ), str( tool_dependency.version ), str( tool_dependency.status ) ) ) can_install_tool_dependency = False else: - # If the tool dependency is being installed by someone else, we don't want to interfere with that. - # This assumes the installation by "someone else" is not hung in an Installing state, which is a - # weakness if "someone else" never repaired it. tool_dependency_installation_directory_contents = os.listdir( install_dir ) if fabric_util.INSTALLATION_LOG in tool_dependency_installation_directory_contents: - print '\nSkipping installation of tool dependency', package_name, 'version', package_version, \ - 'since it is installed in', install_dir, '\n' + # Since this tool dependency's installation directory contains an installation log, we consider it to be + # installed. In some cases the record may be missing from the database due to some activity outside of + # the control of the Tool Shed. Since a new record was created for it and we don't know the state of the + # files on disk, we will set it to an error state (unless we are running Tool Shed functional tests - see + # below). + log.debug( 'Skipping installation of tool dependency %s version %s because it is installed in %s' % \ + ( str( tool_dependency.name ), str( tool_dependency.version ), str( install_dir ) ) ) can_install_tool_dependency = False - # This tool dependency was previously installed, but the record was missing from the database due to some - # activity outside of the control of the tool shed. Since a new record was created for it and we don't know - # the state of the files on disk, we will set it to an error state. If we are running functional tests, the - # state will be set to Installed, because previously compiled tool dependencies are not deleted by default. if app.config.running_functional_tests: + # If we are running functional tests, the state will be set to Installed because previously compiled + # tool dependencies are not deleted by default, from the "install and test" framework.. tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED else: error_message = 'The installation directory for this tool dependency had contents, but the database had no record. ' @@ -351,9 +383,9 @@ tool_dependency.error_message = error_message else: error_message = '\nInstallation path %s for tool dependency %s version %s exists, but the expected file %s' % \ - ( install_dir, package_name, package_version, fabric_util.INSTALLATION_LOG ) - error_message += ' is missing. This indicates an installation error, so the tool dependency is being' - error_message += ' prepared for reinstallation.' + ( str( install_dir ), str( package_name ), str( package_version ), str( fabric_util.INSTALLATION_LOG ) ) + error_message += ' is missing. This indicates an installation error so the tool dependency is being' + error_message += ' prepared for re-installation.' print error_message tool_dependency.status = app.model.ToolDependency.installation_status.NEVER_INSTALLED try: @@ -373,7 +405,7 @@ version=package_version, type='package', status=app.model.ToolDependency.installation_status.INSTALLING, - set_status=False ) + set_status=True ) # Get the information about the current platform in case the tool dependency definition includes tag sets for installing # compiled binaries. platform_info_dict = tool_dependency_util.get_platform_info_dict() @@ -404,13 +436,12 @@ if binary_installed: continue # No platform-specific <actions> recipe has yet resulted in a successful installation. - install_via_fabric( app, - tool_dependency, - install_dir, - package_name=package_name, - actions_elem=actions_elem, - action_elem=None ) - sa_session.refresh( tool_dependency ) + tool_dependency = install_via_fabric( app, + tool_dependency, + install_dir, + package_name=package_name, + actions_elem=actions_elem, + action_elem=None ) if tool_dependency.status == app.model.ToolDependency.installation_status.INSTALLED: # If an <actions> tag was found that matches the current platform, and the install_via_fabric method # did not result in an error state, set binary_installed to True in order to skip any remaining @@ -419,47 +450,53 @@ else: # Process the next matching <actions> tag, or any defined <actions> tags that do not contain platform # dependent recipes. - print 'Error downloading binary for %s version %s: %s' % \ - ( package_name, package_version, tool_dependency.error_message ) + log.debug( 'Error downloading binary for tool dependency %s version %s: %s' % \ + ( str( package_name ), str( package_version ), str( tool_dependency.error_message ) ) ) else: # If no <actions> tags have been defined that match our current platform, or none of the matching # <actions> tags resulted in a successful tool dependency status, proceed with one and only one # <actions> tag that is not defined to be platform-specific. if not binary_installed: - print 'Binary installation did not occur, so proceeding with install and compile recipe.' + log.debug( 'Proceeding with install and compile recipe for tool dependency %s.' % str( tool_dependency.name ) ) # Make sure to reset for installation if attempt at binary installation resulted in an error. + can_install = True if tool_dependency.status != app.model.ToolDependency.installation_status.NEVER_INSTALLED: removed, error_message = tool_dependency_util.remove_tool_dependency( app, tool_dependency ) - install_via_fabric( app, - tool_dependency, - install_dir, - package_name=package_name, - actions_elem=actions_elem, - action_elem=None ) + if not removed: + log.debug( 'Error removing old files from installation directory %s: %s' % \ + ( str( tool_dependency.installation_directory( app ), str( error_message ) ) ) ) + can_install = False + if can_install: + tool_dependency = install_via_fabric( app, + tool_dependency, + install_dir, + package_name=package_name, + actions_elem=actions_elem, + action_elem=None ) # Perform any final actions that have been defined within the actions_group tag set, but outside of # an <actions> tag, such as a set_environment entry, or a download_file or download_by_url command to # retrieve extra data for this tool dependency. Only do this if the tool dependency is not in an error # state, otherwise skip this action. if actions_elem.tag == 'action' and tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: - install_via_fabric( app, - tool_dependency, - install_dir, - package_name=package_name, - actions_elem=None, - action_elem=actions_elem ) + tool_dependency = install_via_fabric( app, + tool_dependency, + install_dir, + package_name=package_name, + actions_elem=None, + action_elem=actions_elem ) else: # <actions> tags outside of an <actions_group> tag shall not check os or architecture, and if the attributes are # defined, they will be ignored. All <actions> tags outside of an <actions_group> tag set shall always be processed. # This is the default and original behavior of the install_package method. - install_via_fabric( app, - tool_dependency, - install_dir, - package_name=package_name, - actions_elem=actions_elems, - action_elem=None ) - sa_session.refresh( tool_dependency ) + tool_dependency = install_via_fabric( app, + tool_dependency, + install_dir, + package_name=package_name, + actions_elem=actions_elems, + action_elem=None ) if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: - print package_name, 'version', package_version, 'installed in', install_dir + log.debug( 'Tool dependency %s version %s has been installed in %s.' % \ + ( str( package_name ), str( package_version ), str( install_dir ) ) ) else: error_message = 'Version %s of the %s package cannot be installed because ' % ( str( package_version ), str( package_name ) ) error_message += 'the recipe for installing the package is missing either an <actions> tag set or an <actions_group> ' @@ -489,7 +526,6 @@ def install_via_fabric( app, tool_dependency, install_dir, package_name=None, proprietary_fabfile_path=None, actions_elem=None, action_elem=None, **kwd ): """Parse a tool_dependency.xml file's <actions> tag set to gather information for the installation via fabric.""" - sa_session = app.model.context.current if not os.path.exists( install_dir ): os.makedirs( install_dir ) @@ -657,7 +693,7 @@ configure_opts = td_common_util.evaluate_template( action_elem.text, install_dir ) action_dict[ 'configure_opts' ] = configure_opts elif action_type == 'setup_r_environment': - # setup an R environment + # setup an R environment. # <action type="setup_r_environment"> # <repository name="package_r_3_0_1" owner="bgruening"> # <package name="R" version="3.0.1" /> @@ -665,18 +701,19 @@ # <!-- allow installing an R packages --> # <package>https://github.com/bgruening/download_store/raw/master/DESeq2-1_0_18/BiocGenerics_0.6.0.tar.gz</package> # </action> - td_common_util.parse_setup_environment_repositories( app, all_env_shell_file_paths, action_elem, action_dict ) + # Discover all child repository dependency tags and define the path to an env.sh file associated with each repository. + # This will potentially update the value of the 'env_shell_file_paths' entry in action_dict. + action_dict = td_common_util.get_env_shell_file_paths_from_setup_environment_elem( app, all_env_shell_file_paths, action_elem, action_dict ) r_packages = list() for env_elem in action_elem: if env_elem.tag == 'package': r_packages.append( env_elem.text.strip() ) - if r_packages: action_dict[ 'r_packages' ] = r_packages else: continue elif action_type == 'setup_ruby_environment': - # setup an Ruby environment + # setup a Ruby environment. # <action type="setup_ruby_environment"> # <repository name="package_ruby_2_0" owner="bgruening"> # <package name="ruby" version="2.0" /> @@ -686,33 +723,32 @@ # <package>protk=1.2.4</package> # <package>http://url-to-some-gem-file.de/protk.gem</package> # </action> - td_common_util.parse_setup_environment_repositories( app, all_env_shell_file_paths, action_elem, action_dict ) - ruby_packages = list() + # Discover all child repository dependency tags and define the path to an env.sh file associated with each repository. + # This will potentially update the value of the 'env_shell_file_paths' entry in action_dict. + action_dict = td_common_util.get_env_shell_file_paths_from_setup_environment_elem( app, all_env_shell_file_paths, action_elem, action_dict ) + ruby_package_tups = [] for env_elem in action_elem: if env_elem.tag == 'package': - """ - A valid gem definition can be: - protk=1.2.4 - protk - ftp://ftp.gruening.de/protk.gem - """ - gem_token = env_elem.text.strip().split('=') - if len(gem_token) == 2: + #A valid gem definition can be: + # protk=1.2.4 + # protk + # ftp://ftp.gruening.de/protk.gem + gem_token = env_elem.text.strip().split( '=' ) + if len( gem_token ) == 2: # version string - gem_name = gem_token[0] - gem_version = gem_token[1] - ruby_packages.append( [gem_name, gem_version] ) + gem_name = gem_token[ 0 ] + gem_version = gem_token[ 1 ] + ruby_package_tups.append( ( gem_name, gem_version ) ) else: # gem name for rubygems.org without version number gem = env_elem.text.strip() - ruby_packages.append( [gem, None] ) - - if ruby_packages: - action_dict[ 'ruby_packages' ] = ruby_packages + ruby_package_tups.append( ( gem, None ) ) + if ruby_package_tups: + action_dict[ 'ruby_package_tups' ] = ruby_package_tups else: continue elif action_type == 'setup_perl_environment': - # setup an Perl environment + # setup a Perl environment. # <action type="setup_perl_environment"> # <repository name="package_perl_5_18" owner="bgruening"> # <package name="perl" version="5.18.1" /> @@ -721,17 +757,17 @@ # <package>XML::Parser</package> # <package>http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz</package> # </action> - td_common_util.parse_setup_environment_repositories( app, all_env_shell_file_paths, action_elem, action_dict ) - perl_packages = list() + # Discover all child repository dependency tags and define the path to an env.sh file associated with each repository. + # This will potentially update the value of the 'env_shell_file_paths' entry in action_dict. + action_dict = td_common_util.get_env_shell_file_paths_from_setup_environment_elem( app, all_env_shell_file_paths, action_elem, action_dict ) + perl_packages = [] for env_elem in action_elem: if env_elem.tag == 'package': - """ - A valid package definition can be: - XML::Parser - http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz - Unfortunately, CPAN does not support versioning. If you want real Reproducibility, - you need to specify the tarball path and the right order of different tarballs manually. - """ + # A valid package definition can be: + # XML::Parser + # http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz + # Unfortunately CPAN does not support versioning, so if you want real reproducibility you need to specify + # the tarball path and the right order of different tarballs manually. perl_packages.append( env_elem.text.strip() ) if perl_packages: action_dict[ 'perl_packages' ] = perl_packages @@ -782,7 +818,8 @@ # run_proprietary_fabric_method( app, elem, proprietary_fabfile_path, install_dir, package_name=package_name ) raise Exception( 'Tool dependency installation using proprietary fabric scripts is not yet supported.' ) else: - install_and_build_package_via_fabric( app, tool_dependency, actions_dict ) + tool_dependency = install_and_build_package_via_fabric( app, tool_dependency, actions_dict ) + return tool_dependency def parse_env_shell_entry( action, name, value, line ): new_value = value @@ -814,7 +851,7 @@ actions_dict = dict( install_dir=dependent_install_dir ) if package_name: actions_dict[ 'package_name' ] = package_name - tool_dependency = None + tool_dependencies = [] action_dict = {} if tool_dependencies_config: required_td_tree, error_message = xml_util.parse_xml( tool_dependencies_config ) @@ -827,17 +864,17 @@ required_td_package_name = required_td_elem.get( 'name', None ) required_td_package_version = required_td_elem.get( 'version', None ) if required_td_package_name==package_name and required_td_package_version==package_version: - tool_dependency, actions = handle_set_environment_entry_for_package( app=app, - install_dir=required_install_dir, - tool_shed_repository=tool_shed_repository, - package_name=package_name, - package_version=package_version, - elem=required_td_elem, - required_repository=required_repository ) + tool_dependencies, actions = handle_set_environment_entry_for_package( app=app, + install_dir=required_install_dir, + tool_shed_repository=tool_shed_repository, + package_name=package_name, + package_version=package_version, + elem=required_td_elem, + required_repository=required_repository ) if actions: actions_dict[ 'actions' ] = actions break - return tool_dependency, actions_dict + return tool_dependencies, actions_dict def run_proprietary_fabric_method( app, elem, proprietary_fabfile_path, install_dir, package_name=None, **kwd ): """ @@ -950,13 +987,22 @@ # Handle setting environment variables using a fabric method. fabric_util.file_append( env_entry, env_file, skip_if_contained=True, make_executable=True ) sa_session.refresh( tool_dependency ) - if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: - tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED - sa_session.add( tool_dependency ) - sa_session.flush() - print 'Environment variable ', env_var_name, 'set in', install_dir + if tool_dependency.status not in [ app.model.ToolDependency.installation_status.ERROR, + app.model.ToolDependency.installation_status.INSTALLED ]: + tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, + tool_dependency=tool_dependency, + status=app.model.ToolDependency.installation_status.INSTALLED, + error_message=None, + remove_from_disk=False ) + log.debug( 'Environment variable %s set in %s for tool dependency %s.' % \ + ( str( env_var_name ), str( install_dir ), str( tool_dependency.name ) ) ) else: - raise NotImplementedError( 'Only set_environment version 1.0 is currently supported (i.e., change your tag to be <set_environment version="1.0">).' ) + error_message = 'Only set_environment version 1.0 is currently supported (i.e., change your tag to be <set_environment version="1.0">).' + tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, + tool_dependency=tool_dependency, + status=app.model.ToolDependency.installation_status.ERROR, + error_message=error_message, + remove_from_disk=False ) return tool_dependency def strip_path( fpath ): diff -r d0ac928b3c1d529e32793a0fc27710e4a1585346 -r f78dcb9eb2897c452c3475d21bd601c0a91b63da lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py @@ -257,6 +257,33 @@ log.debug( error_message ) return env_shell_file_paths +def get_env_shell_file_paths_from_setup_environment_elem( app, all_env_shell_file_paths, elem, action_dict ): + """ + Parse an XML tag set to discover all child repository dependency tags and define the path to an env.sh file associated + with the repository (this requires the repository dependency to be in an installed state). The received action_dict + will be updated with these discovered paths and returned to the caller. This method handles tool dependency definition + tag sets <setup_r_environment>, <setup_ruby_environment> and <setup_perl_environment>. + """ + # An example elem is: + # <action type="setup_perl_environment"> + # <repository name="package_perl_5_18" owner="iuc"> + # <package name="perl" version="5.18.1" /> + # </repository> + # <repository name="package_expat_2_1" owner="iuc" prior_installation_required="True"> + # <package name="expat" version="2.1.0" /> + # </repository> + # <package>http://search.cpan.org/CPAN/authors/id/T/TO/TODDR/XML-Parser-2.41.tar.gz</package> + # <package>http://search.cpan.org/CPAN/authors/id/L/LD/LDS/CGI.pm-3.43.tar.gz</package> + # </action> + for action_elem in elem: + if action_elem.tag == 'repository': + env_shell_file_paths = get_env_shell_file_paths( app, action_elem ) + all_env_shell_file_paths.extend( env_shell_file_paths ) + if all_env_shell_file_paths: + action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths + action_dict[ 'action_shell_file_paths' ] = env_shell_file_paths + return action_dict + def get_env_var_values( install_dir ): env_var_dict = {} env_var_dict[ 'INSTALL_DIR' ] = install_dir @@ -415,16 +442,6 @@ continue return actions_elem_tuples - -def parse_setup_environment_repositories( app, all_env_shell_file_paths, action_elem, action_dict ): - env_shell_file_paths = get_env_shell_file_paths( app, action_elem.find('repository') ) - - all_env_shell_file_paths.extend( env_shell_file_paths ) - if all_env_shell_file_paths: - action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths - action_dict[ 'action_shell_file_paths' ] = env_shell_file_paths - - def url_download( install_dir, downloaded_file_name, download_url, extract=True ): file_path = os.path.join( install_dir, downloaded_file_name ) src = None diff -r d0ac928b3c1d529e32793a0fc27710e4a1585346 -r f78dcb9eb2897c452c3475d21bd601c0a91b63da lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -85,12 +85,11 @@ includes_tool_dependencies = True # Inspect the tool_dependencies dictionary to separate the installed and missing tool dependencies. We don't add to installed_td # and missing_td here because at this point they are empty. - installed_td, missing_td = get_installed_and_missing_tool_dependencies( trans, tool_shed_url, tool_dependencies ) + installed_td, missing_td = get_installed_and_missing_tool_dependencies_for_installing_repository( trans, tool_shed_url, tool_dependencies ) # In cases where a repository dependency is required only for compiling a dependent repository's tool dependency, the value of # repository_dependencies will be an empty dictionary here. if repository_dependencies: # We have a repository with one or more defined repository dependencies. - missing_td = {} if not repository: repository = suc.get_repository_for_dependency_relationship( trans.app, tool_shed_url, name, repository_owner, changeset_revision ) if repository and repository.metadata: @@ -118,9 +117,8 @@ required_tool_dependencies[ td_key ] = td_dict if required_tool_dependencies: # Discover and categorize all tool dependencies defined for this repository's repository dependencies. - required_installed_td, required_missing_td = get_installed_and_missing_tool_dependencies( trans, - tool_shed_url, - required_tool_dependencies ) + required_installed_td, required_missing_td = \ + get_installed_and_missing_tool_dependencies_for_installing_repository( trans, tool_shed_url, required_tool_dependencies ) if required_installed_td: if not includes_tool_dependencies: includes_tool_dependencies = True @@ -291,7 +289,7 @@ missing_repository_dependencies[ 'description' ] = description return installed_repository_dependencies, missing_repository_dependencies -def get_installed_and_missing_tool_dependencies( trans, tool_shed_url, tool_dependencies_dict ): +def get_installed_and_missing_tool_dependencies_for_installing_repository( trans, tool_shed_url, tool_dependencies_dict ): """Return the lists of installed tool dependencies and missing tool dependencies for a set of repositories being installed into Galaxy.""" installed_tool_dependencies = {} missing_tool_dependencies = {} @@ -460,15 +458,18 @@ try: dependencies_ignored = app.toolbox.dependency_manager and not app.toolbox.dependency_manager.uses_tool_shed_dependencies() if dependencies_ignored: - log.info("Skipping package %s, tool shed dependency resolver not enabled." % package_name) - # Tool dependency resolves have been configured and they do not - # include the tool shed. Do not install package. - status = app.model.ToolDependency.installation_status.ERROR + log.debug( "Skipping package %s because tool shed dependency resolver not enabled." % str( package_name ) ) + # Tool dependency resolves have been configured and they do not include the tool shed. Do not install package. if app.toolbox.dependency_manager.find_dep( package_name, package_version, type='package') != INDETERMINATE_DEPENDENCY: ## TODO: Do something here such as marking it installed or ## configured externally. pass - tool_dependency.status = status + tool_dependency = \ + tool_dependency_util.set_tool_dependency_attributes( app, + tool_dependency=tool_dependency, + status=app.model.ToolDependency.installation_status.ERROR, + error_message=None, + remove_from_disk=False ) else: tool_dependency = install_package( app, elem, tool_shed_repository, tool_dependencies=tool_dependencies ) except Exception, e: diff -r d0ac928b3c1d529e32793a0fc27710e4a1585346 -r f78dcb9eb2897c452c3475d21bd601c0a91b63da lib/tool_shed/util/tool_dependency_util.py --- a/lib/tool_shed/util/tool_dependency_util.py +++ b/lib/tool_shed/util/tool_dependency_util.py @@ -221,7 +221,8 @@ return url_template return None -def get_installed_and_missing_tool_dependencies( trans, repository, all_tool_dependencies ): +def get_installed_and_missing_tool_dependencies_for_installed_repository( trans, repository, all_tool_dependencies ): + """Return the lists of installed tool dependencies and missing tool dependencies for a Tool Shed repository that has been installed into Galaxy.""" if all_tool_dependencies: tool_dependencies = {} missing_tool_dependencies = {} @@ -429,7 +430,7 @@ required_repository, installed_changeset_revision = suc.repository_was_previously_installed( trans, tool_shed_url, name, repo_info_tuple ) if required_repository: required_repository_installed_tool_dependencies, required_repository_missing_tool_dependencies = \ - get_installed_and_missing_tool_dependencies( trans, required_repository, tool_dependencies ) + get_installed_and_missing_tool_dependencies_for_installed_repository( trans, required_repository, tool_dependencies ) if required_repository_installed_tool_dependencies: # Add the install_dir attribute to the tool_dependencies. required_repository_installed_tool_dependencies = \ @@ -479,14 +480,16 @@ error_message = '' return removed, error_message -def set_tool_dependency_attributes( trans, tool_dependency, status, error_message, remove_from_disk=False ): +def set_tool_dependency_attributes( app, tool_dependency, status, error_message=None, remove_from_disk=False ): + sa_session = app.model.context.current if remove_from_disk: - installation_directory = tool_dependency.installation_directory( trans.app ) + installation_directory = tool_dependency.installation_directory( app ) removed, err_msg = remove_tool_dependency_installation_directory( installation_directory ) tool_dependency.error_message = error_message tool_dependency.status = status - trans.sa_session.add( tool_dependency ) - trans.sa_session.flush() + sa_session.add( tool_dependency ) + sa_session.flush() + return tool_dependency def tool_dependency_is_orphan( type, name, version, tools ): """ diff -r d0ac928b3c1d529e32793a0fc27710e4a1585346 -r f78dcb9eb2897c452c3475d21bd601c0a91b63da test/unit/tool_shed/test_td_common_util.py --- a/test/unit/tool_shed/test_td_common_util.py +++ b/test/unit/tool_shed/test_td_common_util.py @@ -33,8 +33,7 @@ assert path == join( TEST_INSTALL_DIR, "env.sh" ) assert line == ". /usr/share/R/libs" - -def test_parse_setup_environment_repositories( ): +def test_get_env_shell_file_paths_from_setup_environment_elem( ): xml = """<action name="setup_r_environment"><repository name="package_r_3_0_1" owner="bgruening" toolshed="toolshed.g2.bx.psu.edu" changeset_revision="1234567"><package name="R" version="3.0.1" /> @@ -49,26 +48,25 @@ r_env_sh = '/path/to/go/env.sh' - def mock_get_env_shell_file_paths( app, elem): + def mock_get_env_shell_file_paths( app, elem ): assert app == mock_app assert elem.get( 'name' ) == "package_r_3_0_1" return [ r_env_sh ] - with __mock_common_util_method("get_env_shell_file_paths", mock_get_env_shell_file_paths): - td_common_util.parse_setup_environment_repositories( mock_app, all_env_paths, action_elem, action_dict ) + with __mock_common_util_method( "get_env_shell_file_paths", mock_get_env_shell_file_paths ): + td_common_util.get_env_shell_file_paths_from_setup_environment_elem( mock_app, all_env_paths, action_elem, action_dict ) ## Verify old env files weren't deleted. assert required_for_install_env_sh in all_env_paths ## Verify new ones added. assert r_env_sh in all_env_paths ## env_shell_file_paths includes everything - assert all( [env in action_dict[ 'env_shell_file_paths' ] for env in all_env_paths] ) + assert all( [ env in action_dict[ 'env_shell_file_paths' ] for env in all_env_paths ] ) ## action_shell_file_paths includes only env files defined in ## inside the setup_ action element. assert required_for_install_env_sh not in action_dict[ 'action_shell_file_paths' ] assert r_env_sh in action_dict[ 'action_shell_file_paths' ] - ## Poor man's mocking. Need to get a real mocking library as real Galaxy development ## dependnecy. @contextmanager Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.