1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0d2c45f321ed/ Changeset: 0d2c45f321ed User: greg Date: 2014-05-15 15:59:55 Summary: Refactoring cleanup in the Tool Shed's package installation framework. Affected #: 4 files diff -r 75be6c49a65d8d52920696216ffb5af74a3fdd9c -r 0d2c45f321ed234115a15d46c255818192ba41a2 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -26,8 +26,6 @@ from tool_shed.util import xml_util from tool_shed.galaxy_install import repository_util import tool_shed.galaxy_install.grids.admin_toolshed_grids as admin_toolshed_grids -from tool_shed.galaxy_install.install_manager import InstallManager -from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import TagManager import pkg_resources eggs.require( 'mercurial' ) @@ -458,63 +456,16 @@ message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) err_msg = '' - attr_tups_of_dependencies_for_install = [ ( td.name, td.version, td.type ) for td in tool_dependencies ] tool_shed_repository = tool_dependencies[ 0 ].tool_shed_repository # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = suc.get_config_from_disk( suc.TOOL_DEPENDENCY_DEFINITION_FILENAME, tool_shed_repository.repo_path( trans.app ) ) - # Parse the tool_dependencies.xml config. - tree, error_message = xml_util.parse_xml( tool_dependencies_config ) - installed_tool_dependencies = [] - install_manager = InstallManager() - tag_manager = TagManager() - root = tree.getroot() - for elem in root: - package_name = elem.get( 'name', None ) - package_version = elem.get( 'version', None ) - if package_name and package_version: - # elem is a package tag set. - attr_tup = ( package_name, package_version, 'package' ) - try: - index = attr_tups_of_dependencies_for_install.index( attr_tup ) - except Exception, e: - index = None - if index is not None: - tool_dependency = tool_dependencies[ index ] - tool_dependency, proceed_with_install, action_elem_tuples = \ - tag_manager.process_tag_set( trans.app, - tool_shed_repository, - tool_dependency, - elem, - package_name, - package_version, - from_tool_migration_manager=False, - tool_dependency_db_records=tool_dependencies ) - if proceed_with_install: - try: - tool_dependency = install_manager.install_package( trans.app, - elem, - tool_shed_repository, - tool_dependencies=tool_dependencies, - from_tool_migration_manager=False ) - except Exception, e: - error_message = "Error installing tool dependency package %s version %s: %s" % \ - ( str( package_name ), str( package_version ), str( e ) ) - log.exception( error_message ) - if tool_dependency: - # Since there was an installation error, update the tool dependency status to Error. The - # remove_installation_path option must be left False here. - tool_dependency = \ - tool_dependency_util.handle_tool_dependency_installation_error( trans.app, - tool_dependency, - error_message, - remove_installation_path=False ) - if tool_dependency and tool_dependency.status in [ trans.app.install_model.ToolDependency.installation_status.INSTALLED, - trans.app.install_model.ToolDependency.installation_status.ERROR ]: - installed_tool_dependencies.append( tool_dependency ) - if trans.app.config.manage_dependency_relationships: - # Add the tool_dependency to the in-memory dictionaries in the installed_repository_manager. - trans.app.installed_repository_manager.handle_tool_dependency_install( tool_shed_repository, tool_dependency ) + installed_tool_dependencies = \ + common_install_util.install_specified_packages( app=trans.app, + tool_shed_repository=tool_shed_repository, + tool_dependencies_config=tool_dependencies_config, + tool_dependencies=tool_dependencies, + from_tool_migration_manager=False ) for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status == trans.app.install_model.ToolDependency.installation_status.ERROR: text = util.unicodify( installed_tool_dependency.error_message ) diff -r 75be6c49a65d8d52920696216ffb5af74a3fdd9c -r 0d2c45f321ed234115a15d46c255818192ba41a2 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -22,9 +22,6 @@ from tool_shed.util import tool_util from tool_shed.util import xml_util -from tool_shed.galaxy_install.install_manager import InstallManager -from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import TagManager - from galaxy import eggs eggs.require( 'mercurial' ) @@ -624,58 +621,12 @@ trans.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', install_dir ) - # Parse the tool_dependencies.xml config. - tree, error_message = xml_util.parse_xml( tool_dependencies_config ) - install_manager = InstallManager() - tag_manager = TagManager() - root = tree.getroot() - for elem in root: - package_name = elem.get( 'name', None ) - package_version = elem.get( 'version', None ) - if package_name and package_version: - repository_tool_dependencies = util.listify( tool_shed_repository.tool_dependencies ) - # elem is a package tag set. - attr_tups_of_dependencies_for_install = [ ( td.name, td.version, td.type ) for td in repository_tool_dependencies ] - attr_tup = ( package_name, package_version, 'package' ) - try: - index = attr_tups_of_dependencies_for_install.index( attr_tup ) - except Exception, e: - index = None - if index is not None: - tool_dependency = repository_tool_dependencies[ index ] - tool_dependency, proceed_with_install, action_elem_tuples = \ - tag_manager.process_tag_set( trans.app, - tool_shed_repository, - tool_dependency, - elem, - package_name, - package_version, - from_tool_migration_manager=False, - tool_dependency_db_records=repository_tool_dependencies ) - if proceed_with_install: - try: - tool_dependency = install_manager.install_package( trans.app, - elem, - tool_shed_repository, - tool_dependencies=repository_tool_dependencies, - from_tool_migration_manager=False ) - except Exception, e: - error_message = "Error installing tool dependency package %s version %s: %s" % \ - ( str( package_name ), str( package_version ), str( e ) ) - log.exception( error_message ) - if tool_dependency: - # Since there was an installation error, update the tool dependency status to Error. The - # remove_installation_path option must be left False here. - tool_dependency = \ - tool_dependency_util.handle_tool_dependency_installation_error( trans.app, - tool_dependency, - error_message, - remove_installation_path=False ) - if tool_dependency and tool_dependency.status in [ trans.app.install_model.ToolDependency.installation_status.INSTALLED, - trans.app.install_model.ToolDependency.installation_status.ERROR ]: - if trans.app.config.manage_dependency_relationships: - # Add the tool_dependency to the in-memory dictionaries in the installed_repository_manager. - trans.app.installed_repository_manager.handle_tool_dependency_install( tool_shed_repository, tool_dependency ) + installed_tool_dependencies = \ + common_install_util.install_specified_packages( app=trans.app, + tool_shed_repository=tool_shed_repository, + tool_dependencies_config=tool_dependencies_config, + tool_dependencies=tool_shed_repository.tool_dependencies, + from_tool_migration_manager=False ) suc.remove_dir( work_dir ) suc.update_tool_shed_repository_status( trans.app, tool_shed_repository, @@ -916,72 +867,24 @@ for tool_dependency in repository.missing_tool_dependencies: if tool_dependency.status in [ trans.install_model.ToolDependency.installation_status.ERROR, trans.install_model.ToolDependency.installation_status.INSTALLING ]: - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( trans.app, - tool_dependency=tool_dependency, - status=trans.install_model.ToolDependency.installation_status.UNINSTALLED, - error_message=None, - remove_from_disk=True ) + tool_dependency = \ + tool_dependency_util.set_tool_dependency_attributes( trans.app, + tool_dependency=tool_dependency, + status=trans.install_model.ToolDependency.installation_status.UNINSTALLED, + error_message=None, + remove_from_disk=True ) # Install tool dependencies. suc.update_tool_shed_repository_status( trans.app, repository, trans.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', repository.repo_path( trans.app ) ) - installed_tool_dependencies = [] - # Parse the tool_dependencies.xml config. - tree, error_message = xml_util.parse_xml( tool_dependencies_config ) - if tree is None: - return installed_tool_dependencies - install_manager = InstallManager() - tag_manager = TagManager() - root = tree.getroot() - for elem in root: - package_name = elem.get( 'name', None ) - package_version = elem.get( 'version', None ) - if package_name and package_version: - # elem is a package tag set. - attr_tups_of_dependencies_for_install = [ ( td.name, td.version, td.type ) for td in tool_dependencies ] - attr_tup = ( package_name, package_version, 'package' ) - try: - index = attr_tups_of_dependencies_for_install.index( attr_tup ) - except Exception, e: - index = None - if index is not None: - tool_dependency = tool_dependency_db_records[ index ] - tool_dependency, proceed_with_install, action_elem_tuples = \ - tag_manager.process_tag_set( trans.app, - tool_shed_repository, - tool_dependency, - elem, - package_name, - package_version, - from_tool_migration_manager=False, - tool_dependency_db_records=tool_dependencies ) - if proceed_with_install: - try: - tool_dependency = install_manager.install_package( trans.app, - elem, - tool_shed_repository, - tool_dependencies=tool_dependencies, - from_tool_migration_manager=False ) - except Exception, e: - error_message = "Error installing tool dependency package %s version %s: %s" % \ - ( str( package_name ), str( package_version ), str( e ) ) - log.exception( error_message ) - if tool_dependency: - # Since there was an installation error, update the tool dependency status to Error. The - # remove_installation_path option must be left False here. - tool_dependency = \ - tool_dependency_util.handle_tool_dependency_installation_error( trans.app, - tool_dependency, - error_message, - remove_installation_path=False ) - if tool_dependency and tool_dependency.status in [ app.install_model.ToolDependency.installation_status.INSTALLED, - app.install_model.ToolDependency.installation_status.ERROR ]: - installed_tool_dependencies.append( tool_dependency ) - if app.config.manage_dependency_relationships: - # Add the tool_dependency to the in-memory dictionaries in the installed_repository_manager. - app.installed_repository_manager.handle_tool_dependency_install( tool_shed_repository, tool_dependency ) + installed_tool_dependencies = \ + common_install_util.install_specified_packages( app=trans.app, + tool_shed_repository=repository, + tool_dependencies_config=tool_dependencies_config, + tool_dependencies=repository.tool_dependencies, + from_tool_migration_manager=False ) for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status in [ trans.install_model.ToolDependency.installation_status.ERROR ]: repair_dict = add_repair_dict_entry( repository.name, installed_tool_dependency.error_message ) diff -r 75be6c49a65d8d52920696216ffb5af74a3fdd9c -r 0d2c45f321ed234115a15d46c255818192ba41a2 lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -19,8 +19,6 @@ from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util from tool_shed.util import xml_util -from tool_shed.galaxy_install.install_manager import InstallManager -from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import TagManager from galaxy.util.odict import odict log = logging.getLogger( __name__ ) @@ -447,64 +445,12 @@ self.app.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from disk. tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', repo_install_dir ) - installed_tool_dependencies = [] - # Parse the tool_dependencies.xml config. - tree, error_message = xml_util.parse_xml( tool_dependencies_config ) - install_manager = InstallManager() - tag_manager = TagManager() - root = tree.getroot() - for elem in root: - package_name = elem.get( 'name', None ) - package_version = elem.get( 'version', None ) - if package_name and package_version: - # elem is a package tag set. - tool_dependency = None - for tool_dependency in tool_dependencies: - if package_name == str( tool_dependency.name ) and package_version == str( tool_dependency.version ): - break - if tool_dependency is not None: - attr_tups_of_dependencies_for_install = [ ( td.name, td.version, td.type ) for td in tool_dependencies ] - attr_tup = ( package_name, package_version, 'package' ) - try: - index = attr_tups_of_dependencies_for_install.index( attr_tup ) - except Exception, e: - index = None - if index is not None: - tool_dependency = tool_dependencies[ index ] - tool_dependency, proceed_with_install, action_elem_tuples = \ - tag_manager.process_tag_set( self.app, - tool_shed_repository, - tool_dependency, - elem, - package_name, - package_version, - from_tool_migration_manager=False, - tool_dependency_db_records=tool_dependencies ) - if proceed_with_install: - try: - tool_dependency = install_manager.install_package( self.app, - elem, - tool_shed_repository, - tool_dependencies=tool_dependencies, - from_tool_migration_manager=True ) - except Exception, e: - error_message = "Error installing tool dependency package %s version %s: %s" % \ - ( str( package_name ), str( package_version ), str( e ) ) - log.exception( error_message ) - if tool_dependency: - # Since there was an installation error, update the tool dependency status to Error. The - # remove_installation_path option must be left False here. - tool_dependency = \ - tool_dependency_util.handle_tool_dependency_installation_error( self.app, - tool_dependency, - error_message, - remove_installation_path=False ) - if tool_dependency and tool_dependency.status in [ self.app.install_model.ToolDependency.installation_status.INSTALLED, - self.app.install_model.ToolDependency.installation_status.ERROR ]: - installed_tool_dependencies.append( tool_dependency ) - if self.app.config.manage_dependency_relationships: - # Add the tool_dependency to the in-memory dictionaries in the installed_repository_manager. - self.app.installed_repository_manager.handle_tool_dependency_install( tool_shed_repository, tool_dependency ) + installed_tool_dependencies = \ + common_install_util.install_specified_packages( app=self.app, + tool_shed_repository=tool_shed_repository, + tool_dependencies_config=tool_dependencies_config, + tool_dependencies=tool_dependencies, + from_tool_migration_manager=True ) for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status == self.app.install_model.ToolDependency.installation_status.ERROR: print '\nThe ToolMigrationManager returned the following error while installing tool dependency ', installed_tool_dependency.name, ':' diff -r 75be6c49a65d8d52920696216ffb5af74a3fdd9c -r 0d2c45f321ed234115a15d46c255818192ba41a2 lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -3,7 +3,6 @@ import os import urllib import urllib2 -from galaxy import eggs from galaxy import util from galaxy import web from galaxy.util import json @@ -18,6 +17,9 @@ from tool_shed.util import tool_util from tool_shed.util import xml_util +from tool_shed.galaxy_install.install_manager import InstallManager +from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import TagManager + log = logging.getLogger( __name__ ) def activate_repository( trans, repository ): @@ -481,6 +483,73 @@ all_required_repo_info_dict[ 'all_repo_info_dicts' ] = all_repo_info_dicts return all_required_repo_info_dict +def install_specified_packages( app, tool_shed_repository, tool_dependencies_config, tool_dependencies, from_tool_migration_manager=False ): + """ + Follow the recipe in the received tool_dependencies_config to install specified packages for + repository tools. The received list of tool_dependencies are the database records for those + dependencies defined in the tool_dependencies_config that are to be installed. This list may + be a subset of the set of dependencies defined in the tool_dependencies_config. This allows + for filtering out dependencies that have not been checked for installation on the 'Manage tool + dependencies' page for an installed Tool Shed repository. + """ + attr_tups_of_dependencies_for_install = [ ( td.name, td.version, td.type ) for td in tool_dependencies ] + installed_packages = [] + install_manager = InstallManager() + tag_manager = TagManager() + # Parse the tool_dependencies.xml config. + tree, error_message = xml_util.parse_xml( tool_dependencies_config ) + if tree is None: + log.debug( "The received tool_dependencies.xml file is likely invalid: %s" % str( error_message ) ) + return installed_packages + root = tree.getroot() + for elem in root: + package_name = elem.get( 'name', None ) + package_version = elem.get( 'version', None ) + if package_name and package_version: + # elem is a package tag set. + attr_tup = ( package_name, package_version, 'package' ) + try: + index = attr_tups_of_dependencies_for_install.index( attr_tup ) + except Exception, e: + index = None + if index is not None: + tool_dependency = tool_dependencies[ index ] + tool_dependency, proceed_with_install, action_elem_tuples = \ + tag_manager.process_tag_set( app, + tool_shed_repository, + tool_dependency, + elem, + package_name, + package_version, + from_tool_migration_manager=from_tool_migration_manager, + tool_dependency_db_records=tool_dependencies ) + if proceed_with_install: + try: + tool_dependency = install_manager.install_package( app, + elem, + tool_shed_repository, + tool_dependencies=tool_dependencies, + from_tool_migration_manager=from_tool_migration_manager ) + except Exception, e: + error_message = "Error installing tool dependency package %s version %s: %s" % \ + ( str( package_name ), str( package_version ), str( e ) ) + log.exception( error_message ) + if tool_dependency: + # Since there was an installation error, update the tool dependency status to Error. The + # remove_installation_path option must be left False here. + tool_dependency = \ + tool_dependency_util.handle_tool_dependency_installation_error( app, + tool_dependency, + error_message, + remove_installation_path=False ) + if tool_dependency and tool_dependency.status in [ app.install_model.ToolDependency.installation_status.INSTALLED, + app.install_model.ToolDependency.installation_status.ERROR ]: + installed_packages.append( tool_dependency ) + if app.config.manage_dependency_relationships: + # Add the tool_dependency to the in-memory dictionaries in the installed_repository_manager. + app.installed_repository_manager.handle_tool_dependency_install( tool_shed_repository, tool_dependency ) + return installed_packages + def repository_dependency_needed_only_for_compiling_tool_dependency( repository, repository_dependency ): for rd_tup in repository.tuples_of_repository_dependencies_needed_for_compiling_td: tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = rd_tup Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.