1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7628e1d9d8e2/ Changeset: 7628e1d9d8e2 User: greg Date: 2014-07-15 19:56:12 Summary: Add a new RelationBuilder class to handle building repository relationships, eliminating the use of the repository_dependency_util.py module, and rename the RepositoryDependencyManager class to be RepositoryDependencyInstallManager. Affected #: 13 files diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -22,7 +22,6 @@ from tool_shed.util import hg_util from tool_shed.util import metadata_util from tool_shed.util import readme_util -from tool_shed.util import repository_dependency_util from tool_shed.util import repository_maintenance_util from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util @@ -31,7 +30,7 @@ from tool_shed.galaxy_install import install_manager from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager import tool_shed.galaxy_install.grids.admin_toolshed_grids as admin_toolshed_grids -from tool_shed.galaxy_install.repository_dependencies.repository_dependency_manager import RepositoryDependencyManager +from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager log = logging.getLogger( __name__ ) @@ -1301,7 +1300,7 @@ Reinstall a tool shed repository that has been previously uninstalled, making sure to handle all repository and tool dependencies of the repository. """ - rdm = RepositoryDependencyManager( trans.app ) + rdim = repository_dependency_manager.RepositoryDependencyInstallManager( trans.app ) message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) repository_id = kwd[ 'id' ] @@ -1365,35 +1364,33 @@ else: # Entering this else block occurs only if the tool_shed_repository does not include any valid tools. if install_repository_dependencies: - repository_dependencies = rdm.get_repository_dependencies_for_installed_tool_shed_repository( trans.app, - tool_shed_repository ) + repository_dependencies = \ + rdim.get_repository_dependencies_for_installed_tool_shed_repository( trans.app, + tool_shed_repository ) else: repository_dependencies = None if metadata: tool_dependencies = metadata.get( 'tool_dependencies', None ) else: tool_dependencies = None - repo_info_dict = \ - repository_maintenance_util.create_repo_info_dict( app=trans.app, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.changeset_revision, - ctx_rev=ctx_rev, - repository_owner=tool_shed_repository.owner, - repository_name=tool_shed_repository.name, - repository=None, - repository_metadata=None, - tool_dependencies=tool_dependencies, - repository_dependencies=repository_dependencies ) + repo_info_dict = repository_maintenance_util.create_repo_info_dict( trans.app, + repository_clone_url=repository_clone_url, + changeset_revision=tool_shed_repository.changeset_revision, + ctx_rev=ctx_rev, + repository_owner=tool_shed_repository.owner, + repository_name=tool_shed_repository.name, + tool_dependencies=tool_dependencies, + repository_dependencies=repository_dependencies ) if repo_info_dict not in repo_info_dicts: repo_info_dicts.append( repo_info_dict ) # Make sure all tool_shed_repository records exist. created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts = \ - rdm.create_repository_dependency_objects( tool_path=tool_path, - tool_shed_url=tool_shed_url, - repo_info_dicts=repo_info_dicts, - install_repository_dependencies=install_repository_dependencies, - no_changes_checked=no_changes_checked, - tool_panel_section_id=tool_panel_section_id ) + rdim.create_repository_dependency_objects( tool_path=tool_path, + tool_shed_url=tool_shed_url, + repo_info_dicts=repo_info_dicts, + install_repository_dependencies=install_repository_dependencies, + no_changes_checked=no_changes_checked, + tool_panel_section_id=tool_panel_section_id ) # Default the selected tool panel location for loading tools included in each newly installed required # tool shed repository to the location selected for the repository selected for re-installation. for index, tps_key in enumerate( tool_panel_section_keys ): @@ -1518,10 +1515,11 @@ @web.require_admin def reselect_tool_panel_section( self, trans, **kwd ): """ - Select or change the tool panel section to contain the tools included in the tool shed repository being reinstalled. If there are updates - available for the repository in the tool shed, the tool_dependencies and repository_dependencies associated with the updated changeset revision - will have been retrieved from the tool shed and passed in the received kwd. In this case, the stored tool shed repository metadata from the - Galaxy database will not be used since it is outdated. + Select or change the tool panel section to contain the tools included in the tool shed repository + being reinstalled. If there are updates available for the repository in the tool shed, the + tool_dependencies and repository_dependencies associated with the updated changeset revision will + have been retrieved from the tool shed and passed in the received kwd. In this case, the stored + tool shed repository metadata from the Galaxy database will not be used since it is outdated. """ message = '' status = 'done' @@ -1534,8 +1532,8 @@ tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( tool_shed_repository.tool_shed ) ) tool_path, relative_install_dir = tool_shed_repository.get_tool_relative_path( trans.app ) if latest_changeset_revision and latest_ctx_rev: - # There are updates available in the tool shed for the repository, so use the receieved dependency information which was retrieved from - # the tool shed. + # There are updates available in the tool shed for the repository, so use the received + # dependency information which was retrieved from the tool shed. encoded_updated_repo_info_dict = kwd.get( 'updated_repo_info_dict', None ) updated_repo_info_dict = encoding_util.tool_shed_decode( encoded_updated_repo_info_dict ) readme_files_dict = updated_repo_info_dict.get( 'readme_files_dict', None ) @@ -1584,20 +1582,18 @@ raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) readme_files_dict = json.from_json_string( raw_text ) tool_dependencies = metadata.get( 'tool_dependencies', None ) - rdm = RepositoryDependencyManager( trans.app ) - repository_dependencies = rdm.get_repository_dependencies_for_installed_tool_shed_repository( trans.app, - tool_shed_repository ) - repo_info_dict = \ - repository_maintenance_util.create_repo_info_dict( app=trans.app, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.installed_changeset_revision, - ctx_rev=tool_shed_repository.ctx_rev, - repository_owner=tool_shed_repository.owner, - repository_name=tool_shed_repository.name, - repository=None, - repository_metadata=None, - tool_dependencies=tool_dependencies, - repository_dependencies=repository_dependencies ) + rdim = repository_dependency_manager.RepositoryDependencyInstallManager( trans.app ) + repository_dependencies = \ + rdim.get_repository_dependencies_for_installed_tool_shed_repository( trans.app, + tool_shed_repository ) + repo_info_dict = repository_maintenance_util.create_repo_info_dict( trans.app, + repository_clone_url=repository_clone_url, + changeset_revision=tool_shed_repository.installed_changeset_revision, + ctx_rev=tool_shed_repository.ctx_rev, + repository_owner=tool_shed_repository.owner, + repository_name=tool_shed_repository.name, + tool_dependencies=tool_dependencies, + repository_dependencies=repository_dependencies ) irm = trans.app.installed_repository_manager dependencies_for_repository_dict = irm.get_dependencies_for_repository( tool_shed_url, repo_info_dict, diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -15,6 +15,7 @@ from galaxy.util import json from galaxy.model.orm import and_ from tool_shed.capsule import capsule_manager +from tool_shed.dependencies.repository import relation_builder from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import container_util @@ -22,7 +23,6 @@ from tool_shed.util import hg_util from tool_shed.util import metadata_util from tool_shed.util import readme_util -from tool_shed.util import repository_dependency_util from tool_shed.util import repository_maintenance_util from tool_shed.util import review_util from tool_shed.util import search_util @@ -1251,15 +1251,10 @@ return opened_archive repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) metadata = repository_metadata.metadata + toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) + rb = relation_builder.RelationBuilder( trans.app, repository, repository_metadata, toolshed_base_url ) # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. - repository_dependencies = \ - repository_dependency_util.get_repository_dependencies_for_changeset_revision( app=trans.app, - repository=repository, - repository_metadata=repository_metadata, - toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ), - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None ) + repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() if repository_dependencies: # Only display repository dependencies if they exist. exclude = [ 'datatypes', 'invalid_repository_dependencies', 'invalid_tool_dependencies', 'invalid_tools', @@ -1777,15 +1772,9 @@ if repository_metadata: metadata = repository_metadata.metadata if metadata: - repository_dependencies = \ - repository_dependency_util.get_repository_dependencies_for_changeset_revision( app=trans.app, - repository=repository, - repository_metadata=repository_metadata, - toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ), - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None, - circular_repository_dependencies=None ) + toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) + rb = relation_builder.RelationBuilder( trans.app, repository, repository_metadata, toolshed_base_url ) + repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() if repository_dependencies: return encoding_util.tool_shed_encode( repository_dependencies ) return '' @@ -2413,14 +2402,9 @@ skip_tool_tests_checked = True metadata = repository_metadata.metadata # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. - repository_dependencies = \ - repository_dependency_util.get_repository_dependencies_for_changeset_revision( app=trans.app, - repository=repository, - repository_metadata=repository_metadata, - toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ), - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None ) + toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) + rb = relation_builder.RelationBuilder( trans.app, repository, repository_metadata, toolshed_base_url ) + repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() if str( repository.type ) != rt_util.REPOSITORY_SUITE_DEFINITION: # Handle messaging for resetting repository type to the optimal value. change_repository_type_message = rt_util.generate_message_for_repository_type_change( trans.app, @@ -2461,9 +2445,9 @@ repository_metadata ) heads = hg_util.get_repository_heads( repo ) deprecated_repository_dependency_tups = \ - repository_dependency_util.get_repository_dependency_tups_from_repository_metadata( trans.app, - repository_metadata, - deprecated_only=True ) + metadata_util.get_repository_dependency_tups_from_repository_metadata( trans.app, + repository_metadata, + deprecated_only=True ) return trans.fill_template( '/webapps/tool_shed/repository/manage_repository.mako', repo_name=repo_name, description=description, @@ -2614,14 +2598,8 @@ metadata = repository_metadata.metadata # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) - repository_dependencies = \ - repository_dependency_util.get_repository_dependencies_for_changeset_revision( app=trans.app, - repository=repository, - repository_metadata=repository_metadata, - toolshed_base_url=toolshed_base_url, - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None ) + rb = relation_builder.RelationBuilder( trans.app, repository, repository_metadata, toolshed_base_url ) + repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() if metadata: if 'repository_dependencies' in metadata and not repository_dependencies: # See if we have an invalid repository dependency definition or if the repository dependency is required @@ -2636,8 +2614,8 @@ invalid = True break if invalid: - message = repository_dependency_util.generate_message_for_invalid_repository_dependencies( metadata, - error_from_tuple=False ) + message = metadata_util.generate_message_for_invalid_repository_dependencies( metadata, + error_from_tuple=False ) status = 'error' else: repository_metadata_id = None @@ -3329,14 +3307,9 @@ if repository_metadata: metadata = repository_metadata.metadata # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. - repository_dependencies = \ - repository_dependency_util.get_repository_dependencies_for_changeset_revision( app=trans.app, - repository=repository, - repository_metadata=repository_metadata, - toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ), - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None ) + toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) + rb = relation_builder.RelationBuilder( trans.app, repository, repository_metadata, toolshed_base_url ) + repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() if str( repository.type ) != rt_util.TOOL_DEPENDENCY_DEFINITION: # Handle messaging for orphan tool dependency definitions. orphan_message = tool_dependency_util.generate_message_for_orphan_tool_dependencies( trans, repository, metadata ) diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -14,7 +14,6 @@ from tool_shed.util import commit_util from tool_shed.util import hg_util from tool_shed.util import metadata_util -from tool_shed.util import repository_dependency_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util @@ -303,8 +302,8 @@ status = 'error' # Handle messaging for invalid repository dependencies. invalid_repository_dependencies_message = \ - repository_dependency_util.generate_message_for_invalid_repository_dependencies( metadata_dict, - error_from_tuple=True ) + metadata_util.generate_message_for_invalid_repository_dependencies( metadata_dict, + error_from_tuple=True ) if invalid_repository_dependencies_message: message += invalid_repository_dependencies_message status = 'error' diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/tool_shed/capsule/capsule_manager.py --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -12,18 +12,19 @@ from galaxy.util import asbool from galaxy.util import CHUNK_SIZE from galaxy.util.odict import odict -from tool_shed.dependencies import dependency_manager +from tool_shed.dependencies.repository.relation_builder import RelationBuilder +from tool_shed.dependencies.dependency_manager import RepositoryDependencyAttributeHandler +from tool_shed.dependencies.dependency_manager import ToolDependencyAttributeHandler from tool_shed.util import basic_util from tool_shed.util import commit_util from tool_shed.util import common_util from tool_shed.util import encoding_util from tool_shed.util import hg_util from tool_shed.util import metadata_util -from tool_shed.util import repository_dependency_util from tool_shed.util import repository_maintenance_util from tool_shed.util import shed_util_common as suc from tool_shed.util import xml_util -from tool_shed.galaxy_install.repository_dependencies.repository_dependency_manager import RepositoryDependencyManager +from tool_shed.galaxy_install.repository_dependencies.repository_dependency_manager import RepositoryDependencyInstallManager log = logging.getLogger( __name__ ) @@ -129,8 +130,8 @@ return sub_elements def generate_repository_archive( self, repository, changeset_revision, work_dir ): - rdah = dependency_manager.RepositoryDependencyAttributeHandler( self.app, unpopulate=True ) - tdah = dependency_manager.ToolDependencyAttributeHandler( self.app, unpopulate=True ) + rdah = RepositoryDependencyAttributeHandler( self.app, unpopulate=True ) + tdah = ToolDependencyAttributeHandler( self.app, unpopulate=True ) file_type_str = basic_util.get_file_type_str( changeset_revision, self.file_type ) file_name = '%s-%s' % ( repository.name, file_type_str ) return_code, error_message = hg_util.archive_repository_revision( self.app, @@ -235,21 +236,15 @@ Return a list of dictionaries defining repositories that are required by the repository associated with self.repository_id. """ - rdm = RepositoryDependencyManager( self.app ) + rdim = RepositoryDependencyInstallManager( self.app ) repository = suc.get_repository_in_tool_shed( self.app, self.repository_id ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, self.repository_id, self.changeset_revision ) # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) - repository_dependencies = \ - repository_dependency_util.get_repository_dependencies_for_changeset_revision( app=self.app, - repository=self.repository, - repository_metadata=repository_metadata, - toolshed_base_url=toolshed_base_url, - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None ) + rb = RelationBuilder( self.app, repository, repository_metadata, toolshed_base_url ) + repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() repo = hg_util.get_repo_for_repository( self.app, repository=self.repository, repo_path=None, @@ -265,7 +260,7 @@ str( self.repository.user.username ), repository_dependencies, None ) - all_required_repo_info_dict = rdm.get_required_repo_info_dicts( self.tool_shed_url, [ repo_info_dict ] ) + all_required_repo_info_dict = rdim.get_required_repo_info_dicts( self.tool_shed_url, [ repo_info_dict ] ) all_repo_info_dicts = all_required_repo_info_dict.get( 'all_repo_info_dicts', [] ) return all_repo_info_dicts @@ -651,8 +646,8 @@ def import_repository_archive( self, repository, repository_archive_dict ): """Import a repository archive contained within a repository capsule.""" - rdah = dependency_manager.RepositoryDependencyAttributeHandler( self.app, unpopulate=False ) - tdah = dependency_manager.ToolDependencyAttributeHandler( self.app, unpopulate=False ) + rdah = RepositoryDependencyAttributeHandler( self.app, unpopulate=False ) + tdah = ToolDependencyAttributeHandler( self.app, unpopulate=False ) archive_file_name = repository_archive_dict.get( 'archive_file_name', None ) capsule_file_name = repository_archive_dict[ 'capsule_file_name' ] encoded_file_path = repository_archive_dict[ 'encoded_file_path' ] diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/tool_shed/dependencies/repository/relation_builder.py --- /dev/null +++ b/lib/tool_shed/dependencies/repository/relation_builder.py @@ -0,0 +1,497 @@ +import logging +import os + +from galaxy.util import asbool +from galaxy.util import listify +from tool_shed.util import common_util +from tool_shed.util import container_util +from tool_shed.util import hg_util +from tool_shed.util import metadata_util +from tool_shed.util import shed_util_common as suc + +log = logging.getLogger( __name__ ) + + +class RelationBuilder( object ): + + def __init__( self, app, repository, repository_metadata, tool_shed_url ): + self.all_repository_dependencies = {} + self.app = app + self.circular_repository_dependencies = [] + self.repository = repository + self.repository_metadata = repository_metadata + self.handled_key_rd_dicts = [] + self.key_rd_dicts_to_be_processed = [] + self.tool_shed_url = tool_shed_url + + def can_add_to_key_rd_dicts( self, key_rd_dict, key_rd_dicts ): + """Handle the case where an update to the changeset revision was done.""" + k = key_rd_dict.keys()[ 0 ] + rd = key_rd_dict[ k ] + partial_rd = rd[ 0:3 ] + for kr_dict in key_rd_dicts: + key = kr_dict.keys()[ 0 ] + if key == k: + repository_dependency = kr_dict[ key ] + if repository_dependency[ 0:3 ] == partial_rd: + return False + return True + + def filter_only_if_compiling_contained_td( self, key_rd_dict ): + """ + Return a copy of the received key_rd_dict with repository dependencies that are needed + only_if_compiling_contained_td filtered out of the list of repository dependencies for + each rd_key. + """ + filtered_key_rd_dict = {} + for rd_key, required_rd_tup in key_rd_dict.items(): + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( required_rd_tup ) + if not asbool( only_if_compiling_contained_td ): + filtered_key_rd_dict[ rd_key ] = required_rd_tup + return filtered_key_rd_dict + + def get_prior_installation_required_and_only_if_compiling_contained_td( self ): + """ + This method is called from the tool shed and never Galaxy. If self.all_repository_dependencies + contains a repository dependency tuple that is associated with self.repository, return the + value of the tuple's prior_installation_required component. + """ + cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url( self.tool_shed_url ) + if self.all_repository_dependencies: + for rd_key, rd_tups in self.all_repository_dependencies.items(): + if rd_key in [ 'root_key', 'description' ]: + continue + for rd_tup in rd_tups: + rd_toolshed, \ + rd_name, \ + rd_owner, \ + rd_changeset_revision, \ + rd_prior_installation_required, \ + rd_only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( rd_tup ) + cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed ) + if cleaned_rd_toolshed == cleaned_toolshed_base_url and \ + rd_name == self.repository.name and \ + rd_owner == self.repository.user.username and \ + rd_changeset_revision == self.repository_metadata.changeset_revision: + return rd_prior_installation_required, rd_only_if_compiling_contained_td + elif self.repository_metadata: + # Get the list of changeset revisions from the tool shed to which self.repository may be updated. + metadata = self.repository_metadata.metadata + current_changeset_revision = str( self.repository_metadata.changeset_revision ) + # Get the changeset revision to which the current value of required_repository_changeset_revision + # should be updated if it's not current. + text = suc.get_updated_changeset_revisions( self.app, + name=str( self.repository.name ), + owner=str( self.repository.user.username ), + changeset_revision=current_changeset_revision ) + if text: + valid_changeset_revisions = listify( text ) + if current_changeset_revision not in valid_changeset_revisions: + valid_changeset_revisions.append( current_changeset_revision ) + else: + valid_changeset_revisions = [ current_changeset_revision ] + repository_dependencies_dict = metadata[ 'repository_dependencies' ] + rd_tups = repository_dependencies_dict.get( 'repository_dependencies', [] ) + for rd_tup in rd_tups: + rd_toolshed, \ + rd_name, \ + rd_owner, \ + rd_changeset_revision, \ + rd_prior_installation_required, \ + rd_only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( rd_tup ) + cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed ) + if cleaned_rd_toolshed == cleaned_toolshed_base_url and \ + rd_name == self.repository.name and \ + rd_owner == self.repository.user.username and \ + rd_changeset_revision in valid_changeset_revisions: + return rd_prior_installation_required, rd_only_if_compiling_contained_td + # Default both prior_installation_required and only_if_compiling_contained_td to False. + return 'False', 'False' + + def get_key_for_repository_changeset_revision( self ): + # The received toolshed_base_url must include the port, but doesn't have to include the protocol. + prior_installation_required, only_if_compiling_contained_td = \ + self.get_prior_installation_required_and_only_if_compiling_contained_td() + # Create a key with the value of prior_installation_required defaulted to False. + key = container_util.generate_repository_dependencies_key_for_repository( self.tool_shed_url, + self.repository.name, + self.repository.user.username, + self.repository_metadata.changeset_revision, + prior_installation_required, + only_if_compiling_contained_td ) + return key + + def get_repository_dependencies_for_changeset_revision( self ): + """ + Return a dictionary of all repositories upon which the contents of self.repository_metadata + record depend. The dictionary keys are name-spaced values consisting of: + self.tool_shed_url/repository_name/repository_owner/changeset_revision + and the values are lists of repository_dependency tuples consisting of: + ( self.tool_shed_url, repository_name, repository_owner, changeset_revision ). + This method ensures that all required repositories to the nth degree are returned. + """ + # Assume the current repository does not have repository dependencies defined for it. + current_repository_key = None + metadata = self.repository_metadata.metadata + if metadata: + # The value of self.tool_shed_url must include the port, but doesn't have to include + # the protocol. + if 'repository_dependencies' in metadata: + current_repository_key = self.get_key_for_repository_changeset_revision() + repository_dependencies_dict = metadata[ 'repository_dependencies' ] + if not self.all_repository_dependencies: + self.initialize_all_repository_dependencies( current_repository_key, repository_dependencies_dict ) + # Handle the repository dependencies defined in the current repository, if any, and populate + # the various repository dependency objects for this round of processing. + current_repository_key_rd_dicts = \ + self.populate_repository_dependency_objects_for_processing( current_repository_key, + repository_dependencies_dict ) + if current_repository_key: + if current_repository_key_rd_dicts: + # There should be only a single current_repository_key_rd_dict in this list. + current_repository_key_rd_dict = current_repository_key_rd_dicts[ 0 ] + # Handle circular repository dependencies. + if not self.in_circular_repository_dependencies( current_repository_key_rd_dict ): + if current_repository_key in self.all_repository_dependencies: + self.handle_current_repository_dependency( current_repository_key ) + elif self.key_rd_dicts_to_be_processed: + self.handle_next_repository_dependency() + elif self.key_rd_dicts_to_be_processed: + self.handle_next_repository_dependency() + elif self.key_rd_dicts_to_be_processed: + self.handle_next_repository_dependency() + self.all_repository_dependencies = self.prune_invalid_repository_dependencies( self.all_repository_dependencies ) + return self.all_repository_dependencies + + def get_repository_dependency_as_key( self, repository_dependency ): + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( repository_dependency ) + return container_util.generate_repository_dependencies_key_for_repository( tool_shed, + name, + owner, + changeset_revision, + prior_installation_required, + only_if_compiling_contained_td ) + + def get_updated_changeset_revisions_for_repository_dependencies( self, key_rd_dicts ): + updated_key_rd_dicts = [] + for key_rd_dict in key_rd_dicts: + key = key_rd_dict.keys()[ 0 ] + repository_dependency = key_rd_dict[ key ] + rd_toolshed, \ + rd_name, \ + rd_owner, \ + rd_changeset_revision, \ + rd_prior_installation_required, \ + rd_only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( repository_dependency ) + if suc.tool_shed_is_this_tool_shed( rd_toolshed ): + repository = suc.get_repository_by_name_and_owner( self.app, rd_name, rd_owner ) + if repository: + repository_id = self.app.security.encode_id( repository.id ) + repository_metadata = \ + metadata_util.get_repository_metadata_by_repository_id_changeset_revision( self.app, + repository_id, + rd_changeset_revision ) + if repository_metadata: + # The repository changeset_revision is installable, so no updates are available. + new_key_rd_dict = {} + new_key_rd_dict[ key ] = repository_dependency + updated_key_rd_dicts.append( key_rd_dict ) + else: + # The repository changeset_revision is no longer installable, so see if there's been an update. + repo = hg_util.get_repo_for_repository( self.app, repository=repository, repo_path=None, create=False ) + changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, rd_changeset_revision ) + repository_metadata = \ + metadata_util.get_repository_metadata_by_repository_id_changeset_revision( self.app, + repository_id, + changeset_revision ) + if repository_metadata: + new_key_rd_dict = {} + new_key_rd_dict[ key ] = \ + [ rd_toolshed, \ + rd_name, \ + rd_owner, \ + repository_metadata.changeset_revision, \ + rd_prior_installation_required, \ + rd_only_if_compiling_contained_td ] + # We have the updated changset revision. + updated_key_rd_dicts.append( new_key_rd_dict ) + else: + repository_components_tuple = container_util.get_components_from_key( key ) + components_list = suc.extract_components_from_tuple( repository_components_tuple ) + toolshed, repository_name, repository_owner, repository_changeset_revision = components_list[ 0:4 ] + # For backward compatibility to the 12/20/12 Galaxy release. + if len( components_list ) == 4: + prior_installation_required = 'False' + rd_only_if_compiling_contained_td = 'False' + elif len( components_list ) == 5: + rd_only_if_compiling_contained_td = 'False' + message = "The revision %s defined for repository %s owned by %s is invalid, so repository " % \ + ( str( rd_changeset_revision ), str( rd_name ), str( rd_owner ) ) + message += "dependencies defined for repository %s will be ignored." % str( repository_name ) + log.debug( message ) + else: + repository_components_tuple = container_util.get_components_from_key( key ) + components_list = suc.extract_components_from_tuple( repository_components_tuple ) + toolshed, repository_name, repository_owner, repository_changeset_revision = components_list[ 0:4 ] + message = "The revision %s defined for repository %s owned by %s is invalid, so repository " % \ + ( str( rd_changeset_revision ), str( rd_name ), str( rd_owner ) ) + message += "dependencies defined for repository %s will be ignored." % str( repository_name ) + log.debug( message ) + return updated_key_rd_dicts + + def handle_circular_repository_dependency( self, repository_key, repository_dependency ): + all_repository_dependencies_root_key = self.all_repository_dependencies[ 'root_key' ] + repository_dependency_as_key = self.get_repository_dependency_as_key( repository_dependency ) + repository_key_as_repository_dependency = repository_key.split( container_util.STRSEP ) + self.update_circular_repository_dependencies( repository_key, + repository_dependency, + self.all_repository_dependencies[ repository_dependency_as_key ] ) + if all_repository_dependencies_root_key != repository_dependency_as_key: + self.all_repository_dependencies[ repository_key ] = [ repository_dependency ] + + def handle_current_repository_dependency( self, current_repository_key ): + current_repository_key_rd_dicts = [] + for rd in self.all_repository_dependencies[ current_repository_key ]: + rd_copy = [ str( item ) for item in rd ] + new_key_rd_dict = {} + new_key_rd_dict[ current_repository_key ] = rd_copy + current_repository_key_rd_dicts.append( new_key_rd_dict ) + if current_repository_key_rd_dicts: + self.handle_key_rd_dicts_for_repository( current_repository_key, current_repository_key_rd_dicts ) + return self.get_repository_dependencies_for_changeset_revision() + + def handle_key_rd_dicts_for_repository( self, current_repository_key, repository_key_rd_dicts ): + key_rd_dict = repository_key_rd_dicts.pop( 0 ) + repository_dependency = key_rd_dict[ current_repository_key ] + toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( repository_dependency ) + if suc.tool_shed_is_this_tool_shed( toolshed ): + required_repository = suc.get_repository_by_name_and_owner( self.app, name, owner ) + self.repository = required_repository + repository_id = self.app.security.encode_id( required_repository.id ) + required_repository_metadata = \ + metadata_util.get_repository_metadata_by_repository_id_changeset_revision( self.app, + repository_id, + changeset_revision ) + self.repository_metadata = required_repository_metadata + if required_repository_metadata: + # The required_repository_metadata changeset_revision is installable. + required_metadata = required_repository_metadata.metadata + if required_metadata: + for current_repository_key_rd_dict in repository_key_rd_dicts: + if not self.in_key_rd_dicts( current_repository_key_rd_dict, self.key_rd_dicts_to_be_processed ): + # Add the current repository_dependency into self.key_rd_dicts_to_be_processed. + self.key_rd_dicts_to_be_processed.append( current_repository_key_rd_dict ) + if not self.in_key_rd_dicts( key_rd_dict, self.handled_key_rd_dicts ): + # Add the current repository_dependency into self.handled_key_rd_dicts. + self.handled_key_rd_dicts.append( key_rd_dict ) + if self.in_key_rd_dicts( key_rd_dict, self.key_rd_dicts_to_be_processed ): + # Remove the current repository from self.key_rd_dicts_to_be_processed. + self.key_rd_dicts_to_be_processed = self.remove_from_key_rd_dicts( key_rd_dict, self.key_rd_dicts_to_be_processed ) + else: + # The repository is in a different tool shed, so build an url and send a request. + error_message = "Repository dependencies are currently supported only within the same Tool Shed. " + error_message += "Ignoring repository dependency definition for tool shed " + error_message += "%s, name %s, owner %s, changeset revision %s" % ( toolshed, name, owner, changeset_revision ) + log.debug( error_message ) + + def handle_next_repository_dependency( self ): + next_repository_key_rd_dict = self.key_rd_dicts_to_be_processed.pop( 0 ) + next_repository_key_rd_dicts = [ next_repository_key_rd_dict ] + next_repository_key = next_repository_key_rd_dict.keys()[ 0 ] + self.handle_key_rd_dicts_for_repository( next_repository_key, next_repository_key_rd_dicts ) + return self.get_repository_dependencies_for_changeset_revision() + + def in_all_repository_dependencies( self, repository_key, repository_dependency ): + """ + Return True if { repository_key : repository_dependency } is in self.all_repository_dependencies. + """ + for key, val in self.all_repository_dependencies.items(): + if key != repository_key: + continue + if repository_dependency in val: + return True + return False + + def in_circular_repository_dependencies( self, repository_key_rd_dict ): + """ + Return True if any combination of a circular dependency tuple is the key : value pair defined + in the received repository_key_rd_dict. This means that each circular dependency tuple is converted + into the key : value pair for comparison. + """ + for tup in self.circular_repository_dependencies: + rd_0, rd_1 = tup + rd_0_as_key = self.get_repository_dependency_as_key( rd_0 ) + rd_1_as_key = self.get_repository_dependency_as_key( rd_1 ) + if rd_0_as_key in repository_key_rd_dict and repository_key_rd_dict[ rd_0_as_key ] == rd_1: + return True + if rd_1_as_key in repository_key_rd_dict and repository_key_rd_dict[ rd_1_as_key ] == rd_0: + return True + return False + + def in_key_rd_dicts( self, key_rd_dict, key_rd_dicts ): + """Return True if key_rd_dict is contained in the list of key_rd_dicts.""" + k = key_rd_dict.keys()[ 0 ] + v = key_rd_dict[ k ] + for key_rd_dict in key_rd_dicts: + for key, val in key_rd_dict.items(): + if key == k and val == v: + return True + return False + + def initialize_all_repository_dependencies( self, current_repository_key, repository_dependencies_dict ): + """Initialize the self.all_repository_dependencies dictionary.""" + # It's safe to assume that current_repository_key in this case will have a value. + self.all_repository_dependencies[ 'root_key' ] = current_repository_key + self.all_repository_dependencies[ current_repository_key ] = [] + # Store the value of the 'description' key only once, the first time through this recursive method. + description = repository_dependencies_dict.get( 'description', None ) + self.all_repository_dependencies[ 'description' ] = description + + def is_circular_repository_dependency( self, repository_key, repository_dependency ): + """ + Return True if the received repository_dependency is a key in self.all_repository_dependencies + whose list of repository dependencies includes the received repository_key. + """ + repository_dependency_as_key = self.get_repository_dependency_as_key( repository_dependency ) + repository_key_as_repository_dependency = repository_key.split( container_util.STRSEP ) + for key, val in self.all_repository_dependencies.items(): + if key != repository_dependency_as_key: + continue + if repository_key_as_repository_dependency in val: + return True + return False + + def populate_repository_dependency_objects_for_processing( self, current_repository_key, repository_dependencies_dict ): + """ + The process that discovers all repository dependencies for a specified repository's changeset + revision uses this method to populate the following items for the current processing loop: + filtered_current_repository_key_rd_dicts, self.key_rd_dicts_to_be_processed, + self.handled_key_rd_dicts, self.all_repository_dependencies. Each processing loop may discover + more repository dependencies, so this method is repeatedly called until all repository + dependencies have been discovered. + """ + current_repository_key_rd_dicts = [] + filtered_current_repository_key_rd_dicts = [] + for rd_tup in repository_dependencies_dict[ 'repository_dependencies' ]: + new_key_rd_dict = {} + new_key_rd_dict[ current_repository_key ] = rd_tup + current_repository_key_rd_dicts.append( new_key_rd_dict ) + if current_repository_key_rd_dicts and current_repository_key: + # Remove all repository dependencies that point to a revision within its own repository. + current_repository_key_rd_dicts = \ + self.remove_ropository_dependency_reference_to_self( current_repository_key_rd_dicts ) + current_repository_key_rd_dicts = \ + self.get_updated_changeset_revisions_for_repository_dependencies( current_repository_key_rd_dicts ) + for key_rd_dict in current_repository_key_rd_dicts: + # Filter out repository dependencies that are required only if compiling the dependent + # repository's tool dependency. + key_rd_dict = self.filter_only_if_compiling_contained_td( key_rd_dict ) + if key_rd_dict: + is_circular = False + in_handled_key_rd_dicts = self.in_key_rd_dicts( key_rd_dict, self.handled_key_rd_dicts ) + in_key_rd_dicts_to_be_processed = self.in_key_rd_dicts( key_rd_dict, self.key_rd_dicts_to_be_processed ) + if not in_handled_key_rd_dicts and not in_key_rd_dicts_to_be_processed: + filtered_current_repository_key_rd_dicts.append( key_rd_dict ) + repository_dependency = key_rd_dict[ current_repository_key ] + if current_repository_key in self.all_repository_dependencies: + # Add all repository dependencies for the current repository into its entry + # in self.all_repository_dependencies. + all_repository_dependencies_val = self.all_repository_dependencies[ current_repository_key ] + if repository_dependency not in all_repository_dependencies_val: + all_repository_dependencies_val.append( repository_dependency ) + self.all_repository_dependencies[ current_repository_key ] = all_repository_dependencies_val + elif not self.in_all_repository_dependencies( current_repository_key, repository_dependency ): + # Handle circular repository dependencies. + if self.is_circular_repository_dependency( current_repository_key, repository_dependency ): + is_circular = True + self.handle_circular_repository_dependency( current_repository_key, repository_dependency ) + else: + self.all_repository_dependencies[ current_repository_key ] = [ repository_dependency ] + if not is_circular and self.can_add_to_key_rd_dicts( key_rd_dict, self.key_rd_dicts_to_be_processed ): + new_key_rd_dict = {} + new_key_rd_dict[ current_repository_key ] = repository_dependency + self.key_rd_dicts_to_be_processed.append( new_key_rd_dict ) + return filtered_current_repository_key_rd_dicts + + def prune_invalid_repository_dependencies( self, repository_dependencies ): + """ + Eliminate all invalid entries in the received repository_dependencies dictionary. An entry + is invalid if the value_list of the key/value pair is empty. This occurs when an invalid + combination of tool shed, name , owner, changeset_revision is used and a repository_metadata + record is not found. + """ + valid_repository_dependencies = {} + description = repository_dependencies.get( 'description', None ) + root_key = repository_dependencies.get( 'root_key', None ) + if root_key is None: + return valid_repository_dependencies + for key, value in repository_dependencies.items(): + if key in [ 'description', 'root_key' ]: + continue + if value: + valid_repository_dependencies[ key ] = value + if valid_repository_dependencies: + valid_repository_dependencies[ 'description' ] = description + valid_repository_dependencies[ 'root_key' ] = root_key + return valid_repository_dependencies + + def remove_from_key_rd_dicts( self, key_rd_dict, key_rd_dicts ): + """Eliminate the key_rd_dict from the list of key_rd_dicts if it is contained in the list.""" + k = key_rd_dict.keys()[ 0 ] + v = key_rd_dict[ k ] + clean_key_rd_dicts = [] + for krd_dict in key_rd_dicts: + key = krd_dict.keys()[ 0 ] + val = krd_dict[ key ] + if key == k and val == v: + continue + clean_key_rd_dicts.append( krd_dict ) + return clean_key_rd_dicts + + def remove_ropository_dependency_reference_to_self( self, key_rd_dicts ): + """Remove all repository dependencies that point to a revision within its own repository.""" + clean_key_rd_dicts = [] + key = key_rd_dicts[ 0 ].keys()[ 0 ] + repository_tup = key.split( container_util.STRSEP ) + rd_toolshed, \ + rd_name, \ + rd_owner, \ + rd_changeset_revision, \ + rd_prior_installation_required, \ + rd_only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( repository_tup ) + cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed ) + for key_rd_dict in key_rd_dicts: + k = key_rd_dict.keys()[ 0 ] + repository_dependency = key_rd_dict[ k ] + toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( repository_dependency ) + cleaned_toolshed = common_util.remove_protocol_from_tool_shed_url( toolshed ) + if cleaned_rd_toolshed == cleaned_toolshed and rd_name == name and rd_owner == owner: + debug_msg = "Removing repository dependency for repository %s owned by %s " % ( name, owner ) + debug_msg += 'since it refers to a revision within itself.' + log.debug( debug_msg ) + else: + new_key_rd_dict = {} + new_key_rd_dict[ key ] = repository_dependency + clean_key_rd_dicts.append( new_key_rd_dict ) + return clean_key_rd_dicts + + def update_circular_repository_dependencies( self, repository_key, repository_dependency, repository_dependencies ): + repository_dependency_as_key = self.get_repository_dependency_as_key( repository_dependency ) + repository_key_as_repository_dependency = repository_key.split( container_util.STRSEP ) + if repository_key_as_repository_dependency in repository_dependencies: + found = False + for tup in self.circular_repository_dependencies: + if repository_dependency in tup and repository_key_as_repository_dependency in tup: + # The circular dependency has already been included. + found = True + if not found: + new_circular_tup = [ repository_dependency, repository_key_as_repository_dependency ] + self.circular_repository_dependencies.append( new_circular_tup ) diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -25,7 +25,6 @@ from tool_shed.util import encoding_util from tool_shed.util import hg_util from tool_shed.util import metadata_util -from tool_shed.util import repository_dependency_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util @@ -35,7 +34,7 @@ from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import StepManager from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import TagManager -from tool_shed.galaxy_install.repository_dependencies.repository_dependency_manager import RepositoryDependencyManager +from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager log = logging.getLogger( __name__ ) @@ -612,15 +611,15 @@ tool_panel_section_id = installation_dict[ 'tool_panel_section_id' ] tool_path = installation_dict[ 'tool_path' ] tool_shed_url = installation_dict[ 'tool_shed_url' ] - rdm = RepositoryDependencyManager( self.app ) + rdim = repository_dependency_manager.RepositoryDependencyInstallManager( self.app ) created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts = \ - rdm.create_repository_dependency_objects( tool_path=tool_path, - tool_shed_url=tool_shed_url, - repo_info_dicts=repo_info_dicts, - install_repository_dependencies=install_repository_dependencies, - no_changes_checked=no_changes_checked, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) + rdim.create_repository_dependency_objects( tool_path=tool_path, + tool_shed_url=tool_shed_url, + repo_info_dicts=repo_info_dicts, + install_repository_dependencies=install_repository_dependencies, + no_changes_checked=no_changes_checked, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) return created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts def initiate_repository_installation( self, installation_dict ): diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/tool_shed/galaxy_install/installed_repository_manager.py --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -9,14 +9,13 @@ from tool_shed.util import container_util from tool_shed.util import data_manager_util from tool_shed.util import datatype_util -from tool_shed.util import repository_dependency_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util from tool_shed.util import xml_util from galaxy.model.orm import and_ -from tool_shed.galaxy_install.repository_dependencies.repository_dependency_manager import RepositoryDependencyManager +from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager log = logging.getLogger( __name__ ) @@ -229,7 +228,7 @@ Return dictionaries containing the sets of installed and missing tool dependencies and repository dependencies associated with the repository defined by the received repo_info_dict. """ - rdm = RepositoryDependencyManager( self.app ) + rdim = repository_dependency_manager.RepositoryDependencyInstallManager( self.app ) repository = None installed_rd = {} installed_td = {} @@ -261,7 +260,7 @@ installed_rd, missing_rd = \ self.get_installed_and_missing_repository_dependencies_for_new_or_updated_install( repo_info_tuple ) # Discover all repository dependencies and retrieve information for installing them. - all_repo_info_dict = rdm.get_required_repo_info_dicts( tool_shed_url, util.listify( repo_info_dict ) ) + all_repo_info_dict = rdim.get_required_repo_info_dicts( tool_shed_url, util.listify( repo_info_dict ) ) has_repository_dependencies = all_repo_info_dict.get( 'has_repository_dependencies', False ) has_repository_dependencies_only_if_compiling_contained_td = \ all_repo_info_dict.get( 'has_repository_dependencies_only_if_compiling_contained_td', False ) @@ -299,7 +298,7 @@ missing_td[ td_key ] = td_dict else: # We have a single repository with (possibly) no defined repository dependencies. - all_repo_info_dict = rdm.get_required_repo_info_dicts( tool_shed_url, util.listify( repo_info_dict ) ) + all_repo_info_dict = rdim.get_required_repo_info_dicts( tool_shed_url, util.listify( repo_info_dict ) ) has_repository_dependencies = all_repo_info_dict.get( 'has_repository_dependencies', False ) has_repository_dependencies_only_if_compiling_contained_td = \ all_repo_info_dict.get( 'has_repository_dependencies_only_if_compiling_contained_td', False ) diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/tool_shed/galaxy_install/repair_repository_manager.py --- a/lib/tool_shed/galaxy_install/repair_repository_manager.py +++ b/lib/tool_shed/galaxy_install/repair_repository_manager.py @@ -4,12 +4,11 @@ log = logging.getLogger( __name__ ) from tool_shed.galaxy_install import install_manager -from tool_shed.galaxy_install.repository_dependencies.repository_dependency_manager import RepositoryDependencyManager +from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager from tool_shed.util import common_util from tool_shed.util import container_util from tool_shed.util import shed_util_common as suc -from tool_shed.util import repository_dependency_util from tool_shed.util import repository_maintenance_util from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util @@ -56,7 +55,7 @@ issues with an installed repository that has installation problems somewhere in its dependency hierarchy. """ - rdm = RepositoryDependencyManager( self.app ) + rdim = repository_dependency_manager.RepositoryDependencyInstallManager( self.app ) tsr_ids = [] repo_info_dicts = [] tool_panel_section_keys = [] @@ -64,8 +63,8 @@ irm = install_manager.InstallRepositoryManager( self.app ) # Get a dictionary of all repositories upon which the contents of the current repository_metadata #record depend. - repository_dependencies_dict = rdm.get_repository_dependencies_for_installed_tool_shed_repository( self.app, - repository ) + repository_dependencies_dict = rdim.get_repository_dependencies_for_installed_tool_shed_repository( self.app, + repository ) if repository_dependencies_dict: # Generate the list of installed repositories from the information contained in the # repository_dependencies dictionary. @@ -75,14 +74,14 @@ # repaired in the required order. for installed_repository in installed_repositories: tsr_ids.append( self.app.security.encode_id( installed_repository.id ) ) - repo_info_dict, tool_panel_section_key = self.get_repo_info_dict_for_repair( rdm, + repo_info_dict, tool_panel_section_key = self.get_repo_info_dict_for_repair( rdim, installed_repository ) tool_panel_section_keys.append( tool_panel_section_key ) repo_info_dicts.append( repo_info_dict ) else: # The received repository has no repository dependencies. tsr_ids.append( self.app.security.encode_id( repository.id ) ) - repo_info_dict, tool_panel_section_key = self.get_repo_info_dict_for_repair( rdm, + repo_info_dict, tool_panel_section_key = self.get_repo_info_dict_for_repair( rdim, repository ) tool_panel_section_keys.append( tool_panel_section_key ) repo_info_dicts.append( repo_info_dict ) @@ -95,11 +94,11 @@ repair_dict[ 'ordered_tool_panel_section_keys' ] = ordered_tool_panel_section_keys return repair_dict - def get_repo_info_dict_for_repair( self, rdm, repository ): + def get_repo_info_dict_for_repair( self, rdim, repository ): tool_panel_section_key = None repository_clone_url = common_util.generate_clone_url_for_installed_repository( self.app, repository ) - repository_dependencies = rdm.get_repository_dependencies_for_installed_tool_shed_repository( self.app, - repository ) + repository_dependencies = rdim.get_repository_dependencies_for_installed_tool_shed_repository( self.app, + repository ) metadata = repository.metadata if metadata: tool_dependencies = metadata.get( 'tool_dependencies', None ) diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py --- a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py +++ b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py @@ -19,7 +19,7 @@ log = logging.getLogger( __name__ ) -class RepositoryDependencyManager( object ): +class RepositoryDependencyInstallManager( object ): def __init__( self, app ): self.app = app diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -607,6 +607,48 @@ tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) return '%s/%s/%s/%s' % ( tmp_url, guid_type, obj_id, version ) +def generate_message_for_invalid_repository_dependencies( metadata_dict, error_from_tuple=False ): + """Get or generate and return an error message associated with an invalid repository dependency.""" + message = '' + if metadata_dict: + if error_from_tuple: + # Return the error messages associated with a set of one or more invalid repository dependency tuples. + invalid_repository_dependencies_dict = metadata_dict.get( 'invalid_repository_dependencies', None ) + if invalid_repository_dependencies_dict is not None: + invalid_repository_dependencies = invalid_repository_dependencies_dict.get( 'invalid_repository_dependencies', [] ) + for repository_dependency_tup in invalid_repository_dependencies: + toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error = \ + common_util.parse_repository_dependency_tuple( repository_dependency_tup, contains_error=True ) + if error: + message += '%s ' % str( error ) + else: + # The complete dependency hierarchy could not be determined for a repository being installed into + # Galaxy. This is likely due to invalid repository dependency definitions, so we'll get them from + # the metadata and parse them for display in an error message. This will hopefully communicate the + # problem to the user in such a way that a resolution can be determined. + message += 'The complete dependency hierarchy could not be determined for this repository, so no required ' + message += 'repositories will not be installed. This is likely due to invalid repository dependency definitions. ' + repository_dependencies_dict = metadata_dict.get( 'repository_dependencies', None ) + if repository_dependencies_dict is not None: + rd_tups = repository_dependencies_dict.get( 'repository_dependencies', None ) + if rd_tups is not None: + message += 'Here are the attributes of the dependencies defined for this repository to help determine the ' + message += 'cause of this problem.<br/>' + message += '<table cellpadding="2" cellspacing="2">' + message += '<tr><th>Tool shed</th><th>Repository name</th><th>Owner</th><th>Changeset revision</th>' + message += '<th>Prior install required</th></tr>' + for rd_tup in rd_tups: + tool_shed, name, owner, changeset_revision, pir, oicct = \ + common_util.parse_repository_dependency_tuple( rd_tup ) + if util.asbool( pir ): + pir_str = 'True' + else: + pir_str = '' + message += '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % \ + ( tool_shed, name, owner, changeset_revision, pir_str ) + message += '</table>' + return message + def generate_metadata_for_changeset_revision( app, repository, changeset_revision, repository_clone_url, shed_config_dict=None, relative_install_dir=None, repository_files_dir=None, resetting_all_metadata_on_repository=False, updating_installed_repository=False, @@ -1156,6 +1198,36 @@ relative_path_to_file = relative_path_to_file[ len( shed_config_dict.get( 'tool_path' ) ) + 1: ] return relative_path_to_file +def get_repository_dependency_tups_from_repository_metadata( app, repository_metadata, deprecated_only=False ): + """ + Return a list of of tuples defining repository objects required by the received repository. The returned + list defines the entire repository dependency tree. This method is called only from the Tool Shed. + """ + dependency_tups = [] + if repository_metadata is not None: + metadata = repository_metadata.metadata + if metadata: + repository_dependencies_dict = metadata.get( 'repository_dependencies', None ) + if repository_dependencies_dict is not None: + repository_dependency_tups = repository_dependencies_dict.get( 'repository_dependencies', None ) + if repository_dependency_tups is not None: + # The value of repository_dependency_tups is a list of repository dependency tuples like this: + # ['http://localhost:9009', 'package_samtools_0_1_18', 'devteam', 'ef37fc635cb9', 'False', 'False'] + for repository_dependency_tup in repository_dependency_tups: + toolshed, name, owner, changeset_revision, pir, oicct = \ + common_util.parse_repository_dependency_tuple( repository_dependency_tup ) + repository = suc.get_repository_by_name_and_owner( app, name, owner ) + if repository: + if deprecated_only: + if repository.deprecated: + dependency_tups.append( repository_dependency_tup ) + else: + dependency_tups.append( repository_dependency_tup ) + else: + log.debug( "Cannot locate repository %s owned by %s for inclusion in repository dependency tups." % \ + ( name, owner ) ) + return dependency_tups + def get_repository_metadata_by_id( app, id ): """Get repository metadata from the database""" sa_session = app.model.context.current diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/tool_shed/util/repository_dependency_util.py --- a/lib/tool_shed/util/repository_dependency_util.py +++ /dev/null @@ -1,609 +0,0 @@ -import json -import logging -import os - -from galaxy.util import asbool -from galaxy.util import listify - -import tool_shed.util.shed_util_common as suc -from tool_shed.util import common_util -from tool_shed.util import container_util -from tool_shed.util import encoding_util -from tool_shed.util import hg_util -from tool_shed.util import metadata_util -from tool_shed.util import tool_util - -log = logging.getLogger( __name__ ) - -def can_add_to_key_rd_dicts( key_rd_dict, key_rd_dicts ): - """Handle the case where an update to the changeset revision was done.""" - k = key_rd_dict.keys()[ 0 ] - rd = key_rd_dict[ k ] - partial_rd = rd[ 0:3 ] - for kr_dict in key_rd_dicts: - key = kr_dict.keys()[ 0 ] - if key == k: - repository_dependency = kr_dict[ key ] - if repository_dependency[ 0:3 ] == partial_rd: - return False - return True - -def generate_message_for_invalid_repository_dependencies( metadata_dict, error_from_tuple=False ): - """Get or generate and return an error message associated with an invalid repository dependency.""" - message = '' - if metadata_dict: - if error_from_tuple: - # Return the error messages associated with a set of one or more invalid repository dependency tuples. - invalid_repository_dependencies_dict = metadata_dict.get( 'invalid_repository_dependencies', None ) - if invalid_repository_dependencies_dict is not None: - invalid_repository_dependencies = invalid_repository_dependencies_dict.get( 'invalid_repository_dependencies', [] ) - for repository_dependency_tup in invalid_repository_dependencies: - toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error = \ - common_util.parse_repository_dependency_tuple( repository_dependency_tup, contains_error=True ) - if error: - message += '%s ' % str( error ) - else: - # The complete dependency hierarchy could not be determined for a repository being installed into - # Galaxy. This is likely due to invalid repository dependency definitions, so we'll get them from - # the metadata and parse them for display in an error message. This will hopefully communicate the - # problem to the user in such a way that a resolution can be determined. - message += 'The complete dependency hierarchy could not be determined for this repository, so no required ' - message += 'repositories will not be installed. This is likely due to invalid repository dependency definitions. ' - repository_dependencies_dict = metadata_dict.get( 'repository_dependencies', None ) - if repository_dependencies_dict is not None: - rd_tups = repository_dependencies_dict.get( 'repository_dependencies', None ) - if rd_tups is not None: - message += 'Here are the attributes of the dependencies defined for this repository to help determine the ' - message += 'cause of this problem.<br/>' - message += '<table cellpadding="2" cellspacing="2">' - message += '<tr><th>Tool shed</th><th>Repository name</th><th>Owner</th><th>Changeset revision</th>' - message += '<th>Prior install required</th></tr>' - for rd_tup in rd_tups: - tool_shed, name, owner, changeset_revision, pir, oicct = \ - common_util.parse_repository_dependency_tuple( rd_tup ) - if asbool( pir ): - pir_str = 'True' - else: - pir_str = '' - message += '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % \ - ( tool_shed, name, owner, changeset_revision, pir_str ) - message += '</table>' - return message - -def get_key_for_repository_changeset_revision( app, toolshed_base_url, repository, repository_metadata, all_repository_dependencies ): - # The received toolshed_base_url must include the port, but doesn't have to include the protocol. - prior_installation_required, only_if_compiling_contained_td = \ - get_prior_installation_required_and_only_if_compiling_contained_td( app, - toolshed_base_url, - repository, - repository_metadata, - all_repository_dependencies ) - # Create a key with the value of prior_installation_required defaulted to False. - key = container_util.generate_repository_dependencies_key_for_repository( toolshed_base_url=toolshed_base_url, - repository_name=repository.name, - repository_owner=repository.user.username, - changeset_revision=repository_metadata.changeset_revision, - prior_installation_required=prior_installation_required, - only_if_compiling_contained_td=only_if_compiling_contained_td ) - return key - -def get_prior_installation_required_and_only_if_compiling_contained_td( app, toolshed_base_url, repository, repository_metadata, - all_repository_dependencies ): - """ - This method is called from the tool shed and never Galaxy. If all_repository_dependencies contains - a repository dependency tuple that is associated with the received repository, return the value of - the tuple's prior_installation_required component. - """ - cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url( toolshed_base_url ) - if all_repository_dependencies: - for rd_key, rd_tups in all_repository_dependencies.items(): - if rd_key in [ 'root_key', 'description' ]: - continue - for rd_tup in rd_tups: - rd_toolshed, rd_name, rd_owner, rd_changeset_revision, rd_prior_installation_required, rd_only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( rd_tup ) - cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed ) - if cleaned_rd_toolshed == cleaned_toolshed_base_url and \ - rd_name == repository.name and \ - rd_owner == repository.user.username and \ - rd_changeset_revision == repository_metadata.changeset_revision: - return rd_prior_installation_required, rd_only_if_compiling_contained_td - elif repository_metadata: - # Get the list of changeset revisions from the tool shed to which the repository may be updated. - metadata = repository_metadata.metadata - current_changeset_revision = str( repository_metadata.changeset_revision ) - # Get the changeset revision to which the current value of required_repository_changeset_revision should be updated if it's not current. - text = suc.get_updated_changeset_revisions( app, - name=str( repository.name ), - owner=str( repository.user.username ), - changeset_revision=current_changeset_revision ) - if text: - valid_changeset_revisions = listify( text ) - if current_changeset_revision not in valid_changeset_revisions: - valid_changeset_revisions.append( current_changeset_revision ) - else: - valid_changeset_revisions = [ current_changeset_revision ] - repository_dependencies_dict = metadata[ 'repository_dependencies' ] - rd_tups = repository_dependencies_dict.get( 'repository_dependencies', [] ) - for rd_tup in rd_tups: - rd_toolshed, rd_name, rd_owner, rd_changeset_revision, rd_prior_installation_required, rd_only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( rd_tup ) - cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed ) - if cleaned_rd_toolshed == cleaned_toolshed_base_url and \ - rd_name == repository.name and \ - rd_owner == repository.user.username and \ - rd_changeset_revision in valid_changeset_revisions: - return rd_prior_installation_required, rd_only_if_compiling_contained_td - # Default both prior_installation_required and only_if_compiling_contained_td to False. - return 'False', 'False' - -def get_repository_dependency_as_key( repository_dependency ): - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( repository_dependency ) - return container_util.generate_repository_dependencies_key_for_repository( tool_shed, - name, - owner, - changeset_revision, - prior_installation_required, - only_if_compiling_contained_td ) - -def get_repository_dependencies_for_changeset_revision( app, repository, repository_metadata, toolshed_base_url, - key_rd_dicts_to_be_processed=None, all_repository_dependencies=None, - handled_key_rd_dicts=None, circular_repository_dependencies=None ): - """ - Return a dictionary of all repositories upon which the contents of the received - repository_metadata record depend. The dictionary keys are name-spaced values - consisting of: - toolshed_base_url/repository_name/repository_owner/changeset_revision - and the values are lists of repository_dependency tuples consisting of: - ( toolshed_base_url, repository_name, repository_owner, changeset_revision ). - This method ensures that all required repositories to the nth degree are returned. - """ - if handled_key_rd_dicts is None: - handled_key_rd_dicts = [] - if all_repository_dependencies is None: - all_repository_dependencies = {} - if key_rd_dicts_to_be_processed is None: - key_rd_dicts_to_be_processed = [] - if circular_repository_dependencies is None: - circular_repository_dependencies = [] - # Assume the current repository does not have repository dependencies defined for it. - current_repository_key = None - metadata = repository_metadata.metadata - if metadata: - # The value of the received toolshed_base_url must include the port, but doesn't have - # to include the protocol. - if 'repository_dependencies' in metadata: - current_repository_key = get_key_for_repository_changeset_revision( app, - toolshed_base_url, - repository, - repository_metadata, - all_repository_dependencies ) - repository_dependencies_dict = metadata[ 'repository_dependencies' ] - if not all_repository_dependencies: - all_repository_dependencies = initialize_all_repository_dependencies( current_repository_key, - repository_dependencies_dict, - all_repository_dependencies ) - # Handle the repository dependencies defined in the current repository, if any, and populate - # the various repository dependency objects for this round of processing. - current_repository_key_rd_dicts, key_rd_dicts_to_be_processed, handled_key_rd_dicts, all_repository_dependencies = \ - populate_repository_dependency_objects_for_processing( app, - current_repository_key, - repository_dependencies_dict, - key_rd_dicts_to_be_processed, - handled_key_rd_dicts, - circular_repository_dependencies, - all_repository_dependencies ) - if current_repository_key: - if current_repository_key_rd_dicts: - # There should be only a single current_repository_key_rd_dict in this list. - current_repository_key_rd_dict = current_repository_key_rd_dicts[ 0 ] - # Handle circular repository dependencies. - if not in_circular_repository_dependencies( current_repository_key_rd_dict, - circular_repository_dependencies ): - if current_repository_key in all_repository_dependencies: - handle_current_repository_dependency( app, - current_repository_key, - key_rd_dicts_to_be_processed, - all_repository_dependencies, - handled_key_rd_dicts, - circular_repository_dependencies ) - elif key_rd_dicts_to_be_processed: - handle_next_repository_dependency( app, - key_rd_dicts_to_be_processed, - all_repository_dependencies, - handled_key_rd_dicts, - circular_repository_dependencies ) - elif key_rd_dicts_to_be_processed: - handle_next_repository_dependency( app, - key_rd_dicts_to_be_processed, - all_repository_dependencies, - handled_key_rd_dicts, - circular_repository_dependencies ) - elif key_rd_dicts_to_be_processed: - handle_next_repository_dependency( app, - key_rd_dicts_to_be_processed, - all_repository_dependencies, - handled_key_rd_dicts, - circular_repository_dependencies ) - all_repository_dependencies = prune_invalid_repository_dependencies( all_repository_dependencies ) - return all_repository_dependencies - -def get_repository_dependency_tups_from_repository_metadata( app, repository_metadata, deprecated_only=False ): - """ - Return a list of of tuples defining repository objects required by the received repository. The returned - list defines the entire repository dependency tree. This method is called only from the Tool Shed. - """ - dependency_tups = [] - if repository_metadata is not None: - metadata = repository_metadata.metadata - if metadata: - repository_dependencies_dict = metadata.get( 'repository_dependencies', None ) - if repository_dependencies_dict is not None: - repository_dependency_tups = repository_dependencies_dict.get( 'repository_dependencies', None ) - if repository_dependency_tups is not None: - # The value of repository_dependency_tups is a list of repository dependency tuples like this: - # ['http://localhost:9009', 'package_samtools_0_1_18', 'devteam', 'ef37fc635cb9', 'False', 'False'] - for repository_dependency_tup in repository_dependency_tups: - toolshed, name, owner, changeset_revision, pir, oicct = \ - common_util.parse_repository_dependency_tuple( repository_dependency_tup ) - repository = suc.get_repository_by_name_and_owner( app, name, owner ) - if repository: - if deprecated_only: - if repository.deprecated: - dependency_tups.append( repository_dependency_tup ) - else: - dependency_tups.append( repository_dependency_tup ) - else: - log.debug( "Cannot locate repository %s owned by %s for inclusion in repository dependency tups." % \ - ( name, owner ) ) - return dependency_tups - -def get_updated_changeset_revisions_for_repository_dependencies( app, key_rd_dicts ): - updated_key_rd_dicts = [] - for key_rd_dict in key_rd_dicts: - key = key_rd_dict.keys()[ 0 ] - repository_dependency = key_rd_dict[ key ] - rd_toolshed, rd_name, rd_owner, rd_changeset_revision, rd_prior_installation_required, rd_only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( repository_dependency ) - if suc.tool_shed_is_this_tool_shed( rd_toolshed ): - repository = suc.get_repository_by_name_and_owner( app, rd_name, rd_owner ) - if repository: - repository_metadata = \ - metadata_util.get_repository_metadata_by_repository_id_changeset_revision( app, - app.security.encode_id( repository.id ), - rd_changeset_revision ) - if repository_metadata: - # The repository changeset_revision is installable, so no updates are available. - new_key_rd_dict = {} - new_key_rd_dict[ key ] = repository_dependency - updated_key_rd_dicts.append( key_rd_dict ) - else: - # The repository changeset_revision is no longer installable, so see if there's been an update. - repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) - changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, rd_changeset_revision ) - repository_metadata = \ - metadata_util.get_repository_metadata_by_repository_id_changeset_revision( app, - app.security.encode_id( repository.id ), - changeset_revision ) - if repository_metadata: - new_key_rd_dict = {} - new_key_rd_dict[ key ] = \ - [ rd_toolshed, rd_name, rd_owner, repository_metadata.changeset_revision, rd_prior_installation_required, rd_only_if_compiling_contained_td ] - # We have the updated changset revision. - updated_key_rd_dicts.append( new_key_rd_dict ) - else: - repository_components_tuple = container_util.get_components_from_key( key ) - components_list = suc.extract_components_from_tuple( repository_components_tuple ) - toolshed, repository_name, repository_owner, repository_changeset_revision = components_list[ 0:4 ] - # For backward compatibility to the 12/20/12 Galaxy release. - if len( components_list ) == 4: - prior_installation_required = 'False' - rd_only_if_compiling_contained_td = 'False' - elif len( components_list ) == 5: - rd_only_if_compiling_contained_td = 'False' - message = "The revision %s defined for repository %s owned by %s is invalid, so repository dependencies defined for repository %s will be ignored." % \ - ( str( rd_changeset_revision ), str( rd_name ), str( rd_owner ), str( repository_name ) ) - log.debug( message ) - else: - repository_components_tuple = container_util.get_components_from_key( key ) - components_list = suc.extract_components_from_tuple( repository_components_tuple ) - toolshed, repository_name, repository_owner, repository_changeset_revision = components_list[ 0:4 ] - message = "The revision %s defined for repository %s owned by %s is invalid, so repository dependencies defined for repository %s will be ignored." % \ - ( str( rd_changeset_revision ), str( rd_name ), str( rd_owner ), str( repository_name ) ) - log.debug( message ) - return updated_key_rd_dicts - -def handle_circular_repository_dependency( repository_key, repository_dependency, circular_repository_dependencies, - handled_key_rd_dicts, all_repository_dependencies ): - all_repository_dependencies_root_key = all_repository_dependencies[ 'root_key' ] - repository_dependency_as_key = get_repository_dependency_as_key( repository_dependency ) - repository_key_as_repository_dependency = repository_key.split( container_util.STRSEP ) - update_circular_repository_dependencies( repository_key, - repository_dependency, - all_repository_dependencies[ repository_dependency_as_key ], - circular_repository_dependencies ) - if all_repository_dependencies_root_key != repository_dependency_as_key: - all_repository_dependencies[ repository_key ] = [ repository_dependency ] - return circular_repository_dependencies, handled_key_rd_dicts, all_repository_dependencies - -def handle_current_repository_dependency( app, current_repository_key, key_rd_dicts_to_be_processed, all_repository_dependencies, handled_key_rd_dicts, - circular_repository_dependencies ): - current_repository_key_rd_dicts = [] - for rd in all_repository_dependencies[ current_repository_key ]: - rd_copy = [ str( item ) for item in rd ] - new_key_rd_dict = {} - new_key_rd_dict[ current_repository_key ] = rd_copy - current_repository_key_rd_dicts.append( new_key_rd_dict ) - if current_repository_key_rd_dicts: - toolshed, required_repository, required_repository_metadata, repository_key_rd_dicts, key_rd_dicts_to_be_processed, handled_key_rd_dicts = \ - handle_key_rd_dicts_for_repository( app, - current_repository_key, - current_repository_key_rd_dicts, - key_rd_dicts_to_be_processed, - handled_key_rd_dicts, - circular_repository_dependencies ) - return get_repository_dependencies_for_changeset_revision( app=app, - repository=required_repository, - repository_metadata=required_repository_metadata, - toolshed_base_url=toolshed, - key_rd_dicts_to_be_processed=key_rd_dicts_to_be_processed, - all_repository_dependencies=all_repository_dependencies, - handled_key_rd_dicts=handled_key_rd_dicts, - circular_repository_dependencies=circular_repository_dependencies ) - -def handle_key_rd_dicts_for_repository( app, current_repository_key, repository_key_rd_dicts, key_rd_dicts_to_be_processed, handled_key_rd_dicts, circular_repository_dependencies ): - key_rd_dict = repository_key_rd_dicts.pop( 0 ) - repository_dependency = key_rd_dict[ current_repository_key ] - toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( repository_dependency ) - if suc.tool_shed_is_this_tool_shed( toolshed ): - required_repository = suc.get_repository_by_name_and_owner( app, name, owner ) - required_repository_metadata = \ - metadata_util.get_repository_metadata_by_repository_id_changeset_revision( app, - app.security.encode_id( required_repository.id ), - changeset_revision ) - if required_repository_metadata: - # The required_repository_metadata changeset_revision is installable. - required_metadata = required_repository_metadata.metadata - if required_metadata: - for current_repository_key_rd_dict in repository_key_rd_dicts: - if not in_key_rd_dicts( current_repository_key_rd_dict, key_rd_dicts_to_be_processed ): - key_rd_dicts_to_be_processed.append( current_repository_key_rd_dict ) - # Mark the current repository_dependency as handled_key_rd_dicts. - if not in_key_rd_dicts( key_rd_dict, handled_key_rd_dicts ): - handled_key_rd_dicts.append( key_rd_dict ) - # Remove the current repository from the list of repository_dependencies to be processed. - if in_key_rd_dicts( key_rd_dict, key_rd_dicts_to_be_processed ): - key_rd_dicts_to_be_processed = remove_from_key_rd_dicts( key_rd_dict, key_rd_dicts_to_be_processed ) - else: - # The repository is in a different tool shed, so build an url and send a request. - error_message = "Repository dependencies are currently supported only within the same Tool Shed. Ignoring repository dependency definition " - error_message += "for tool shed %s, name %s, owner %s, changeset revision %s" % ( toolshed, name, owner, changeset_revision ) - log.debug( error_message ) - return toolshed, required_repository, required_repository_metadata, repository_key_rd_dicts, key_rd_dicts_to_be_processed, handled_key_rd_dicts - -def handle_next_repository_dependency( app, key_rd_dicts_to_be_processed, all_repository_dependencies, handled_key_rd_dicts, - circular_repository_dependencies ): - next_repository_key_rd_dict = key_rd_dicts_to_be_processed.pop( 0 ) - next_repository_key_rd_dicts = [ next_repository_key_rd_dict ] - next_repository_key = next_repository_key_rd_dict.keys()[ 0 ] - toolshed, required_repository, required_repository_metadata, repository_key_rd_dicts, key_rd_dicts_to_be_processed, handled_key_rd_dicts = \ - handle_key_rd_dicts_for_repository( app, - next_repository_key, - next_repository_key_rd_dicts, - key_rd_dicts_to_be_processed, - handled_key_rd_dicts, - circular_repository_dependencies ) - return get_repository_dependencies_for_changeset_revision( app=app, - repository=required_repository, - repository_metadata=required_repository_metadata, - toolshed_base_url=toolshed, - key_rd_dicts_to_be_processed=key_rd_dicts_to_be_processed, - all_repository_dependencies=all_repository_dependencies, - handled_key_rd_dicts=handled_key_rd_dicts, - circular_repository_dependencies=circular_repository_dependencies ) - -def in_all_repository_dependencies( repository_key, repository_dependency, all_repository_dependencies ): - """Return True if { repository_key : repository_dependency } is in all_repository_dependencies.""" - for key, val in all_repository_dependencies.items(): - if key != repository_key: - continue - if repository_dependency in val: - return True - return False - -def in_circular_repository_dependencies( repository_key_rd_dict, circular_repository_dependencies ): - """ - Return True if any combination of a circular dependency tuple is the key : value pair defined - in the received repository_key_rd_dict. This means that each circular dependency tuple is converted - into the key : value pair for comparison. - """ - for tup in circular_repository_dependencies: - rd_0, rd_1 = tup - rd_0_as_key = get_repository_dependency_as_key( rd_0 ) - rd_1_as_key = get_repository_dependency_as_key( rd_1 ) - if rd_0_as_key in repository_key_rd_dict and repository_key_rd_dict[ rd_0_as_key ] == rd_1: - return True - if rd_1_as_key in repository_key_rd_dict and repository_key_rd_dict[ rd_1_as_key ] == rd_0: - return True - return False - -def initialize_all_repository_dependencies( current_repository_key, repository_dependencies_dict, all_repository_dependencies ): - """Initialize the all_repository_dependencies dictionary.""" - # It's safe to assume that current_repository_key in this case will have a value. - all_repository_dependencies[ 'root_key' ] = current_repository_key - all_repository_dependencies[ current_repository_key ] = [] - # Store the value of the 'description' key only once, the first time through this recursive method. - description = repository_dependencies_dict.get( 'description', None ) - all_repository_dependencies[ 'description' ] = description - return all_repository_dependencies - -def in_key_rd_dicts( key_rd_dict, key_rd_dicts ): - """Return True if key_rd_dict is contained in the list of key_rd_dicts.""" - k = key_rd_dict.keys()[ 0 ] - v = key_rd_dict[ k ] - for key_rd_dict in key_rd_dicts: - for key, val in key_rd_dict.items(): - if key == k and val == v: - return True - return False - -def is_circular_repository_dependency( repository_key, repository_dependency, all_repository_dependencies ): - """ - Return True if the received repository_dependency is a key in all_repository_dependencies whose list of repository dependencies - includes the received repository_key. - """ - repository_dependency_as_key = get_repository_dependency_as_key( repository_dependency ) - repository_key_as_repository_dependency = repository_key.split( container_util.STRSEP ) - for key, val in all_repository_dependencies.items(): - if key != repository_dependency_as_key: - continue - if repository_key_as_repository_dependency in val: - return True - return False - -def filter_only_if_compiling_contained_td( key_rd_dict ): - """ - Return a copy of the received key_rd_dict with repository dependencies that are needed - only_if_compiling_contained_td filtered out of the list of repository dependencies for - each rd_key. - """ - filtered_key_rd_dict = {} - for rd_key, required_rd_tup in key_rd_dict.items(): - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( required_rd_tup ) - if not asbool( only_if_compiling_contained_td ): - filtered_key_rd_dict[ rd_key ] = required_rd_tup - return filtered_key_rd_dict - -def populate_repository_dependency_objects_for_processing( app, current_repository_key, repository_dependencies_dict, - key_rd_dicts_to_be_processed, handled_key_rd_dicts, - circular_repository_dependencies, all_repository_dependencies ): - """ - The process that discovers all repository dependencies for a specified repository's - changeset revision uses this method to populate the following items for the current - processing loop: filtered_current_repository_key_rd_dicts, key_rd_dicts_to_be_processed, - handled_key_rd_dicts, all_repository_dependencies. Each processing loop may discover - more repository dependencies, so this method is repeatedly called until all repository - dependencies have been discovered. - """ - current_repository_key_rd_dicts = [] - filtered_current_repository_key_rd_dicts = [] - for rd_tup in repository_dependencies_dict[ 'repository_dependencies' ]: - new_key_rd_dict = {} - new_key_rd_dict[ current_repository_key ] = rd_tup - current_repository_key_rd_dicts.append( new_key_rd_dict ) - if current_repository_key_rd_dicts and current_repository_key: - # Remove all repository dependencies that point to a revision within its own repository. - current_repository_key_rd_dicts = remove_ropository_dependency_reference_to_self( current_repository_key_rd_dicts ) - current_repository_key_rd_dicts = \ - get_updated_changeset_revisions_for_repository_dependencies( app, current_repository_key_rd_dicts ) - for key_rd_dict in current_repository_key_rd_dicts: - # Filter out repository dependencies that are required only if compiling the dependent repository's tool dependency. - key_rd_dict = filter_only_if_compiling_contained_td( key_rd_dict ) - if key_rd_dict: - is_circular = False - if not in_key_rd_dicts( key_rd_dict, handled_key_rd_dicts ) and not in_key_rd_dicts( key_rd_dict, - key_rd_dicts_to_be_processed ): - filtered_current_repository_key_rd_dicts.append( key_rd_dict ) - repository_dependency = key_rd_dict[ current_repository_key ] - if current_repository_key in all_repository_dependencies: - # Add all repository dependencies for the current repository into its entry in all_repository_dependencies. - all_repository_dependencies_val = all_repository_dependencies[ current_repository_key ] - if repository_dependency not in all_repository_dependencies_val: - all_repository_dependencies_val.append( repository_dependency ) - all_repository_dependencies[ current_repository_key ] = all_repository_dependencies_val - elif not in_all_repository_dependencies( current_repository_key, repository_dependency, all_repository_dependencies ): - # Handle circular repository dependencies. - if is_circular_repository_dependency( current_repository_key, - repository_dependency, - all_repository_dependencies ): - is_circular = True - circular_repository_dependencies, handled_key_rd_dicts, all_repository_dependencies = \ - handle_circular_repository_dependency( current_repository_key, - repository_dependency, - circular_repository_dependencies, - handled_key_rd_dicts, - all_repository_dependencies ) - else: - all_repository_dependencies[ current_repository_key ] = [ repository_dependency ] - if not is_circular and can_add_to_key_rd_dicts( key_rd_dict, key_rd_dicts_to_be_processed ): - new_key_rd_dict = {} - new_key_rd_dict[ current_repository_key ] = repository_dependency - key_rd_dicts_to_be_processed.append( new_key_rd_dict ) - return filtered_current_repository_key_rd_dicts, key_rd_dicts_to_be_processed, handled_key_rd_dicts, all_repository_dependencies - -def prune_invalid_repository_dependencies( repository_dependencies ): - """ - Eliminate all invalid entries in the received repository_dependencies dictionary. An entry - is invalid if the value_list of the key/value pair is empty. This occurs when an invalid - combination of tool shed, name , owner, changeset_revision is used and a repository_metadata - record is not found. - """ - valid_repository_dependencies = {} - description = repository_dependencies.get( 'description', None ) - root_key = repository_dependencies.get( 'root_key', None ) - if root_key is None: - return valid_repository_dependencies - for key, value in repository_dependencies.items(): - if key in [ 'description', 'root_key' ]: - continue - if value: - valid_repository_dependencies[ key ] = value - if valid_repository_dependencies: - valid_repository_dependencies[ 'description' ] = description - valid_repository_dependencies[ 'root_key' ] = root_key - return valid_repository_dependencies - -def remove_from_key_rd_dicts( key_rd_dict, key_rd_dicts ): - """Eliminate the key_rd_dict from the list of key_rd_dicts if it is contained in the list.""" - k = key_rd_dict.keys()[ 0 ] - v = key_rd_dict[ k ] - clean_key_rd_dicts = [] - for krd_dict in key_rd_dicts: - key = krd_dict.keys()[ 0 ] - val = krd_dict[ key ] - if key == k and val == v: - continue - clean_key_rd_dicts.append( krd_dict ) - return clean_key_rd_dicts - -def remove_ropository_dependency_reference_to_self( key_rd_dicts ): - """Remove all repository dependencies that point to a revision within its own repository.""" - clean_key_rd_dicts = [] - key = key_rd_dicts[ 0 ].keys()[ 0 ] - repository_tup = key.split( container_util.STRSEP ) - rd_toolshed, rd_name, rd_owner, rd_changeset_revision, rd_prior_installation_required, rd_only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( repository_tup ) - cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed ) - for key_rd_dict in key_rd_dicts: - k = key_rd_dict.keys()[ 0 ] - repository_dependency = key_rd_dict[ k ] - toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( repository_dependency ) - cleaned_toolshed = common_util.remove_protocol_from_tool_shed_url( toolshed ) - if cleaned_rd_toolshed == cleaned_toolshed and rd_name == name and rd_owner == owner: - debug_msg = "Removing repository dependency for repository %s owned by %s " % ( name, owner ) - debug_msg += 'since it refers to a revision within itself.' - log.debug( debug_msg ) - else: - new_key_rd_dict = {} - new_key_rd_dict[ key ] = repository_dependency - clean_key_rd_dicts.append( new_key_rd_dict ) - return clean_key_rd_dicts - -def update_circular_repository_dependencies( repository_key, repository_dependency, repository_dependencies, - circular_repository_dependencies ): - repository_dependency_as_key = get_repository_dependency_as_key( repository_dependency ) - repository_key_as_repository_dependency = repository_key.split( container_util.STRSEP ) - if repository_key_as_repository_dependency in repository_dependencies: - found = False - for tup in circular_repository_dependencies: - if repository_dependency in tup and repository_key_as_repository_dependency in tup: - # The circular dependency has already been included. - found = True - if not found: - new_circular_tup = [ repository_dependency, repository_key_as_repository_dependency ] - circular_repository_dependencies.append( new_circular_tup ) - return circular_repository_dependencies diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/tool_shed/util/repository_maintenance_util.py --- a/lib/tool_shed/util/repository_maintenance_util.py +++ b/lib/tool_shed/util/repository_maintenance_util.py @@ -2,10 +2,10 @@ import logging import os import re -import tool_shed.util.shed_util_common as suc +from tool_shed.dependencies.repository import relation_builder from tool_shed.util import common_util from tool_shed.util import hg_util -from tool_shed.util import repository_dependency_util +from tool_shed.util import shed_util_common as suc from galaxy import util from galaxy import web from galaxy.web.form_builder import build_select_field @@ -90,16 +90,9 @@ metadata = repository_metadata.metadata if metadata: tool_shed_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) + rb = relation_builder.RelationBuilder( app, repository, repository_metadata, tool_shed_url ) # Get a dictionary of all repositories upon which the contents of the received repository depends. - repository_dependencies = \ - repository_dependency_util.get_repository_dependencies_for_changeset_revision( app=app, - repository=repository, - repository_metadata=repository_metadata, - toolshed_base_url=tool_shed_url, - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None, - circular_repository_dependencies=None ) + repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() tool_dependencies = metadata.get( 'tool_dependencies', {} ) if tool_dependencies: new_tool_dependencies = {} Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.