1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c4a97053aca3/ Changeset: c4a97053aca3 User: greg Date: 2014-01-03 20:21:29 Summary: Raise an exception if attempting to install a repository into Galaxy that contains an invalid repository dependency definition (implying a bug in the Tool Shed framework). Affected #: 5 files diff -r 10b7c04f919ff6d7fc1d03ba88ebb282645fcbba -r c4a97053aca397ca783079a1e05c0cd2f921fb88 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -1440,16 +1440,17 @@ tool_path, relative_install_dir = repository.get_tool_relative_path( trans.app ) if relative_install_dir: original_metadata_dict = repository.metadata - metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=trans.app, - repository=repository, - changeset_revision=repository.changeset_revision, - repository_clone_url=repository_clone_url, - shed_config_dict = repository.get_shed_config_dict( trans.app ), - relative_install_dir=relative_install_dir, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=False, - persist=False ) + metadata_dict, invalid_file_tups = \ + metadata_util.generate_metadata_for_changeset_revision( app=trans.app, + repository=repository, + changeset_revision=repository.changeset_revision, + repository_clone_url=repository_clone_url, + shed_config_dict = repository.get_shed_config_dict( trans.app ), + relative_install_dir=relative_install_dir, + repository_files_dir=None, + resetting_all_metadata_on_repository=False, + updating_installed_repository=False, + persist=False ) repository.metadata = metadata_dict if metadata_dict != original_metadata_dict: suc.update_in_shed_tool_config( trans.app, repository ) @@ -1607,7 +1608,11 @@ changeset_revision = kwd.get( 'changeset_revision', None ) latest_changeset_revision = kwd.get( 'latest_changeset_revision', None ) latest_ctx_rev = kwd.get( 'latest_ctx_rev', None ) - repository = suc.get_tool_shed_repository_by_shed_name_owner_changeset_revision( trans.app, tool_shed_url, name, owner, changeset_revision ) + repository = suc.get_tool_shed_repository_by_shed_name_owner_changeset_revision( trans.app, + tool_shed_url, + name, + owner, + changeset_revision ) if changeset_revision and latest_changeset_revision and latest_ctx_rev: if changeset_revision == latest_changeset_revision: message = "The installed repository named '%s' is current, there are no updates available. " % name @@ -1627,16 +1632,17 @@ if repository.includes_data_managers: data_manager_util.remove_from_data_manager( trans.app, repository ) # Update the repository metadata. - metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=trans.app, - repository=repository, - changeset_revision=latest_changeset_revision, - repository_clone_url=repository_clone_url, - shed_config_dict = repository.get_shed_config_dict( trans.app ), - relative_install_dir=relative_install_dir, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=True, - persist=True ) + metadata_dict, invalid_file_tups = \ + metadata_util.generate_metadata_for_changeset_revision( app=trans.app, + repository=repository, + changeset_revision=latest_changeset_revision, + repository_clone_url=repository_clone_url, + shed_config_dict=repository.get_shed_config_dict( trans.app ), + relative_install_dir=relative_install_dir, + repository_files_dir=None, + resetting_all_metadata_on_repository=False, + updating_installed_repository=True, + persist=True ) repository.metadata = metadata_dict # Update the repository.changeset_revision column in the database. repository.changeset_revision = latest_changeset_revision @@ -1674,10 +1680,15 @@ repository_tools_tups ) # Create tool_dependency records if necessary. if 'tool_dependencies' in metadata_dict: - tool_dependencies = tool_dependency_util.create_tool_dependency_objects( trans.app, repository, relative_install_dir, set_status=False ) - message = "The installed repository named '%s' has been updated to change set revision '%s'. " % ( name, latest_changeset_revision ) + tool_dependencies = tool_dependency_util.create_tool_dependency_objects( trans.app, + repository, + relative_install_dir, + set_status=False ) + message = "The installed repository named '%s' has been updated to change set revision '%s'. " % \ + ( name, latest_changeset_revision ) # See if any tool dependencies can be installed. - shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) + shed_tool_conf, tool_path, relative_install_dir = \ + suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) if repository.missing_tool_dependencies: message += "Click the name of one of the missing tool dependencies listed below to install tool dependencies." else: diff -r 10b7c04f919ff6d7fc1d03ba88ebb282645fcbba -r c4a97053aca397ca783079a1e05c0cd2f921fb88 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -330,12 +330,13 @@ tool_panel_elems.append( elem ) return tool_panel_elems - def handle_repository_contents( self, tool_shed_repository, repository_clone_url, relative_install_dir, repository_elem, install_dependencies, is_repository_dependency=False ): + def handle_repository_contents( self, tool_shed_repository, repository_clone_url, relative_install_dir, repository_elem, + install_dependencies, is_repository_dependency=False ): """ - Generate the metadata for the installed tool shed repository, among other things. If the installed tool_shed_repository contains tools - that are loaded into the Galaxy tool panel, this method will automatically eliminate all entries for each of the tools defined in the - received repository_elem from all non-shed-related tool panel configuration files since the entries are automatically added to the reserved - migrated_tools_conf.xml file as part of the migration process. + Generate the metadata for the installed tool shed repository, among other things. If the installed tool_shed_repository + contains tools that are loaded into the Galaxy tool panel, this method will automatically eliminate all entries for each + of the tools defined in the received repository_elem from all non-shed-related tool panel configuration files since the + entries are automatically added to the reserved migrated_tools_conf.xml file as part of the migration process. """ tool_configs_to_filter = [] tool_panel_dict_for_display = odict() @@ -351,14 +352,16 @@ # See if tool_config is defined inside of a section in self.proprietary_tool_panel_elems. is_displayed, tool_sections = self.get_containing_tool_sections( tool_config ) if is_displayed: - tool_panel_dict_for_tool_config = tool_util.generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=tool_sections ) + tool_panel_dict_for_tool_config = \ + tool_util.generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=tool_sections ) # The tool-panel_dict has the following structure. - # {<Tool guid> : [{ tool_config : <tool_config_file>, id: <ToolSection id>, version : <ToolSection version>, name : <TooSection name>}]} + # {<Tool guid> : [{ tool_config : <tool_config_file>, id: <ToolSection id>, version : <ToolSection version>, + # name : <TooSection name>}]} for k, v in tool_panel_dict_for_tool_config.items(): tool_panel_dict_for_display[ k ] = v for tool_panel_dict in v: - # Keep track of tool config file names associated with entries that have been made to the migrated_tools_conf.xml file so - # they can be eliminated from all non-shed-related tool panel configs. + # Keep track of tool config file names associated with entries that have been made to the + # migrated_tools_conf.xml file so they can be eliminated from all non-shed-related tool panel configs. tool_config_file = tool_panel_dict.get( 'tool_config', None ) if tool_config_file: if tool_config_file not in tool_configs_to_filter: @@ -375,23 +378,28 @@ log.exception( "Exception attempting to filter and persist non-shed-related tool panel configs:\n%s" % str( e ) ) finally: lock.release() - metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=self.app, - repository=tool_shed_repository, - changeset_revision=tool_shed_repository.changeset_revision, - repository_clone_url=repository_clone_url, - shed_config_dict = self.shed_config_dict, - relative_install_dir=relative_install_dir, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=False, - persist=True ) + metadata_dict, invalid_file_tups = \ + metadata_util.generate_metadata_for_changeset_revision( app=self.app, + repository=tool_shed_repository, + changeset_revision=tool_shed_repository.changeset_revision, + repository_clone_url=repository_clone_url, + shed_config_dict = self.shed_config_dict, + relative_install_dir=relative_install_dir, + repository_files_dir=None, + resetting_all_metadata_on_repository=False, + updating_installed_repository=False, + persist=True ) tool_shed_repository.metadata = metadata_dict self.app.install_model.context.add( tool_shed_repository ) self.app.install_model.context.flush() has_tool_dependencies = self.__has_tool_dependencies( metadata_dict ) if has_tool_dependencies: - # All tool_dependency objects must be created before the tools are processed even if no tool dependencies will be installed. - tool_dependencies = tool_dependency_util.create_tool_dependency_objects( self.app, tool_shed_repository, relative_install_dir, set_status=True ) + # All tool_dependency objects must be created before the tools are processed even if no + # tool dependencies will be installed. + tool_dependencies = tool_dependency_util.create_tool_dependency_objects( self.app, + tool_shed_repository, + relative_install_dir, + set_status=True ) else: tool_dependencies = None if 'tools' in metadata_dict: diff -r 10b7c04f919ff6d7fc1d03ba88ebb282645fcbba -r c4a97053aca397ca783079a1e05c0cd2f921fb88 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -339,23 +339,24 @@ changeset_revision_dict[ 'ctx_rev' ] = None return changeset_revision_dict -def handle_repository_contents( trans, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir, tool_shed=None, tool_section=None, shed_tool_conf=None, - reinstalling=False ): +def handle_repository_contents( trans, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir, + tool_shed=None, tool_section=None, shed_tool_conf=None, reinstalling=False ): """ - Generate the metadata for the installed tool shed repository, among other things. This method is called from Galaxy (never the tool shed) - when an administrator is installing a new repository or reinstalling an uninstalled repository. + Generate the metadata for the installed tool shed repository, among other things. This method is called from Galaxy + (never the tool shed) when an administrator is installing a new repository or reinstalling an uninstalled repository. """ shed_config_dict = trans.app.toolbox.get_shed_config_dict_by_filename( shed_tool_conf ) - metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=trans.app, - repository=tool_shed_repository, - changeset_revision=tool_shed_repository.changeset_revision, - repository_clone_url=repository_clone_url, - shed_config_dict=shed_config_dict, - relative_install_dir=relative_install_dir, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=False, - persist=True ) + metadata_dict, invalid_file_tups = \ + metadata_util.generate_metadata_for_changeset_revision( app=trans.app, + repository=tool_shed_repository, + changeset_revision=tool_shed_repository.changeset_revision, + repository_clone_url=repository_clone_url, + shed_config_dict=shed_config_dict, + relative_install_dir=relative_install_dir, + repository_files_dir=None, + resetting_all_metadata_on_repository=False, + updating_installed_repository=False, + persist=True ) tool_shed_repository.metadata = metadata_dict # Update the tool_shed_repository.tool_shed_status column in the database. tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( trans.app, tool_shed_repository ) @@ -364,13 +365,20 @@ trans.install_model.context.add( tool_shed_repository ) trans.install_model.context.flush() if 'tool_dependencies' in metadata_dict and not reinstalling: - tool_dependencies = tool_dependency_util.create_tool_dependency_objects( trans.app, tool_shed_repository, relative_install_dir, set_status=True ) + tool_dependencies = tool_dependency_util.create_tool_dependency_objects( trans.app, + tool_shed_repository, + relative_install_dir, + set_status=True ) if 'sample_files' in metadata_dict: sample_files = metadata_dict.get( 'sample_files', [] ) tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files ) - tool_data_table_conf_filename, tool_data_table_elems = tool_util.install_tool_data_tables( trans.app, tool_shed_repository, tool_index_sample_files ) + tool_data_table_conf_filename, tool_data_table_elems = \ + tool_util.install_tool_data_tables( trans.app, tool_shed_repository, tool_index_sample_files ) if tool_data_table_elems: - trans.app.tool_data_tables.add_new_entries_from_config_file( tool_data_table_conf_filename, None, trans.app.config.shed_tool_data_table_config, persist=True ) + trans.app.tool_data_tables.add_new_entries_from_config_file( tool_data_table_conf_filename, + None, + trans.app.config.shed_tool_data_table_config, + persist=True ) if 'tools' in metadata_dict: tool_panel_dict = tool_util.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) sample_files = metadata_dict.get( 'sample_files', [] ) @@ -380,10 +388,19 @@ repository_tools_tups = suc.get_repository_tools_tups( trans.app, metadata_dict ) if repository_tools_tups: # Handle missing data table entries for tool parameters that are dynamically generated select lists. - repository_tools_tups = tool_util.handle_missing_data_table_entry( trans.app, relative_install_dir, tool_path, repository_tools_tups ) + repository_tools_tups = tool_util.handle_missing_data_table_entry( trans.app, + relative_install_dir, + tool_path, + repository_tools_tups ) # Handle missing index files for tool parameters that are dynamically generated select lists. - repository_tools_tups, sample_files_copied = tool_util.handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups, sample_files_copied ) - # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. + repository_tools_tups, sample_files_copied = \ + tool_util.handle_missing_index_file( trans.app, + tool_path, + sample_files, + repository_tools_tups, + sample_files_copied ) + # Copy remaining sample files included in the repository to the ~/tool-data directory of the + # local Galaxy instance. tool_util.copy_sample_files( trans.app, sample_files, tool_path=tool_path, sample_files_copied=sample_files_copied ) tool_util.add_to_tool_panel( app=trans.app, repository_name=tool_shed_repository.name, @@ -413,16 +430,18 @@ files_dir = os.path.join( shed_config_dict[ 'tool_path' ], files_dir ) datatypes_config = suc.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, files_dir ) # Load data types required by tools. - converter_path, display_path = datatype_util.alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, files_dir, override=False ) + converter_path, display_path = \ + datatype_util.alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, files_dir, override=False ) if converter_path or display_path: # Create a dictionary of tool shed repository related information. - repository_dict = datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, - name=tool_shed_repository.name, - owner=tool_shed_repository.owner, - installed_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dicts=metadata_dict.get( 'tools', [] ), - converter_path=converter_path, - display_path=display_path ) + repository_dict = \ + datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, + name=tool_shed_repository.name, + owner=tool_shed_repository.owner, + installed_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dicts=metadata_dict.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) if converter_path: # Load proprietary datatype converters trans.app.datatypes_registry.load_datatype_converters( trans.app.toolbox, installed_repository_dict=repository_dict ) diff -r 10b7c04f919ff6d7fc1d03ba88ebb282645fcbba -r c4a97053aca397ca783079a1e05c0cd2f921fb88 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -568,23 +568,25 @@ tmp_url = suc.clean_repository_clone_url( repository_clone_url ) return '%s/%s/%s/%s' % ( tmp_url, guid_type, obj_id, version ) -def generate_metadata_for_changeset_revision( app, repository, changeset_revision, repository_clone_url, shed_config_dict=None, relative_install_dir=None, - repository_files_dir=None, resetting_all_metadata_on_repository=False, updating_installed_repository=False, +def generate_metadata_for_changeset_revision( app, repository, changeset_revision, repository_clone_url, + shed_config_dict=None, relative_install_dir=None, repository_files_dir=None, + resetting_all_metadata_on_repository=False, updating_installed_repository=False, persist=False ): """ - Generate metadata for a repository using it's files on disk. To generate metadata for changeset revisions older than the repository tip, - the repository will have been cloned to a temporary location and updated to a specified changeset revision to access that changeset revision's - disk files, so the value of repository_files_dir will not always be repository.repo_path( app ) (it could be an absolute path to a temporary - directory containing a clone). If it is an absolute path, the value of relative_install_dir must contain repository.repo_path( app ). + Generate metadata for a repository using it's files on disk. To generate metadata for changeset revisions older than + the repository tip, the repository will have been cloned to a temporary location and updated to a specified changeset + revision to access that changeset revision's disk files, so the value of repository_files_dir will not always be + repository.repo_path( app ) (it could be an absolute path to a temporary directory containing a clone). If it is an + absolute path, the value of relative_install_dir must contain repository.repo_path( app ). - The value of persist will be True when the installed repository contains a valid tool_data_table_conf.xml.sample file, in which case the entries - should ultimately be persisted to the file referred to by app.config.shed_tool_data_table_config. + The value of persist will be True when the installed repository contains a valid tool_data_table_conf.xml.sample file, + in which case the entries should ultimately be persisted to the file referred to by app.config.shed_tool_data_table_config. """ if shed_config_dict is None: shed_config_dict = {} if updating_installed_repository: - # Keep the original tool shed repository metadata if setting metadata on a repository installed into a local Galaxy instance for which - # we have pulled updates. + # Keep the original tool shed repository metadata if setting metadata on a repository installed into a local Galaxy + # instance for which we have pulled updates. original_repository_metadata = repository.metadata else: original_repository_metadata = None @@ -603,11 +605,12 @@ if resetting_all_metadata_on_repository: if not relative_install_dir: raise Exception( "The value of repository.repo_path( app ) must be sent when resetting all metadata on a repository." ) - # Keep track of the location where the repository is temporarily cloned so that we can strip the path when setting metadata. The value of - # repository_files_dir is the full path to the temporary directory to which the repository was cloned. + # Keep track of the location where the repository is temporarily cloned so that we can strip the path when setting metadata. + # The value of repository_files_dir is the full path to the temporary directory to which the repository was cloned. work_dir = repository_files_dir files_dir = repository_files_dir - # Since we're working from a temporary directory, we can safely copy sample files included in the repository to the repository root. + # Since we're working from a temporary directory, we can safely copy sample files included in the repository to the repository + # root. app.config.tool_data_path = repository_files_dir app.config.tool_data_table_config_path = repository_files_dir else: @@ -624,10 +627,11 @@ if datatypes_config: metadata_dict = generate_datatypes_metadata( app, repository, repository_clone_url, files_dir, datatypes_config, metadata_dict ) # Get the relative path to all sample files included in the repository for storage in the repository's metadata. - sample_file_metadata_paths, sample_file_copy_paths = get_sample_files_from_disk( repository_files_dir=files_dir, - tool_path=shed_config_dict.get( 'tool_path' ), - relative_install_dir=relative_install_dir, - resetting_all_metadata_on_repository=resetting_all_metadata_on_repository ) + sample_file_metadata_paths, sample_file_copy_paths = \ + get_sample_files_from_disk( repository_files_dir=files_dir, + tool_path=shed_config_dict.get( 'tool_path' ), + relative_install_dir=relative_install_dir, + resetting_all_metadata_on_repository=resetting_all_metadata_on_repository ) if sample_file_metadata_paths: metadata_dict[ 'sample_files' ] = sample_file_metadata_paths # Copy all sample files included in the repository to a single directory location so we can load tools that depend on them. @@ -636,10 +640,11 @@ # If the list of sample files includes a tool_data_table_conf.xml.sample file, laad it's table elements into memory. relative_path, filename = os.path.split( sample_file ) if filename == 'tool_data_table_conf.xml.sample': - new_table_elems, error_message = app.tool_data_tables.add_new_entries_from_config_file( config_filename=sample_file, - tool_data_path=app.config.tool_data_path, - shed_tool_data_table_config=app.config.shed_tool_data_table_config, - persist=False ) + new_table_elems, error_message = \ + app.tool_data_tables.add_new_entries_from_config_file( config_filename=sample_file, + tool_data_path=app.config.tool_data_path, + shed_tool_data_table_config=app.config.shed_tool_data_table_config, + persist=False ) if error_message: invalid_file_tups.append( ( filename, error_message ) ) for root, dirs, files in os.walk( files_dir ): @@ -650,7 +655,9 @@ # See if we have a repository dependencies defined. if name == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: path_to_repository_dependencies_config = os.path.join( root, name ) - metadata_dict, error_message = generate_repository_dependency_metadata( app, path_to_repository_dependencies_config, metadata_dict ) + metadata_dict, error_message = generate_repository_dependency_metadata( app, + path_to_repository_dependencies_config, + metadata_dict ) if error_message: invalid_file_tups.append( ( name, error_message ) ) # See if we have one or more READ_ME files. @@ -666,8 +673,11 @@ elif name not in NOT_TOOL_CONFIGS and name.endswith( '.xml' ): full_path = str( os.path.abspath( os.path.join( root, name ) ) ) if os.path.getsize( full_path ) > 0: - if not ( checkers.check_binary( full_path ) or checkers.check_image( full_path ) or checkers.check_gzip( full_path )[ 0 ] - or checkers.check_bz2( full_path )[ 0 ] or checkers.check_zip( full_path ) ): + if not ( checkers.check_binary( full_path ) or + checkers.check_image( full_path ) or + checkers.check_gzip( full_path )[ 0 ] or + checkers.check_bz2( full_path )[ 0 ] or + checkers.check_zip( full_path ) ): # Make sure we're looking at a tool config and not a display application config or something else. element_tree, error_message = xml_util.parse_xml( full_path ) if element_tree is None: @@ -676,13 +686,15 @@ element_tree_root = element_tree.getroot() is_tool = element_tree_root.tag == 'tool' if is_tool: - tool, valid, error_message = tool_util.load_tool_from_config( app, app.security.encode_id( repository.id ), full_path ) + tool, valid, error_message = \ + tool_util.load_tool_from_config( app, app.security.encode_id( repository.id ), full_path ) if tool is None: if not valid: invalid_tool_configs.append( name ) invalid_file_tups.append( ( name, error_message ) ) else: - invalid_files_and_errors_tups = tool_util.check_tool_input_params( app, files_dir, name, tool, sample_file_copy_paths ) + invalid_files_and_errors_tups = \ + tool_util.check_tool_input_params( app, files_dir, name, tool, sample_file_copy_paths ) can_set_metadata = True for tup in invalid_files_and_errors_tups: if name in tup: @@ -690,15 +702,17 @@ invalid_tool_configs.append( name ) break if can_set_metadata: - relative_path_to_tool_config = get_relative_path_to_repository_file( root, - name, - relative_install_dir, - work_dir, - shed_config_dict, - resetting_all_metadata_on_repository ) - - - metadata_dict = generate_tool_metadata( relative_path_to_tool_config, tool, repository_clone_url, metadata_dict ) + relative_path_to_tool_config = \ + get_relative_path_to_repository_file( root, + name, + relative_install_dir, + work_dir, + shed_config_dict, + resetting_all_metadata_on_repository ) + metadata_dict = generate_tool_metadata( relative_path_to_tool_config, + tool, + repository_clone_url, + metadata_dict ) else: for tup in invalid_files_and_errors_tups: invalid_file_tups.append( tup ) @@ -752,8 +766,9 @@ def generate_package_dependency_metadata( app, elem, valid_tool_dependencies_dict, invalid_tool_dependencies_dict ): """ - Generate the metadata for a tool dependencies package defined for a repository. The value of package_name must match the value of the "package" - type in the tool config's <requirements> tag set. This method is called from both Galaxy and the tool shed. + Generate the metadata for a tool dependencies package defined for a repository. The value of package_name must + match the value of the "package" type in the tool config's <requirements> tag set. This method is called from + both Galaxy and the tool shed. """ repository_dependency_is_valid = True repository_dependency_tup = [] @@ -833,22 +848,23 @@ valid_repository_dependencies_dict = dict( description=root.get( 'description' ) ) valid_repository_dependency_tups = [] for repository_elem in root.findall( 'repository' ): - repository_dependency_tup, repository_dependency_is_valid, error_message = handle_repository_elem( app, - repository_elem, - only_if_compiling_contained_td=False ) + repository_dependency_tup, repository_dependency_is_valid, err_msg = \ + handle_repository_elem( app, repository_elem, only_if_compiling_contained_td=False ) if repository_dependency_is_valid: valid_repository_dependency_tups.append( repository_dependency_tup ) else: # Append the error_message to the repository dependencies tuple. - toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = repository_dependency_tup + toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + repository_dependency_tup repository_dependency_tup = ( toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, - error_message ) + err_msg ) invalid_repository_dependency_tups.append( repository_dependency_tup ) + error_message += err_msg if invalid_repository_dependency_tups: invalid_repository_dependencies_dict[ 'repository_dependencies' ] = invalid_repository_dependency_tups metadata_dict[ 'invalid_repository_dependencies' ] = invalid_repository_dependencies_dict @@ -1142,50 +1158,72 @@ def handle_repository_elem( app, repository_elem, only_if_compiling_contained_td=False ): """ - Process the received repository_elem which is a <repository> tag either from a repository_dependencies.xml file or a tool_dependencies.xml file. - If the former, we're generating repository dependencies metadata for a repository in the tool shed. If the latter, we're generating package - dependency metadata within Galaxy or the tool shed. + Process the received repository_elem which is a <repository> tag either from a repository_dependencies.xml + file or a tool_dependencies.xml file. If the former, we're generating repository dependencies metadata for + a repository in the tool shed. If the latter, we're generating package dependency metadata within Galaxy or + the tool shed. """ sa_session = app.model.context.current is_valid = True error_message = '' - toolshed = repository_elem.get( 'toolshed' ) + toolshed = repository_elem.get( 'toolshed', None ) + name = repository_elem.get( 'name', None ) + owner = repository_elem.get( 'owner', None ) + changeset_revision = repository_elem.get( 'changeset_revision', None ) + prior_installation_required = str( repository_elem.get( 'prior_installation_required', False ) ) + if app.name == 'galaxy': + # We're installing a repository into Galaxy, so make sure its contained repository dependency definition + # is valid. + if toolshed is None or name is None or owner is None or changeset_revision is None: + # Raise an exception here instead of returning an error_message to keep the installation from + # proceeding. Reaching here implies a bug in the Tool Shed framework. + error_message = 'Installation halted because the following repository dependency definition is invalid:\n' + error_message += xml_util.xml_to_string( repository_elem, use_indent=True ) + raise Exception( error_message ) if not toolshed: # Default to the current tool shed. toolshed = str( url_for( '/', qualified=True ) ).rstrip( '/' ) + repository_dependency_tup = [ toolshed, + name, + owner, + changeset_revision, + prior_installation_required, + str( only_if_compiling_contained_td ) ] cleaned_toolshed = td_common_util.clean_tool_shed_url( toolshed ) - name = repository_elem.get( 'name' ) - owner = repository_elem.get( 'owner' ) - changeset_revision = repository_elem.get( 'changeset_revision' ) - prior_installation_required = str( repository_elem.get( 'prior_installation_required', False ) ) - repository_dependency_tup = [ toolshed, name, owner, changeset_revision, prior_installation_required, str( only_if_compiling_contained_td ) ] user = None repository = None if app.name == 'galaxy': - # We're in Galaxy. We reach here when we're generating the metadata for a tool dependencies package defined for a repository or when we're - # generating metadata for an installed repository. See if we can locate the installed repository via the changeset_revision defined in the - # repository_elem (it may be outdated). If we're successful in locating an installed repository with the attributes defined in the - # repository_elem, we know it is valid. + # We're in Galaxy. We reach here when we're generating the metadata for a tool dependencies package defined + # for a repository or when we're generating metadata for an installed repository. See if we can locate the + # installed repository via the changeset_revision defined in the repository_elem (it may be outdated). If we're + # successful in locating an installed repository with the attributes defined in the repository_elem, we know it + # is valid. repository = suc.get_repository_for_dependency_relationship( app, cleaned_toolshed, name, owner, changeset_revision ) if repository: return repository_dependency_tup, is_valid, error_message else: - # Send a request to the tool shed to retrieve appropriate additional changeset revisions with which the repository may have been installed. + # Send a request to the tool shed to retrieve appropriate additional changeset revisions with which the repository + # may have been installed. text = install_util.get_updated_changeset_revisions_from_tool_shed( app, toolshed, name, owner, changeset_revision ) if text: updated_changeset_revisions = util.listify( text ) for updated_changeset_revision in updated_changeset_revisions: - repository = suc.get_repository_for_dependency_relationship( app, cleaned_toolshed, name, owner, updated_changeset_revision ) + repository = suc.get_repository_for_dependency_relationship( app, + cleaned_toolshed, + name, + owner, + updated_changeset_revision ) if repository: return repository_dependency_tup, is_valid, error_message - # Don't generate an error message for missing repository dependencies that are required only if compiling the dependent repository's - # tool dependency. + # Don't generate an error message for missing repository dependencies that are required only if compiling the + # dependent repository's tool dependency. if not only_if_compiling_contained_td: - # We'll currently default to setting the repository dependency definition as invalid if an installed repository cannot be found. - # This may not be ideal because the tool shed may have simply been inaccessible when metadata was being generated for the installed - # tool shed repository. - error_message = "Ignoring invalid repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s "% \ - ( toolshed, name, owner, changeset_revision ) + # We'll currently default to setting the repository dependency definition as invalid if an installed repository + # cannot be found. This may not be ideal because the tool shed may have simply been inaccessible when metadata + # was being generated for the installed tool shed repository. + error_message = "Ignoring invalid repository dependency definition for tool shed %s, name %s, owner %s, " % \ + ( toolshed, name, owner ) + error_message += "changeset revision %s." % changeset_revision log.debug( error_message ) is_valid = False return repository_dependency_tup, is_valid, error_message @@ -1197,9 +1235,9 @@ .filter( app.model.User.table.c.username == owner ) \ .one() except Exception, e: - error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s "% \ - ( toolshed, name, owner, changeset_revision ) - error_message += "because the owner is invalid. " + error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, " % \ + ( toolshed, name, owner ) + error_message += "changeset revision %s because the owner is invalid. " % changeset_revision log.debug( error_message ) is_valid = False return repository_dependency_tup, is_valid, error_message @@ -1209,21 +1247,26 @@ app.model.Repository.table.c.user_id == user.id ) ) \ .one() except: - error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s "% \ - ( toolshed, name, owner, changeset_revision ) - error_message += "because the name is invalid. " + error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, " % \ + ( toolshed, name, owner ) + error_message += "changeset revision %s because the name is invalid. " % changeset_revision log.debug( error_message ) is_valid = False return repository_dependency_tup, is_valid, error_message repo = hg.repository( suc.get_configured_ui(), repository.repo_path( app ) ) - # The received changeset_revision may be None since defining it in the dependency definition is optional. If this is the case, - # the default will be to set it's value to the repository dependency tip revision. This probably occurs only when handling - # circular dependency definitions. + # The received changeset_revision may be None since defining it in the dependency definition is optional. + # If this is the case, the default will be to set it's value to the repository dependency tip revision. + # This probably occurs only when handling circular dependency definitions. tip_ctx = repo.changectx( repo.changelog.tip() ) # Make sure the repo.changlog includes at least 1 revision. if changeset_revision is None and tip_ctx.rev() >= 0: changeset_revision = str( tip_ctx ) - repository_dependency_tup = [ toolshed, name, owner, changeset_revision, prior_installation_required, str( only_if_compiling_contained_td ) ] + repository_dependency_tup = [ toolshed, + name, + owner, + changeset_revision, + prior_installation_required, + str( only_if_compiling_contained_td ) ] return repository_dependency_tup, is_valid, error_message else: # Find the specified changeset revision in the repository's changelog to see if it's valid. @@ -1234,16 +1277,17 @@ found = True break if not found: - error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s "% \ - ( toolshed, name, owner, changeset_revision ) - error_message += "because the changeset revision is invalid. " + error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, " % \ + ( toolshed, name, owner ) + error_message += "changeset revision %s because the changeset revision is invalid. " % changeset_revision log.debug( error_message ) is_valid = False return repository_dependency_tup, is_valid, error_message else: # Repository dependencies are currently supported within a single tool shed. - error_message = "Repository dependencies are currently supported only within the same tool shed. Ignoring repository dependency definition " - error_message += "for tool shed %s, name %s, owner %s, changeset revision %s. " % ( toolshed, name, owner, changeset_revision ) + error_message = "Repository dependencies are currently supported only within the same tool shed. Ignoring " + error_message += "repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s. " % \ + ( toolshed, name, owner, changeset_revision ) log.debug( error_message ) is_valid = False return repository_dependency_tup, is_valid, error_message @@ -1811,8 +1855,8 @@ def set_repository_metadata( trans, repository, content_alert_str='', **kwd ): """ - Set metadata using the repository's current disk files, returning specific error messages (if any) to alert the repository owner that the changeset - has problems. + Set metadata using the repository's current disk files, returning specific error messages (if any) to alert the + repository owner that the changeset has problems. """ message = '' status = 'done' diff -r 10b7c04f919ff6d7fc1d03ba88ebb282645fcbba -r c4a97053aca397ca783079a1e05c0cd2f921fb88 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -945,7 +945,10 @@ return has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td def get_repository_for_dependency_relationship( app, tool_shed, name, owner, changeset_revision ): - """Return an installed tool_shed_repository database record that is defined by either the current changeset revision or the installed_changeset_revision.""" + """ + Return an installed tool_shed_repository database record that is defined by either the current changeset + revision or the installed_changeset_revision. + """ # This method is used only in Galaxy, not the tool shed. if tool_shed.endswith( '/' ): tool_shed = tool_shed.rstrip( '/' ) @@ -1330,8 +1333,8 @@ def get_updated_changeset_revisions( trans, name, owner, changeset_revision ): """ - Return a string of comma-separated changeset revision hashes for all available updates to the received changeset revision for the repository - defined by the received name and owner. + Return a string of comma-separated changeset revision hashes for all available updates to the received changeset + revision for the repository defined by the received name and owner. """ repository = get_repository_by_name_and_owner( trans.app, name, owner ) repo_dir = repository.repo_path( trans.app ) @@ -1351,8 +1354,8 @@ def get_url_from_tool_shed( app, tool_shed ): """ - The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is something like: - http://toolshed.g2.bx.psu.edu/ + The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is + something like: http://toolshed.g2.bx.psu.edu/ """ for shed_name, shed_url in app.tool_shed_registry.tool_sheds.items(): if shed_url.find( tool_shed ) >= 0: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.