[galaxy-commits] commit/galaxy-central: greg: Enhance tool shed repository metadata to include information about invalid tool dependencies, invalid repository dependencies and orphan tool dependenccies. Display invalid dependencies in separate containers, and display appropriate warning messages when browsing repositories that contain orphan dependencies.

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ec4da708e451/ changeset: ec4da708e451 user: greg date: 2013-02-15 22:41:24 summary: Enhance tool shed repository metadata to include information about invalid tool dependencies, invalid repository dependencies and orphan tool dependenccies. Display invalid dependencies in separate containers, and display appropriate warning messages when browsing repositories that contain orphan dependencies. affected #: 14 files diff -r b9a5aee402b31c2a316de0f68efe66fdb141418e -r ec4da708e45175610a33d16fcfc68732e425f54f lib/galaxy/util/shed_util_common.py --- a/lib/galaxy/util/shed_util_common.py +++ b/lib/galaxy/util/shed_util_common.py @@ -129,30 +129,32 @@ requirements_dict[ 'install_dir' ] = install_dir tool_dependencies[ dependency_key ] = requirements_dict return tool_dependencies -def add_tool_shed_orphan_settings_to_tool_dependencies( tool_dependencies, tools ): - """Inspect all received tool dependencies and label those that are orphans within the repository in the tool shed.""" +def add_orphan_settings_to_tool_dependencies( tool_dependencies, orphan_tool_dependencies ): + """Inspect all received tool dependencies and label those that are orphans within the repository.""" + orphan_env_dependencies = orphan_tool_dependencies.get( 'set_environment', None ) new_tool_dependencies = {} if tool_dependencies: for td_key, requirements_dict in tool_dependencies.items(): if td_key in [ 'set_environment' ]: - new_set_environment_dict_list = [] - for set_environment_dict in requirements_dict: - type = 'set_environment' - name = set_environment_dict.get( 'name', None ) - version = None - is_orphan_in_tool_shed = tool_dependency_is_orphan_in_tool_shed( type, name, version, tools ) - set_environment_dict[ 'is_orphan_in_tool_shed' ] = is_orphan_in_tool_shed - new_set_environment_dict_list.append( set_environment_dict ) + # "set_environment": [{"name": "R_SCRIPT_PATH", "type": "set_environment"}] + if orphan_env_dependencies: + new_set_environment_dict_list = [] + for set_environment_dict in requirements_dict: + if set_environment_dict in orphan_env_dependencies: + set_environment_dict[ 'is_orphan' ] = True + else: + set_environment_dict[ 'is_orphan' ] = False + new_set_environment_dict_list.append( set_environment_dict ) new_tool_dependencies[ td_key ] = new_set_environment_dict_list - new_tool_dependencies[ td_key ] = new_set_environment_dict_list + else: + new_tool_dependencies[ td_key ] = requirements_dict else: - type = requirements_dict.get( 'type', 'package' ) - name = requirements_dict.get( 'name', None ) - version = requirements_dict.get( 'version', None ) - if type and name: - is_orphan_in_tool_shed = tool_dependency_is_orphan_in_tool_shed( type, name, version, tools ) - requirements_dict[ 'is_orphan_in_tool_shed' ] = is_orphan_in_tool_shed - new_tool_dependencies[ td_key ] = requirements_dict + # {"R/2.15.1": {"name": "R", "readme": "some string", "type": "package", "version": "2.15.1"} + if td_key in orphan_tool_dependencies: + requirements_dict[ 'is_orphan' ] = True + else: + requirements_dict[ 'is_orphan' ] = False + new_tool_dependencies[ td_key ] = requirements_dict return new_tool_dependencies def add_tool_versions( trans, id, repository_metadata, changeset_revisions ): # Build a dictionary of { 'tool id' : 'parent tool id' } pairs for each tool in repository_metadata. @@ -318,6 +320,24 @@ datatypes = metadata[ 'datatypes' ] folder_id, datatypes_root_folder = container_util.build_datatypes_folder( trans, folder_id, datatypes ) containers_dict[ 'datatypes' ] = datatypes_root_folder + # Invalid repository dependencies container. + if metadata: + if 'invalid_repository_dependencies' in metadata: + invalid_repository_dependencies = metadata[ 'invalid_repository_dependencies' ] + folder_id, invalid_repository_dependencies_root_folder = \ + container_util.build_invalid_repository_dependencies_root_folder( trans, + folder_id, + invalid_repository_dependencies ) + containers_dict[ 'invalid_repository_dependencies' ] = invalid_repository_dependencies_root_folder + # Invalid tool dependencies container. + if metadata: + if 'invalid_tool_dependencies' in metadata: + invalid_tool_dependencies = metadata[ 'invalid_tool_dependencies' ] + folder_id, invalid_tool_dependencies_root_folder = \ + container_util.build_invalid_tool_dependencies_root_folder( trans, + folder_id, + invalid_tool_dependencies ) + containers_dict[ 'invalid_tool_dependencies' ] = invalid_tool_dependencies_root_folder # Invalid tools container. if metadata: if 'invalid_tools' in metadata: @@ -348,8 +368,9 @@ if 'tool_dependencies' in metadata: tool_dependencies = metadata[ 'tool_dependencies' ] if trans.webapp.name == 'community': - tools = metadata.get( 'tools', None ) - tool_dependencies = add_tool_shed_orphan_settings_to_tool_dependencies( tool_dependencies, tools ) + if 'orphan_tool_dependencies' in metadata: + orphan_tool_dependencies = metadata[ 'orphan_tool_dependencies' ] + tool_dependencies = add_orphan_settings_to_tool_dependencies( tool_dependencies, orphan_tool_dependencies ) folder_id, tool_dependencies_root_folder = container_util.build_tool_dependencies_folder( trans, folder_id, tool_dependencies, @@ -1123,20 +1144,28 @@ if datatypes: metadata_dict[ 'datatypes' ] = datatypes return metadata_dict -def generate_environment_dependency_metadata( elem, tool_dependencies_dict ): - """The value of env_var_name must match the value of the "set_environment" type in the tool config's <requirements> tag set.""" +def generate_environment_dependency_metadata( elem, valid_tool_dependencies_dict ): + """ + The value of env_var_name must match the value of the "set_environment" type in the tool config's <requirements> tag set, or the tool dependency + will be considered an orphan. Tool dependencies of type set_environment are always defined as valid, but may be orphans. + """ + # The value of the received elem looks something like this: + # <set_environment version="1.0"> + # <environment_variable name="JAVA_JAR_PATH" action="set_to">$INSTALL_DIR</environment_variable> + # </set_environment> requirements_dict = {} for env_elem in elem: + # <environment_variable name="JAVA_JAR_PATH" action="set_to">$INSTALL_DIR</environment_variable> env_name = env_elem.get( 'name', None ) if env_name: requirements_dict[ 'name' ] = env_name requirements_dict[ 'type' ] = 'set_environment' if requirements_dict: - if 'set_environment' in tool_dependencies_dict: - tool_dependencies_dict[ 'set_environment' ].append( requirements_dict ) + if 'set_environment' in valid_tool_dependencies_dict: + valid_tool_dependencies_dict[ 'set_environment' ].append( requirements_dict ) else: - tool_dependencies_dict[ 'set_environment' ] = [ requirements_dict ] - return tool_dependencies_dict + valid_tool_dependencies_dict[ 'set_environment' ] = [ requirements_dict ] + return valid_tool_dependencies_dict def generate_message_for_invalid_tools( trans, invalid_file_tups, repository, metadata_dict, as_html=True, displaying_invalid_tool=False ): if as_html: new_line = '<br/>' @@ -1173,6 +1202,43 @@ correction_msg = exception_msg.replace( '<br/>', new_line ).replace( '<b>', bold_start ).replace( '</b>', bold_end ) message += "%s%s%s - %s%s" % ( bold_start, tool_file, bold_end, correction_msg, new_line ) return message +def generate_message_for_orphan_tool_dependencies( metadata_dict ): + """ + The introduction of the support for orphan tool dependency definitions in tool shed repositories has resulted in the inability + to define an improperly configured tool dependency definition / tool config requirements tag combination as an invalid tool + dependency. This is certainly a weakness which cannot be correctly handled since now the only way to categorize a tool dependency + as invalid is if it consists of a complex repository dependency that is invalid. Any tool dependency definition other than those + is considered valid but perhaps an orphan due to it's actual invalidity. + """ + message = '' + status = 'done' + if metadata_dict: + orphan_tool_dependencies = metadata_dict.get( 'orphan_tool_dependencies', None ) + if orphan_tool_dependencies: + if 'tools' not in metadata_dict and 'invalid_tools' not in metadata_dict: + message += "This repository contains no tools, so these tool dependencies are considered orphans within this repository.<br/>" + for td_key, requirements_dict in orphan_tool_dependencies.items(): + if td_key == 'set_environment': + # "set_environment": [{"name": "R_SCRIPT_PATH", "type": "set_environment"}] + message += "The settings for <b>name</b> and <b>type</b> from a contained tool configuration file's <b>requirement</b> tag " + message += "does not match the information for the following tool dependency definitions in the <b>tool_dependencies.xml</b> " + message += "file, so these tool dependencies are considered orphans within this repository.<br/>" + for env_requirements_dict in requirements_dict: + name = env_requirements_dict[ 'name' ] + type = env_requirements_dict[ 'type' ] + message += "<b>* name:</b> %s, <b>type:</b> %s<br/>" % ( str( name ), str( type ) ) + else: + # "R/2.15.1": {"name": "R", "readme": "some string", "type": "package", "version": "2.15.1"} + message += "The settings for <b>name</b>, <b>version</b> and <b>type</b> from a contained tool configuration file's " + message += "<b>requirement</b> tag does not match the information for the following tool dependency definitions in the " + message += "<b>tool_dependencies.xml</b> file, so these tool dependencies are considered orphans within this repository.<br/>" + name = requirements_dict[ 'name' ] + type = requirements_dict[ 'type' ] + version = requirements_dict[ 'version' ] + message += "<b>* name:</b> %s, <b>type:</b> %s, <b>version:</b> %s<br/>" % ( str( name ), str( type ), str( version ) ) + message += "<br/>" + status = 'warning' + return message, status def generate_metadata_for_changeset_revision( app, repository, changeset_revision, repository_clone_url, shed_config_dict=None, relative_install_dir=None, repository_files_dir=None, resetting_all_metadata_on_repository=False, updating_installed_repository=False, persist=False ): @@ -1255,18 +1321,9 @@ # See if we have a repository dependencies defined. if name == 'repository_dependencies.xml': path_to_repository_dependencies_config = os.path.join( root, name ) - if app.name == 'community': - metadata_dict, error_message = generate_repository_dependency_metadata_for_tool_shed( app, - path_to_repository_dependencies_config, - metadata_dict ) - if error_message: - invalid_file_tups.append( ( name, error_message ) ) - elif app.name == 'galaxy': - metadata_dict, error_message = generate_repository_dependency_metadata_for_installed_repository( app, - path_to_repository_dependencies_config, - metadata_dict ) - if error_message: - invalid_file_tups.append( ( name, error_message ) ) + metadata_dict, error_message = generate_repository_dependency_metadata( app, path_to_repository_dependencies_config, metadata_dict ) + if error_message: + invalid_file_tups.append( ( name, error_message ) ) # See if we have one or more READ_ME files. elif name.lower() in readme_file_names: relative_path_to_readme = get_relative_path_to_repository_file( root, @@ -1350,11 +1407,12 @@ app.config.tool_data_path = original_tool_data_path app.config.tool_data_table_config_path = original_tool_data_table_config_path return metadata_dict, invalid_file_tups -def generate_package_dependency_metadata( app, elem, tool_dependencies_dict ): +def generate_package_dependency_metadata( app, elem, valid_tool_dependencies_dict, invalid_tool_dependencies_dict ): """ Generate the metadata for a tool dependencies package defined for a repository. The value of package_name must match the value of the "package" type in the tool config's <requirements> tag set. This method is called from both Galaxy and the tool shed. """ + repository_dependency_is_valid = True repository_dependency_tup = [] requirements_dict = {} error_message = '' @@ -1368,75 +1426,55 @@ if sub_elem.tag == 'readme': requirements_dict[ 'readme' ] = sub_elem.text elif sub_elem.tag == 'repository': - # We have a complex repository dependency. - current_rd_tups, error_message = handle_repository_elem( app=app, - repository_elem=sub_elem, - repository_dependencies_tups=None ) - if current_rd_tups: - repository_dependency_tup = current_rd_tups[ 0 ] + # We have a complex repository dependency. If the retruned value of repository_dependency_is_valid is True, the tool + # dependency definition will be set as invalid. This is currently the only case where a tool dependency definition is + # considered invalid. + repository_dependencies_tup, repository_dependency_is_valid, error_message = handle_repository_elem( app=app, + repository_elem=sub_elem ) if requirements_dict: dependency_key = '%s/%s' % ( package_name, package_version ) - tool_dependencies_dict[ dependency_key ] = requirements_dict - return tool_dependencies_dict, repository_dependency_tup, error_message -def generate_repository_dependency_metadata_for_installed_repository( app, repository_dependencies_config, metadata_dict ): + if repository_dependency_is_valid: + valid_tool_dependencies_dict[ dependency_key ] = requirements_dict + else: + # Append the error message to the requirements_dict. + requirements_dict[ 'error' ] = error_message + invalid_tool_dependencies_dict[ dependency_key ] = requirements_dict + return valid_tool_dependencies_dict, invalid_tool_dependencies_dict, repository_dependency_tup, repository_dependency_is_valid, error_message +def generate_repository_dependency_metadata( app, repository_dependencies_config, metadata_dict ): """ Generate a repository dependencies dictionary based on valid information defined in the received repository_dependencies_config. This method - is called only from Galaxy. + is called from the tool shed as well as from Galaxy. """ - repository_dependencies_tups = [] error_message = '' try: # Make sure we're looking at a valid repository_dependencies.xml file. tree = util.parse_xml( repository_dependencies_config ) root = tree.getroot() - is_valid = root.tag == 'repositories' + xml_is_valid = root.tag == 'repositories' except Exception, e: error_message = "Error parsing %s, exception: %s" % ( repository_dependencies_config, str( e ) ) log.debug( error_message ) - is_valid = False - if is_valid: - sa_session = app.model.context.current + xml_is_valid = False + if xml_is_valid: + invalid_repository_dependencies_dict = dict( description=root.get( 'description' ) ) + invalid_repository_dependencies_tups = [] + valid_repository_dependencies_dict = dict( description=root.get( 'description' ) ) + valid_repository_dependencies_tups = [] for repository_elem in root.findall( 'repository' ): - toolshed = repository_elem.attrib[ 'toolshed' ] - name = repository_elem.attrib[ 'name' ] - owner = repository_elem.attrib[ 'owner'] - changeset_revision = repository_elem.attrib[ 'changeset_revision' ] - repository_dependencies_tup = ( toolshed, name, owner, changeset_revision ) - if repository_dependencies_tup not in repository_dependencies_tups: - repository_dependencies_tups.append( repository_dependencies_tup ) - if repository_dependencies_tups: - repository_dependencies_dict = dict( description=root.get( 'description' ), - repository_dependencies=repository_dependencies_tups ) - metadata_dict[ 'repository_dependencies' ] = repository_dependencies_dict - return metadata_dict, error_message -def generate_repository_dependency_metadata_for_tool_shed( app, repository_dependencies_config, metadata_dict ): - """ - Generate a repository dependencies dictionary based on valid information defined in the received repository_dependencies_config. This method - is called only from the tool shed. - """ - repository_dependencies_tups = [] - error_message = '' - try: - # Make sure we're looking at a valid repository_dependencies.xml file. - tree = util.parse_xml( repository_dependencies_config ) - root = tree.getroot() - is_valid = root.tag == 'repositories' - except Exception, e: - error_message = "Error parsing %s, exception: %s" % ( repository_dependencies_config, str( e ) ) - log.debug( error_message ) - is_valid = False - if is_valid: - for repository_elem in root.findall( 'repository' ): - current_rd_tups, error_message = handle_repository_elem( app, repository_elem, repository_dependencies_tups ) - if error_message: - # Log the problem, but generate metadata for the invalid repository dependencies. - log.debug( error_message ) - for crdt in current_rd_tups: - repository_dependencies_tups.append( crdt ) - if repository_dependencies_tups: - repository_dependencies_dict = dict( description=root.get( 'description' ), - repository_dependencies=repository_dependencies_tups ) - metadata_dict[ 'repository_dependencies' ] = repository_dependencies_dict + repository_dependencies_tup, repository_dependency_is_valid, error_message = handle_repository_elem( app, repository_elem ) + if repository_dependency_is_valid: + valid_repository_dependencies_tups.append( repository_dependencies_tup ) + else: + # Append the error_message to the repository dependencies tuple. + toolshed, name, owner, changeset_revision = repository_dependencies_tup + repository_dependencies_tup = ( toolshed, name, owner, changeset_revision, error_message ) + invalid_repository_dependencies_tups.append( repository_dependencies_tup ) + if invalid_repository_dependencies_tups: + invalid_repository_dependencies_dict[ 'repository_dependencies' ] = invalid_repository_dependencies_tups + metadata_dict[ 'invalid_repository_dependencies' ] = invalid_repository_dependencies_dict + if valid_repository_dependencies_tups: + valid_repository_dependencies_dict[ 'repository_dependencies' ] = valid_repository_dependencies_tups + metadata_dict[ 'repository_dependencies' ] = valid_repository_dependencies_dict return metadata_dict, error_message def generate_sharable_link_for_repository_in_tool_shed( trans, repository, changeset_revision=None ): """Generate the URL for sharing a repository that is in the tool shed.""" @@ -1452,12 +1490,27 @@ If the combination of name, version and type of each element is defined in the <requirement> tag for at least one tool in the repository, then update the received metadata_dict with information from the parsed tool_dependencies_config. """ + """ + "{"orphan_tool_dependencies": + {"bwa/0.5.9": + {"name": "bwa", + "readme": "\\nCompiling BWA requires zlib and libpthread to be present on your system.\\n ", + "type": "package", "version": "0.5.9"}}, + "tool_dependencies": + {"bwa/0.5.9": + {"name": "bwa", + "readme": "\\nCompiling BWA requires zlib and libpthread to be present on your system.\\n ", + "type": "package", + "version": "0.5.9"}}}" + """ error_message = '' if original_repository_metadata: # Keep a copy of the original tool dependencies dictionary and the list of tool dictionaries in the metadata. - original_tool_dependencies_dict = original_repository_metadata.get( 'tool_dependencies', None ) + original_valid_tool_dependencies_dict = original_repository_metadata.get( 'tool_dependencies', None ) + original_invalid_tool_dependencies_dict = original_repository_metadata.get( 'invalid_tool_dependencies', None ) else: - original_tool_dependencies_dict = None + original_valid_tool_dependencies_dict = None + original_invalid_tool_dependencies_dict = None try: tree = ElementTree.parse( tool_dependencies_config ) except Exception, e: @@ -1466,81 +1519,53 @@ return metadata_dict, error_message root = tree.getroot() ElementInclude.include( root ) - tool_dependencies_dict = {} - repository_dependency_tups = [] + tool_dependency_is_valid = True + valid_tool_dependencies_dict = {} + invalid_tool_dependencies_dict = {} + valid_repository_dependency_tups = [] + invalid_repository_dependency_tups = [] + description = root.get( 'description' ) for elem in root: if elem.tag == 'package': - tool_dependencies_dict, repository_dependency_tup, message = generate_package_dependency_metadata( app, elem, tool_dependencies_dict ) - if repository_dependency_tup and repository_dependency_tup not in repository_dependency_tups: - repository_dependency_tups.append( repository_dependency_tup ) - if message: - log.debug( message ) - error_message = '%s %s' % ( error_message, message ) + valid_tool_dependencies_dict, invalid_tool_dependencies_dict, repository_dependency_tup, repository_dependency_is_valid, message = \ + generate_package_dependency_metadata( app, elem, valid_tool_dependencies_dict, invalid_tool_dependencies_dict ) + if repository_dependency_is_valid: + if repository_dependency_tup and repository_dependency_tup not in valid_repository_dependency_tups: + # We have a valid complex repository dependency. + valid_repository_dependency_tups.append( repository_dependency_tup ) + else: + if repository_dependency_tup and repository_dependency_tup not in invalid_repository_dependency_tups: + # We have an invalid complex repository dependency, so mark the tool dependency as invalid. + tool_dependency_is_valid = False + # Append the error message to the invalid repository dependency tuple. + repository_dependency_tup.append( message ) + invalid_repository_dependency_tups.append( repository_dependency_tup ) + error_message = '%s %s' % ( error_message, message ) elif elem.tag == 'set_environment': - tool_dependencies_dict = generate_environment_dependency_metadata( elem, tool_dependencies_dict ) - if tool_dependencies_dict: - if original_tool_dependencies_dict: + # Tool dependencies of this type are always considered valid, but may be orphans. + valid_tool_dependencies_dict = generate_environment_dependency_metadata( elem, valid_tool_dependencies_dict ) + if valid_tool_dependencies_dict: + if original_valid_tool_dependencies_dict: # We're generating metadata on an update pulled to a tool shed repository installed into a Galaxy instance, so handle changes to # tool dependencies appropriately. - handle_existing_tool_dependencies_that_changed_in_update( app, repository, original_tool_dependencies_dict, tool_dependencies_dict ) - metadata_dict[ 'tool_dependencies' ] = tool_dependencies_dict - if repository_dependency_tups: - repository_dependencies_dict = metadata_dict.get( 'repository_dependencies', None ) - for repository_dependency_tup in repository_dependency_tups: - rd_tool_shed, rd_name, rd_owner, rd_changeset_revision = repository_dependency_tup - if app.name == 'community': - if tool_shed_is_this_tool_shed( rd_tool_shed ): - # Make sure the repository name id valid. - valid_named_repository = get_repository_by_name( app, rd_name ) - if valid_named_repository: - # See if the owner is valid. - valid_owned_repository = get_repository_by_name_and_owner( app, rd_name, rd_owner ) - if valid_owned_repository: - # See if the defined changeset revision is valid. - if not changeset_is_valid( app, valid_owned_repository, rd_changeset_revision ): - err_msg = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s "% \ - ( rd_tool_shed, rd_name, rd_owner, rd_changeset_revision ) - err_msg += "because the changeset revision is invalid. " - log.debug( err_msg ) - error_message += err_msg - else: - err_msg = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s "% \ - ( rd_tool_shed, rd_name, rd_owner, rd_changeset_revision ) - err_msg += "because the owner is invalid. " - log.debug( err_msg ) - error_message += err_msg - else: - err_msg = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s "% \ - ( rd_tool_shed, rd_name, rd_owner, rd_changeset_revision ) - err_msg += "because the name is invalid. " - log.debug( err_msg ) - error_message += err_msg - else: - err_msg = "Repository dependencies are currently supported only within the same tool shed. Ignoring repository dependency definition " - err_msg += "for tool shed %s, name %s, owner %s, changeset revision %s. " % ( rd_tool_shed, rd_name, rd_owner, rd_changeset_revision ) - log.debug( err_msg ) - error_message += err_msg - else: - repository_owner = repository.owner - rd_key = container_util.generate_repository_dependencies_key_for_repository( toolshed_base_url=rd_tool_shed, - repository_name=rd_name, - repository_owner=rd_owner, - changeset_revision=rd_changeset_revision ) - if repository_dependencies_dict: - if rd_key in repository_dependencies_dict: - repository_dependencies = repository_dependencies_dict[ rd_key ] - for repository_dependency_tup in repository_dependency_tups: - if repository_dependency_tup not in repository_dependencies: - repository_dependencies.append( repository_dependency_tup ) - repository_dependencies_dict[ rd_key ] = repository_dependencies - else: - repository_dependencies_dict[ rd_key ] = repository_dependency_tups - else: - repository_dependencies_dict = dict( root_key=rd_key, - description=root.get( 'description' ), - repository_dependencies=repository_dependency_tups ) - if repository_dependencies_dict: - metadata_dict[ 'repository_dependencies' ] = repository_dependencies_dict + handle_existing_tool_dependencies_that_changed_in_update( app, repository, original_valid_tool_dependencies_dict, valid_tool_dependencies_dict ) + metadata_dict[ 'tool_dependencies' ] = valid_tool_dependencies_dict + if invalid_tool_dependencies_dict: + metadata_dict[ 'invalid_tool_dependencies' ] = invalid_tool_dependencies_dict + if valid_repository_dependency_tups: + metadata_dict = update_repository_dependencies_metadata( metadata=metadata_dict, + repository_dependency_tups=valid_repository_dependency_tups, + is_valid=True, + description=description ) + if invalid_repository_dependency_tups: + metadata_dict = update_repository_dependencies_metadata( metadata=metadata_dict, + repository_dependency_tups=invalid_repository_dependency_tups, + is_valid=False, + description=description ) + # Determine and store orphan tool dependencies. + orphan_tool_dependencies = get_orphan_tool_dependencies( metadata_dict ) + if orphan_tool_dependencies: + metadata_dict[ 'orphan_tool_dependencies' ] = orphan_tool_dependencies return metadata_dict, error_message def generate_tool_elem( tool_shed, repository_name, changeset_revision, owner, tool_file_path, tool, tool_section ): if tool_section is not None: @@ -1926,6 +1951,33 @@ sorted_changeset_tups = sorted( changeset_tups ) sorted_changeset_revisions = [ changeset_tup[ 1 ] for changeset_tup in sorted_changeset_tups ] return sorted_changeset_revisions +def get_orphan_tool_dependencies( metadata ): + """Inspect tool dependencies included in the received metadata and determine if any of them are orphans within the repository.""" + orphan_tool_dependencies_dict = {} + if metadata: + tools = metadata.get( 'tools', None ) + tool_dependencies = metadata.get( 'tool_dependencies', None ) + if tool_dependencies: + for td_key, requirements_dict in tool_dependencies.items(): + if td_key in [ 'set_environment' ]: + for set_environment_dict in requirements_dict: + type = 'set_environment' + name = set_environment_dict.get( 'name', None ) + version = None + if name: + if tool_dependency_is_orphan( type, name, version, tools ): + if td_key in orphan_tool_dependencies_dict: + orphan_tool_dependencies_dict[ td_key ].append( set_environment_dict ) + else: + orphan_tool_dependencies_dict[ td_key ] = [ set_environment_dict ] + else: + type = requirements_dict.get( 'type', None ) + name = requirements_dict.get( 'name', None ) + version = requirements_dict.get( 'version', None ) + if type and name: + if tool_dependency_is_orphan( type, name, version, tools ): + orphan_tool_dependencies_dict[ td_key ] = requirements_dict + return orphan_tool_dependencies_dict def get_parent_id( trans, id, old_id, version, guid, changeset_revisions ): parent_id = None # Compare from most recent to oldest. @@ -2409,6 +2461,14 @@ ( str( rd_changeset_revision ), str( rd_name ), str( rd_owner ), str( repository_name ) ) log.debug( message ) return updated_key_rd_dicts +def get_updated_changeset_revisions_from_tool_shed( tool_shed_url, name, owner, changeset_revision ): + """Get all appropriate newer changeset revisions for the repository defined by the received tool_shed_url / name / owner combination.""" + url = url_join( tool_shed_url, + 'repository/updated_changeset_revisions?name=%s&owner=%s&changeset_revision=%s' % ( name, owner, changeset_revision ) ) + response = urllib2.urlopen( url ) + text = response.read() + response.close() + return text def get_url_from_repository_tool_shed( app, repository ): """ The stored value of repository.tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is @@ -2635,63 +2695,78 @@ all_repository_dependencies=all_repository_dependencies, handled_key_rd_dicts=handled_key_rd_dicts, circular_repository_dependencies=circular_repository_dependencies ) -def handle_repository_elem( app, repository_elem, repository_dependencies_tups ): +def handle_repository_elem( app, repository_elem ): """ Process the received repository_elem which is a <repository> tag either from a repository_dependencies.xml file or a tool_dependencies.xml file. If the former, we're generating repository dependencies metadata for a repository in the tool shed. If the latter, we're generating package - dependency metadata with in Galaxy or the tool shed. + dependency metadata within Galaxy or the tool shed. """ - if repository_dependencies_tups is None: - new_rd_tups = [] - else: - new_rd_tups = [ rdt for rdt in repository_dependencies_tups ] + sa_session = app.model.context.current + is_valid = True error_message = '' - sa_session = app.model.context.current toolshed = repository_elem.attrib[ 'toolshed' ] name = repository_elem.attrib[ 'name' ] owner = repository_elem.attrib[ 'owner' ] changeset_revision = repository_elem.attrib[ 'changeset_revision' ] + repository_dependencies_tup = ( toolshed, name, owner, changeset_revision ) user = None repository = None if app.name == 'galaxy': - # We're in Galaxy. - try: - repository = sa_session.query( app.model.ToolShedRepository ) \ - .filter( and_( app.model.ToolShedRepository.table.c.name == name, - app.model.ToolShedRepository.table.c.owner == owner ) ) \ - .first() - except: - error_message = "Invalid name <b>%s</b> or owner <b>%s</b> defined for repository. Repository dependencies will be ignored." % ( name, owner ) + # We're in Galaxy. We reach here when we're generating the metadata for a tool dependencies package defined for a repository or when we're + # generating metadata for an installed repository.. + #try: + # See if we can locate the installed repository via the changeset_revision defined in the repository_elem (it may be outdated). If we're + # successful in locating an installed repository with the attributes defined in the repository_elem, we know it is valid. + repository = get_repository_for_dependency_relationship( app, toolshed, name, owner, changeset_revision ) + if repository: + return repository_dependencies_tup, is_valid, error_message + else: + # Send a request to the tool shed to retrieve appropriate additional changeset revisions with which the repository may have been installed. + try: + # Hopefully the tool shed is accessible. + text = get_updated_changeset_revisions_from_tool_shed( toolshed, name, owner, changeset_revision ) + except: + text = None + if text: + updated_changeset_revisions = util.listify( text ) + for updated_changeset_revision in updated_changeset_revisions: + repository = get_repository_for_dependency_relationship( app, toolshed, name, owner, updated_changeset_revision ) + if repository: + return repository_dependencies_tup, is_valid, error_message + # We'll currently default to setting the repository dependency definition as invalid if an installed repository cannot be found. + # This may not be ideal because the tool shed may have simply been inaccessible when metadata was being generated for the installed + # tool shed repository. + error_message = "Ignoring invalid repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s "% \ + ( toolshed, name, owner, changeset_revision ) log.debug( error_message ) - return new_rd_tups, error_message - repository_dependencies_tup = ( toolshed, name, owner, changeset_revision ) - if repository_dependencies_tup not in new_rd_tups: - new_rd_tups.append( repository_dependencies_tup ) + is_valid = False + return repository_dependencies_tup, is_valid, error_message else: # We're in the tool shed. if tool_shed_is_this_tool_shed( toolshed ): - # Append the repository dependency definition regardless of whether it's valid or not, as Galaxy needs this to - # properly display an error when the repository dependency is invalid at the time of installation. - repository_dependencies_tup = ( toolshed, name, owner, changeset_revision ) - if repository_dependencies_tup not in new_rd_tups: - new_rd_tups.append( repository_dependencies_tup ) try: user = sa_session.query( app.model.User ) \ .filter( app.model.User.table.c.username == owner ) \ .one() - except Exception, e: - error_message = "Invalid owner <b>%s</b> defined for repository <b>%s</b>. Repository dependencies will be ignored." % ( str( owner ), str( name ) ) + except Exception, e: + error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s "% \ + ( toolshed, name, owner, changeset_revision ) + error_message += "because the owner is invalid. " log.debug( error_message ) - return new_rd_tups, error_message + is_valid = False + return repository_dependencies_tup, is_valid, error_message try: repository = sa_session.query( app.model.Repository ) \ .filter( and_( app.model.Repository.table.c.name == name, app.model.Repository.table.c.user_id == user.id ) ) \ .one() except: - error_message = "Invalid repository name <b>%s</b> defined. Repository dependencies will be ignored." % str( name ) + error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s "% \ + ( toolshed, name, owner, changeset_revision ) + error_message += "because the name is invalid. " log.debug( error_message ) - return new_rd_tups, error_message + is_valid = False + return repository_dependencies_tup, is_valid, error_message # Find the specified changeset revision in the repository's changelog to see if it's valid. found = False repo = hg.repository( get_configured_ui(), repository.repo_path( app ) ) @@ -2701,15 +2776,20 @@ found = True break if not found: - error_message = "Invalid changeset revision <b>%s</b> defined. Repository dependencies will be ignored." % str( changeset_revision ) + error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s "% \ + ( toolshed, name, owner, changeset_revision ) + error_message += "because the changeset revision is invalid. " log.debug( error_message ) - return new_rd_tups, error_message + is_valid = False + return repository_dependencies_tup, is_valid, error_message else: - # Repository dependencies are currentlhy supported within a single tool shed. - error_message = "Invalid tool shed <b>%s</b> defined for repository <b>%s</b>. " % ( toolshed, name ) - error_message += "Repository dependencies are currently supported within a single tool shed." + # Repository dependencies are currently supported within a single tool shed. + error_message = "Repository dependencies are currently supported only within the same tool shed. Ignoring repository dependency definition " + error_message += "for tool shed %s, name %s, owner %s, changeset revision %s. " % ( toolshed, name, owner, changeset_revision ) log.debug( error_message ) - return new_rd_tups, error_message + is_valid = False + return repository_dependencies_tup, is_valid, error_message + return repository_dependencies_tup, is_valid, error_message def handle_sample_files_and_load_tool_from_disk( trans, repo_files_dir, tool_config_filepath, work_dir ): # Copy all sample files from disk to a temporary directory since the sample files may be in multiple directories. message = '' @@ -2761,31 +2841,6 @@ message = str( e ) error = True return error, message -def has_orphan_tool_dependencies_in_tool_shed( metadata ): - """Inspect tool dependencies included in the received metadata and determine if any of them are orphans within the repository in the tool shed.""" - if metadata: - tools = metadata.get( 'tools', None ) - tool_dependencies = metadata.get( 'tool_dependencies', None ) - if tool_dependencies: - for td_key, requirements_dict in tool_dependencies.items(): - if td_key in [ 'set_environment' ]: - for set_environment_dict in requirements_dict: - type = 'set_environment' - name = set_environment_dict.get( 'name', None ) - version = None - if name: - is_orphan_in_tool_shed = tool_dependency_is_orphan_in_tool_shed( type, name, version, tools ) - if is_orphan_in_tool_shed: - return True - else: - type = requirements_dict.get( 'type', None ) - name = requirements_dict.get( 'name', None ) - version = requirements_dict.get( 'version', None ) - if type and name: - is_orphan_in_tool_shed = tool_dependency_is_orphan_in_tool_shed( type, name, version, tools ) - if is_orphan_in_tool_shed: - return True - return False def has_previous_repository_reviews( trans, repository, changeset_revision ): """Determine if a repository has a changeset revision review prior to the received changeset revision.""" repo = hg.repository( get_configured_ui(), repository.repo_path( trans.app ) ) @@ -2839,6 +2894,8 @@ return True return False def is_downloadable( metadata_dict ): + # NOTE: although repository README files are considered Galaxy utilities, they have no effect on determining if a revision is instakllable. + # See the comments in the compare_readme_files() method. if 'datatypes' in metadata_dict: # We have proprietary datatypes. return True @@ -3640,6 +3697,7 @@ """Set metadata on the repository tip in the tool shed - this method is not called from Galaxy.""" error_message, status = set_repository_metadata( trans, repository, content_alert_str=content_alert_str, **kwd ) if error_message: + # FIXME: This probably should not redirect since this method is called from the upload controller as well as the repository controller. # If there is an error, display it. return trans.response.send_redirect( web.url_for( controller='repository', action='manage_repository', @@ -3654,7 +3712,7 @@ except: file_name = fpath return file_name -def tool_dependency_is_orphan_in_tool_shed( type, name, version, tools ): +def tool_dependency_is_orphan( type, name, version, tools ): """ Determine if the combination of the received type, name and version is defined in the <requirement> tag for at least one tool in the received list of tools. If not, the tool dependency defined by the combination is considered an orphan in it's repository in the tool shed. @@ -3841,6 +3899,39 @@ # It would be nice if we could use mercurial's purge extension to remove untracked files. The problem is that # purging is not supported by the mercurial API. commands.update( get_configured_ui(), repo, rev=ctx_rev ) +def update_repository_dependencies_metadata( metadata, repository_dependency_tups, is_valid, description ): + if is_valid: + repository_dependencies_dict = metadata.get( 'repository_dependencies', None ) + else: + repository_dependencies_dict = metadata.get( 'invalid_repository_dependencies', None ) + for repository_dependency_tup in repository_dependency_tups: + if is_valid: + tool_shed, name, owner, changeset_revision = repository_dependency_tup + else: + tool_shed, name, owner, changeset_revision, error_message = repository_dependency_tup + rd_key = container_util.generate_repository_dependencies_key_for_repository( toolshed_base_url=tool_shed, + repository_name=name, + repository_owner=owner, + changeset_revision=changeset_revision ) + if repository_dependencies_dict: + if rd_key in repository_dependencies_dict: + repository_dependencies = repository_dependencies_dict[ rd_key ] + for repository_dependency_tup in repository_dependency_tups: + if repository_dependency_tup not in repository_dependencies: + repository_dependencies.append( repository_dependency_tup ) + repository_dependencies_dict[ rd_key ] = repository_dependencies + else: + repository_dependencies_dict[ rd_key ] = repository_dependency_tups + else: + repository_dependencies_dict = dict( root_key=rd_key, + description=description, + repository_dependencies=repository_dependency_tups ) + if repository_dependencies_dict: + if is_valid: + metadata[ 'repository_dependencies' ] = repository_dependencies_dict + else: + metadata[ 'invalid_repository_dependencies' ] = repository_dependencies_dict + return metadata def url_join( *args ): parts = [] for arg in args: diff -r b9a5aee402b31c2a316de0f68efe66fdb141418e -r ec4da708e45175610a33d16fcfc68732e425f54f lib/galaxy/webapps/community/controllers/repository.py --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -2001,11 +2001,9 @@ all_repository_dependencies=None, handled_key_rd_dicts=None ) if metadata: - if 'repository_dependencies' in metadata and not repository_dependencies: - message += 'The repository dependency definitions for this repository are invalid and will be ignored. Make sure valid <b>toolshed</b>, ' - message += '<b>name</b>, <b>owner</b> and <b>changeset_revision</b> values are defined in the contained <b>repository_dependencies.xml</b> ' - message += 'file to correct this problem.' - status = 'error' + if 'orphan_tool_dependencies' in metadata: + orphan_message, status = suc.generate_message_for_orphan_tool_dependencies( metadata ) + message += orphan_message if is_malicious: if trans.app.security_agent.can_push( trans.app, trans.user, repository ): message += malicious_error_can_push @@ -2221,7 +2219,7 @@ message = suc.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, metadata_dict ) status = 'error' else: - message = "All repository metadata has been reset." + message = "All repository metadata has been reset. " status = 'done' return trans.response.send_redirect( web.url_for( controller='repository', action='manage_repository', @@ -2755,9 +2753,9 @@ all_repository_dependencies=None, handled_key_rd_dicts=None ) if metadata: - if 'repository_dependencies' in metadata and not repository_dependencies: - message += 'The repository dependency definitions for this repository are invalid and will be ignored.' - status = 'error' + if 'orphan_tool_dependencies' in metadata: + orphan_message, status = suc.generate_message_for_orphan_tool_dependencies( metadata ) + message += orphan_message else: repository_metadata_id = None metadata = None diff -r b9a5aee402b31c2a316de0f68efe66fdb141418e -r ec4da708e45175610a33d16fcfc68732e425f54f lib/galaxy/webapps/community/controllers/upload.py --- a/lib/galaxy/webapps/community/controllers/upload.py +++ b/lib/galaxy/webapps/community/controllers/upload.py @@ -156,7 +156,7 @@ else: source_type = "file" source = uploaded_file_filename - message = "The %s '%s' has been successfully%suploaded to the repository. " % ( source_type, source, uncompress_str ) + message = "The %s <b>%s</b> has been successfully%suploaded to the repository. " % ( source_type, source, uncompress_str ) if istar and ( undesirable_dirs_removed or undesirable_files_removed ): items_removed = undesirable_dirs_removed + undesirable_files_removed message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) were removed from the archive. " % items_removed @@ -169,16 +169,32 @@ suc.set_repository_metadata_due_to_new_tip( trans, repository, content_alert_str=content_alert_str, **kwd ) # Provide a warning message if a tool_dependencies.xml file is provided, but tool dependencies weren't loaded due to a requirement tag mismatch # or some other problem. - if suc.get_config_from_disk( 'tool_dependencies.xml', repo_dir ): - if repository.metadata_revisions: - # A repository's metadata revisions are order descending by update_time, so the zeroth revision will be the tip just after an upload. - metadata_dict = repository.metadata_revisions[0].metadata - else: - metadata_dict = {} - if suc.has_orphan_tool_dependencies_in_tool_shed( metadata_dict ): - message += 'Name, version and type from a tool requirement tag does not match the information in the "tool_dependencies.xml file", ' - message += 'so one or more of the defined tool dependencies are considered orphans within this repository.' - status = 'warning' + if repository.metadata_revisions: + # A repository's metadata revisions are order descending by update_time, so the zeroth revision will be the tip just after an upload. + metadata_dict = repository.metadata_revisions[0].metadata + else: + metadata_dict = {} + # Handle messaging for orphan tool dependencies. + orphan_message, status = suc.generate_message_for_orphan_tool_dependencies( metadata_dict ) + if orphan_message: + message += orphan_message + # Display message for invalid tool sependencies. + invalid_tool_dependencies = metadata_dict.get( 'invalid_tool_dependencies', None ) + if invalid_tool_dependencies: + for td_key, requirement_dict in invalid_tool_dependencies.items(): + error = requirement_dict.get( 'error', None ) + if error: + message = "%s %s" % ( message, str( error ) ) + status = 'error' + # Display message for invalid repository dependencies. + invalid_repository_dependencies_dict = metadata_dict.get( 'invalid_repository_dependencies', None ) + if invalid_repository_dependencies_dict: + invalid_repository_dependencies = invalid_repository_dependencies_dict[ 'invalid_repository_dependencies' ] + for repository_dependency_tup in invalid_repository_dependencies: + toolshed, name, owner, changeset_revision, error = repository_dependency_tup + if error: + message += "%s %s" % ( message, str( error ) ) + status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. suc.reset_tool_data_tables( trans.app ) trans.response.send_redirect( web.url_for( controller='repository', diff -r b9a5aee402b31c2a316de0f68efe66fdb141418e -r ec4da708e45175610a33d16fcfc68732e425f54f lib/galaxy/webapps/community/util/container_util.py --- a/lib/galaxy/webapps/community/util/container_util.py +++ b/lib/galaxy/webapps/community/util/container_util.py @@ -15,6 +15,8 @@ self.description = None self.datatypes = [] self.folders = [] + self.invalid_repository_dependencies = [] + self.invalid_tool_dependencies = [] self.invalid_tools = [] self.valid_tools = [] self.tool_dependencies = [] @@ -54,6 +56,16 @@ self.mimetype = mimetype self.subclass = subclass +class InvalidRepositoryDependency( object ): + """Invalid repository dependency definition object""" + def __init__( self, id=None, toolshed=None, repository_name=None, repository_owner=None, changeset_revision=None, error=None ): + self.id = id + self.toolshed = toolshed + self.repository_name = repository_name + self.repository_owner = repository_owner + self.changeset_revision = changeset_revision + self.error = error + class InvalidTool( object ): """Invalid tool object""" def __init__( self, id=None, tool_config=None, repository_id=None, changeset_revision=None, repository_installation_status=None ): @@ -63,6 +75,15 @@ self.changeset_revision = changeset_revision self.repository_installation_status = repository_installation_status +class InvalidToolDependency( object ): + """Invalid tool dependency definition object""" + def __init__( self, id=None, name=None, version=None, type=None, error=None ): + self.id = id + self.name = name + self.version = version + self.type = type + self.error = error + class ReadMe( object ): """Readme text object""" def __init__( self, id=None, name=None, text=None ): @@ -158,6 +179,79 @@ else: datatypes_root_folder = None return folder_id, datatypes_root_folder +def build_invalid_repository_dependencies_root_folder( trans, folder_id, invalid_repository_dependencies_dict ): + """Return a folder hierarchy containing invalid repository dependencies.""" + label = 'Invalid repository dependencies' + if invalid_repository_dependencies_dict: + invalid_repository_dependency_id = 0 + folder_id += 1 + invalid_repository_dependencies_root_folder = Folder( id=folder_id, key='root', label='root', parent=None ) + folder_id += 1 + invalid_repository_dependencies_folder = Folder( id=folder_id, + key='invalid_repository_dependencies', + label=label, + parent=invalid_repository_dependencies_root_folder ) + invalid_repository_dependencies_root_folder.folders.append( invalid_repository_dependencies_folder ) + invalid_repository_dependencies = invalid_repository_dependencies_dict[ 'repository_dependencies' ] + for invalid_repository_dependency in invalid_repository_dependencies: + folder_id += 1 + invalid_repository_dependency_id += 1 + toolshed, name, owner, changeset_revision, error = invalid_repository_dependency + key = generate_repository_dependencies_key_for_repository( toolshed, name, owner, changeset_revision ) + label = "Repository <b>%s</b> revision <b>%s</b> owned by <b>%s</b>" % ( name, changeset_revision, owner ) + folder = Folder( id=folder_id, + key=key, + label=label, + parent=invalid_repository_dependencies_folder ) + ird = InvalidRepositoryDependency( id=invalid_repository_dependency_id, + toolshed=toolshed, + repository_name=name, + repository_owner=owner, + changeset_revision=changeset_revision, + error=error ) + folder.invalid_repository_dependencies.append( ird ) + invalid_repository_dependencies_folder.folders.append( folder ) + else: + invalid_repository_dependencies_root_folder = None + return folder_id, invalid_repository_dependencies_root_folder +def build_invalid_tool_dependencies_root_folder( trans, folder_id, invalid_tool_dependencies_dict ): + """Return a folder hierarchy containing invalid tool dependencies.""" + # # INvalid tool dependencies are always packages like: + # {"R/2.15.1": {"name": "R", "readme": "some string", "type": "package", "version": "2.15.1" "error" : "some sting" } + label = 'Invalid tool dependencies' + if invalid_tool_dependencies_dict: + invalid_tool_dependency_id = 0 + folder_id += 1 + invalid_tool_dependencies_root_folder = Folder( id=folder_id, key='root', label='root', parent=None ) + folder_id += 1 + invalid_tool_dependencies_folder = Folder( id=folder_id, + key='invalid_tool_dependencies', + label=label, + parent=invalid_tool_dependencies_root_folder ) + invalid_tool_dependencies_root_folder.folders.append( invalid_tool_dependencies_folder ) + for td_key, requirements_dict in invalid_tool_dependencies_dict.items(): + folder_id += 1 + invalid_tool_dependency_id += 1 + name = requirements_dict[ 'name' ] + type = requirements_dict[ 'type' ] + version = requirements_dict[ 'version' ] + error = requirements_dict[ 'error' ] + key = generate_tool_dependencies_key( name, version, type ) + label = "Version <b>%s</b> of the <b>%s</b><b>%s</b>" % ( version, name, type ) + folder = Folder( id=folder_id, + key=key, + label=label, + parent=invalid_tool_dependencies_folder ) + itd = InvalidToolDependency( id=invalid_tool_dependency_id, + name=name, + version=version, + type=type, + error=error ) + folder.invalid_tool_dependencies.append( itd ) + invalid_tool_dependencies_folder.folders.append( folder ) + else: + invalid_tool_dependencies_root_folder = None + return folder_id, invalid_tool_dependencies_root_folder def build_invalid_tools_folder( trans, folder_id, invalid_tool_configs, changeset_revision, repository=None, label='Invalid tools' ): """Return a folder hierarchy containing invalid tools.""" # TODO: Should we display invalid tools on the tool panel selection page when installing the repository into Galaxy? @@ -346,7 +440,7 @@ if dependency_key in [ 'set_environment' ]: for set_environment_dict in requirements_dict: if trans.webapp.name == 'community': - is_orphan = set_environment_dict.get( 'is_orphan_in_tool_shed', False ) + is_orphan = set_environment_dict.get( 'is_orphan', False ) else: # TODO: handle this is Galaxy is_orphan = False @@ -373,7 +467,7 @@ folder.tool_dependencies.append( tool_dependency ) else: if trans.webapp.name == 'community': - is_orphan = requirements_dict.get( 'is_orphan_in_tool_shed', False ) + is_orphan = requirements_dict.get( 'is_orphan', False ) else: # TODO: handle this is Galaxy is_orphan = False @@ -476,6 +570,8 @@ str( repository_owner ), STRSEP, str( changeset_revision ) ) +def generate_tool_dependencies_key( name, version, type ): + return '%s%s%s%s%s' % ( str( name ), STRSEP, str( version ), STRSEP, str( type ) ) def get_folder( folder, key ): if folder.key == key: return folder diff -r b9a5aee402b31c2a316de0f68efe66fdb141418e -r ec4da708e45175610a33d16fcfc68732e425f54f templates/webapps/community/repository/common.mako --- a/templates/webapps/community/repository/common.mako +++ b/templates/webapps/community/repository/common.mako @@ -224,6 +224,10 @@ folder_label = "%s<i> - installation of these additional repositories is required</i>" % folder_label if trans.webapp.name == 'galaxy': col_span_str = 'colspan="4"' + elif folder.label == 'Invalid repository dependencies': + folder_label = "%s<i> - click the repository dependency to see why it is invalid</i>" % folder_label + elif folder.label == 'Invalid tool dependencies': + folder_label = "%s<i> - click the tool dependency to see why it is invalid</i>" % folder_label elif folder.label == 'Valid tools': col_span_str = 'colspan="3"' if folder.description: @@ -266,10 +270,16 @@ %for readme in folder.readme_files: ${render_readme( readme, pad, my_row, row_counter )} %endfor + %for invalid_repository_dependency in folder.invalid_repository_dependencies: + ${render_invalid_repository_dependency( invalid_repository_dependency, pad, my_row, row_counter )} + %endfor %for index, repository_dependency in enumerate( folder.repository_dependencies ): <% row_is_header = index == 0 %> ${render_repository_dependency( repository_dependency, pad, my_row, row_counter, row_is_header )} %endfor + %for invalid_tool_dependency in folder.invalid_tool_dependencies: + ${render_invalid_tool_dependency( invalid_tool_dependency, pad, my_row, row_counter )} + %endfor %for index, tool_dependency in enumerate( folder.tool_dependencies ): <% row_is_header = index == 0 %> ${render_tool_dependency( tool_dependency, pad, my_row, row_counter, row_is_header )} @@ -321,6 +331,25 @@ %></%def> +<%def name="render_invalid_repository_dependency( invalid_repository_dependency, pad, parent, row_counter )"> + <% + encoded_id = trans.security.encode_id( invalid_repository_dependency.id ) + %> + <tr class="datasetRow" + %if parent is not None: + parent="${parent}" + %endif + id="libraryItem-${encoded_id}"> + <td style="padding-left: ${pad+20}px;"> + ${ invalid_repository_dependency.error | h } + </td> + </tr> + <% + my_row = row_counter.count + row_counter.increment() + %> +</%def> + <%def name="render_invalid_tool( invalid_tool, pad, parent, row_counter, valid=True )"><% encoded_id = trans.security.encode_id( invalid_tool.id ) %><tr class="datasetRow" @@ -344,6 +373,25 @@ %></%def> +<%def name="render_invalid_tool_dependency( invalid_tool_dependency, pad, parent, row_counter )"> + <% + encoded_id = trans.security.encode_id( invalid_tool_dependency.id ) + %> + <tr class="datasetRow" + %if parent is not None: + parent="${parent}" + %endif + id="libraryItem-${encoded_id}"> + <td style="padding-left: ${pad+20}px;"> + ${ invalid_tool_dependency.error | h } + </td> + </tr> + <% + my_row = row_counter.count + row_counter.increment() + %> +</%def> + <%def name="render_readme( readme, pad, parent, row_counter )"><% from galaxy.util.shed_util_common import to_safe_string @@ -581,15 +629,24 @@ datatypes_root_folder = containers_dict.get( 'datatypes', None ) invalid_tools_root_folder = containers_dict.get( 'invalid_tools', None ) + invalid_repository_dependencies_root_folder = containers_dict.get( 'invalid_repository_dependencies', None ) readme_files_root_folder = containers_dict.get( 'readme_files', None ) repository_dependencies_root_folder = containers_dict.get( 'repository_dependencies', None ) missing_repository_dependencies_root_folder = containers_dict.get( 'missing_repository_dependencies', None ) + invalid_tool_dependencies_root_folder = containers_dict.get( 'invalid_tool_dependencies', None ) tool_dependencies_root_folder = containers_dict.get( 'tool_dependencies', None ) missing_tool_dependencies_root_folder = containers_dict.get( 'missing_tool_dependencies', None ) valid_tools_root_folder = containers_dict.get( 'valid_tools', None ) workflows_root_folder = containers_dict.get( 'workflows', None ) has_contents = datatypes_root_folder or invalid_tools_root_folder or valid_tools_root_folder or workflows_root_folder + has_dependencies = \ + invalid_repository_dependencies_root_folder or \ + invalid_tool_dependencies_root_folder or \ + missing_repository_dependencies_root_folder or \ + repository_dependencies_root_folder or \ + tool_dependencies_root_folder or \ + missing_tool_dependencies_root_folder class RowCounter( object ): def __init__( self ): @@ -611,10 +668,17 @@ </div></div> %endif - %if missing_repository_dependencies_root_folder or repository_dependencies_root_folder or tool_dependencies_root_folder or missing_tool_dependencies_root_folder: + %if has_dependencies: <div class="toolForm"><div class="toolFormTitle">Dependencies of this repository</div><div class="toolFormBody"> + %if invalid_repository_dependencies_root_folder: + <p/> + <% row_counter = RowCounter() %> + <table cellspacing="2" cellpadding="2" border="0" width="100%" class="tables container-table" id="invalid_repository_dependencies"> + ${render_folder( invalid_repository_dependencies_root_folder, 0, parent=None, row_counter=row_counter, is_root_folder=True )} + </table> + %endif %if missing_repository_dependencies_root_folder: <p/><% row_counter = RowCounter() %> @@ -629,6 +693,13 @@ ${render_folder( repository_dependencies_root_folder, 0, parent=None, row_counter=row_counter, is_root_folder=True )} </table> %endif + %if invalid_tool_dependencies_root_folder: + <p/> + <% row_counter = RowCounter() %> + <table cellspacing="2" cellpadding="2" border="0" width="100%" class="tables container-table" id="invalid_tool_dependencies"> + ${render_folder( invalid_tool_dependencies_root_folder, 0, parent=None, row_counter=row_counter, is_root_folder=True )} + </table> + %endif %if tool_dependencies_root_folder: <p/><% row_counter = RowCounter() %> diff -r b9a5aee402b31c2a316de0f68efe66fdb141418e -r ec4da708e45175610a33d16fcfc68732e425f54f templates/webapps/community/repository/view_workflow.mako --- a/templates/webapps/community/repository/view_workflow.mako +++ b/templates/webapps/community/repository/view_workflow.mako @@ -21,7 +21,6 @@ browse_label = 'Browse or delete repository tip files' else: browse_label = 'Browse repository tip files' - has_readme = metadata and 'readme_files' in metadata # <li><a class="action-button" href="${h.url_for( controller='repository', action='install_repositories_by_revision', repository_ids=trans.security.encode_id( repository.id ), changeset_revisions=changeset_revision )}">Install repository to Galaxy</a></li> %> diff -r b9a5aee402b31c2a316de0f68efe66fdb141418e -r ec4da708e45175610a33d16fcfc68732e425f54f test/tool_shed/functional/test_0010_repository_with_tool_dependencies.py --- a/test/tool_shed/functional/test_0010_repository_with_tool_dependencies.py +++ b/test/tool_shed/functional/test_0010_repository_with_tool_dependencies.py @@ -98,7 +98,7 @@ uncompress_file=False, remove_repo_files_not_in_tar=False, commit_message='Uploaded invalid tool dependency XML.', - strings_displayed=[ 'Name, version and type from a tool requirement tag does not match' ], + strings_displayed=[ 'The settings for <b>name</b>, <b>version</b> and <b>type</b> from a contained tool configuration' ], strings_not_displayed=[] ) def test_0035_upload_valid_tool_dependency_xml( self ): '''Upload tool_dependencies.xml defining version 0.9.4_9696d0ce8a962f7bb61c4791be5ce44312b81cf8 of the freebayes package.''' diff -r b9a5aee402b31c2a316de0f68efe66fdb141418e -r ec4da708e45175610a33d16fcfc68732e425f54f test/tool_shed/functional/test_0030_repository_dependency_revisions.py --- a/test/tool_shed/functional/test_0030_repository_dependency_revisions.py +++ b/test/tool_shed/functional/test_0030_repository_dependency_revisions.py @@ -174,6 +174,7 @@ repository_metadata = [ ( metadata.metadata, metadata.changeset_revision ) for metadata in self.get_repository_metadata( repository ) ] datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) datatypes_tip = self.get_repository_tip( datatypes_repository ) + strings_displayed = [] # Iterate through all metadata revisions and check for repository dependencies. for metadata, changeset_revision in repository_metadata: # Add the dependency description and datatypes repository details to the strings to check. diff -r b9a5aee402b31c2a316de0f68efe66fdb141418e -r ec4da708e45175610a33d16fcfc68732e425f54f test/tool_shed/functional/test_0100_complex_repository_dependencies.py --- a/test/tool_shed/functional/test_0100_complex_repository_dependencies.py +++ b/test/tool_shed/functional/test_0100_complex_repository_dependencies.py @@ -44,7 +44,7 @@ uncompress_file=False, remove_repo_files_not_in_tar=False, commit_message='Uploaded tool_dependencies.xml.', - strings_displayed=[ 'Name, version and type from a tool requirement tag does not match' ], + strings_displayed=[ 'The settings for <b>name</b>, <b>version</b> and <b>type</b> from a contained tool' ], strings_not_displayed=[] ) self.display_manage_repository_page( repository, strings_displayed=[ 'Tool dependencies', 'may not be', 'in this repository' ] ) def test_0010_create_bwa_base_repository( self ): @@ -79,7 +79,7 @@ owner = tool_repository.user.username changeset_revision = self.get_repository_tip( tool_repository ) self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=True, package='bwa', version='0.5.9' ) - strings_displayed = [ 'Invalid tool shed <b>%s</b> defined' % url ] + strings_displayed = [ 'Repository dependencies are currently supported only within the same tool shed' ] self.upload_file( repository, filename='tool_dependencies.xml', filepath=dependency_path, @@ -100,7 +100,7 @@ owner = tool_repository.user.username changeset_revision = self.get_repository_tip( tool_repository ) self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=True, package='bwa', version='0.5.9' ) - strings_displayed = [ 'Invalid repository name <b>%s</b> defined.' % name ] + strings_displayed = [ 'because the name is invalid' ] self.upload_file( base_repository, filename='tool_dependencies.xml', filepath=dependency_path, @@ -121,7 +121,7 @@ owner = 'invalid_owner!?' changeset_revision = self.get_repository_tip( tool_repository ) self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=True, package='bwa', version='0.5.9' ) - strings_displayed = [ 'Invalid owner <b>%s</b> defined' % owner ] + strings_displayed = [ 'because the owner is invalid.' ] self.upload_file( base_repository, filename='tool_dependencies.xml', filepath=dependency_path, @@ -142,7 +142,7 @@ owner = tool_repository.user.username changeset_revision = '1234abcd' self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=True, package='bwa', version='0.5.9' ) - strings_displayed = [ 'Invalid changeset revision <b>%s</b> defined.' % changeset_revision ] + strings_displayed = [ 'because the changeset revision is invalid.' ] self.upload_file( base_repository, filename='tool_dependencies.xml', filepath=dependency_path, diff -r b9a5aee402b31c2a316de0f68efe66fdb141418e -r ec4da708e45175610a33d16fcfc68732e425f54f test/tool_shed/functional/test_0110_invalid_simple_repository_dependencies.py --- a/test/tool_shed/functional/test_0110_invalid_simple_repository_dependencies.py +++ b/test/tool_shed/functional/test_0110_invalid_simple_repository_dependencies.py @@ -82,7 +82,7 @@ owner = repository.user.username changeset_revision = self.get_repository_tip( repository ) self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=False, description='This is invalid.' ) - strings_displayed = [ 'Invalid tool shed <b>%s</b> defined for repository <b>%s</b>' % ( url, repository.name ) ] + strings_displayed = [ 'Repository dependencies are currently supported only within the same tool shed' ] self.upload_file( emboss_repository, filename='repository_dependencies.xml', filepath=dependency_path, @@ -103,7 +103,7 @@ owner = repository.user.username changeset_revision = self.get_repository_tip( repository ) self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=False, description='This is invalid.' ) - strings_displayed = [ 'Invalid repository name <b>%s</b> defined.' % name ] + strings_displayed = [ 'because the name is invalid.' ] self.upload_file( emboss_repository, filename='repository_dependencies.xml', filepath=dependency_path, @@ -124,7 +124,7 @@ owner = '!?invalid?!' changeset_revision = self.get_repository_tip( repository ) self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=False, description='This is invalid.' ) - strings_displayed = [ 'Invalid owner <b>%s</b> defined for repository <b>%s</b>' % ( owner, repository.name ) ] + strings_displayed = [ 'because the owner is invalid.' ] self.upload_file( emboss_repository, filename='repository_dependencies.xml', filepath=dependency_path, @@ -145,7 +145,7 @@ owner = repository.user.username changeset_revision = '!?invalid?!' self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=False, description='This is invalid.' ) - strings_displayed = [ 'Invalid changeset revision <b>%s</b> defined.' % changeset_revision ] + strings_displayed = [ 'because the changeset revision is invalid.' ] self.upload_file( emboss_repository, filename='repository_dependencies.xml', filepath=dependency_path, diff -r b9a5aee402b31c2a316de0f68efe66fdb141418e -r ec4da708e45175610a33d16fcfc68732e425f54f test/tool_shed/functional/test_1010_install_repository_with_tool_dependencies.py --- a/test/tool_shed/functional/test_1010_install_repository_with_tool_dependencies.py +++ b/test/tool_shed/functional/test_1010_install_repository_with_tool_dependencies.py @@ -80,7 +80,7 @@ uncompress_file=False, remove_repo_files_not_in_tar=False, commit_message='Uploaded invalid tool dependency XML.', - strings_displayed=[ 'Name, version and type from a tool requirement tag does not match' ], + strings_displayed=[ 'The settings for <b>name</b>, <b>version</b> and <b>type</b> from a contained tool configuration' ], strings_not_displayed=[] ) self.upload_file( repository, filename=os.path.join( 'freebayes', 'tool_dependencies.xml' ), diff -r b9a5aee402b31c2a316de0f68efe66fdb141418e -r ec4da708e45175610a33d16fcfc68732e425f54f test/tool_shed/functional/test_1100_install_repository_with_complex_dependencies.py --- a/test/tool_shed/functional/test_1100_install_repository_with_complex_dependencies.py +++ b/test/tool_shed/functional/test_1100_install_repository_with_complex_dependencies.py @@ -53,7 +53,7 @@ uncompress_file=False, remove_repo_files_not_in_tar=False, commit_message='Uploaded tool_dependencies.xml.', - strings_displayed=[ 'Name, version and type from a tool requirement tag does not match' ], + strings_displayed=[ 'The settings for <b>name</b>, <b>version</b> and <b>type</b> from a contained tool' ], strings_not_displayed=[] ) self.display_manage_repository_page( repository, strings_displayed=[ 'Tool dependencies', 'may not be', 'in this repository' ] ) def test_0010_create_bwa_base_repository( self ): @@ -92,7 +92,7 @@ owner = tool_repository.user.username changeset_revision = self.get_repository_tip( tool_repository ) self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=True, package='bwa', version='0.5.9' ) - strings_displayed = [ 'Invalid tool shed <b>%s</b> defined' % url ] + strings_displayed = [ 'Repository dependencies are currently supported only within the same tool shed' ] self.upload_file( repository, filename='tool_dependencies.xml', filepath=dependency_path, @@ -115,7 +115,7 @@ owner = tool_repository.user.username changeset_revision = self.get_repository_tip( tool_repository ) self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=True, package='bwa', version='0.5.9' ) - strings_displayed = [ 'Invalid repository name <b>%s</b> defined.' % name ] + strings_displayed = [ 'because the name is invalid.' ] self.upload_file( base_repository, filename='tool_dependencies.xml', filepath=dependency_path, @@ -138,7 +138,7 @@ owner = 'invalid_owner!?' changeset_revision = self.get_repository_tip( tool_repository ) self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=True, package='bwa', version='0.5.9' ) - strings_displayed = [ 'Invalid owner <b>%s</b> defined' % owner ] + strings_displayed = [ 'because the owner is invalid.' ] self.upload_file( base_repository, filename='tool_dependencies.xml', filepath=dependency_path, @@ -161,7 +161,7 @@ owner = tool_repository.user.username changeset_revision = '1234abcd' self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=True, package='bwa', version='0.5.9' ) - strings_displayed = [ 'Invalid changeset revision <b>%s</b> defined.' % changeset_revision ] + strings_displayed = [ 'because the changeset revision is invalid.' ] self.upload_file( base_repository, filename='tool_dependencies.xml', filepath=dependency_path, diff -r b9a5aee402b31c2a316de0f68efe66fdb141418e -r ec4da708e45175610a33d16fcfc68732e425f54f test/tool_shed/functional/test_1110_install_repository_with_invalid_repository_dependency.py --- a/test/tool_shed/functional/test_1110_install_repository_with_invalid_repository_dependency.py +++ b/test/tool_shed/functional/test_1110_install_repository_with_invalid_repository_dependency.py @@ -90,7 +90,7 @@ owner = repository.user.username changeset_revision = self.get_repository_tip( repository ) self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=False, description='This is invalid.' ) - strings_displayed = [ 'Invalid tool shed <b>%s</b> defined for repository <b>%s</b>' % ( url, repository.name ) ] + strings_displayed = [ 'Repository dependencies are currently supported only within the same tool shed' ] self.upload_file( emboss_repository, filename='repository_dependencies.xml', filepath=dependency_path, @@ -113,7 +113,7 @@ owner = repository.user.username changeset_revision = self.get_repository_tip( repository ) self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=False, description='This is invalid.' ) - strings_displayed = [ 'Invalid repository name <b>%s</b> defined.' % name ] + strings_displayed = [ 'because the name is invalid.' ] self.upload_file( emboss_repository, filename='repository_dependencies.xml', filepath=dependency_path, @@ -136,7 +136,7 @@ owner = '!?invalid?!' changeset_revision = self.get_repository_tip( repository ) self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=False, description='This is invalid.' ) - strings_displayed = [ 'Invalid owner <b>%s</b> defined for repository <b>%s</b>' % ( owner, repository.name ) ] + strings_displayed = [ 'because the owner is invalid' ] self.upload_file( emboss_repository, filename='repository_dependencies.xml', filepath=dependency_path, @@ -159,7 +159,7 @@ owner = repository.user.username changeset_revision = '!?invalid?!' self.generate_invalid_dependency_xml( xml_filename, url, name, owner, changeset_revision, complex=False, description='This is invalid.' ) - strings_displayed = [ 'Invalid changeset revision <b>%s</b> defined.' % changeset_revision ] + strings_displayed = [ 'because the changeset revision is invalid.' ] self.upload_file( emboss_repository, filename='repository_dependencies.xml', filepath=dependency_path, @@ -174,7 +174,7 @@ self.galaxy_logout() self.galaxy_login( email=common.admin_email, username=common.admin_username ) repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) - preview_strings_displayed = [ 'emboss_0110', self.get_repository_tip( repository ), 'will be ignored' ] + preview_strings_displayed = [ 'emboss_0110', self.get_repository_tip( repository ), 'Ignoring repository dependency definition' ] self.install_repository( emboss_repository_name, common.test_user_1_name, category_name, diff -r b9a5aee402b31c2a316de0f68efe66fdb141418e -r ec4da708e45175610a33d16fcfc68732e425f54f test/tool_shed/functional/test_1210_uninstall_reinstall_repository_with_tool_dependencies.py --- a/test/tool_shed/functional/test_1210_uninstall_reinstall_repository_with_tool_dependencies.py +++ b/test/tool_shed/functional/test_1210_uninstall_reinstall_repository_with_tool_dependencies.py @@ -74,7 +74,7 @@ uncompress_file=False, remove_repo_files_not_in_tar=False, commit_message='Uploaded invalid tool dependency XML.', - strings_displayed=[ 'Name, version and type from a tool requirement tag does not match' ], + strings_displayed=[ 'The settings for <b>name</b>, <b>version</b> and <b>type</b> from a contained tool configuration' ], strings_not_displayed=[] ) self.upload_file( repository, filename=os.path.join( 'freebayes', 'tool_dependencies.xml' ), Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.