1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f86fcd8dd3c3/ Changeset: f86fcd8dd3c3 User: greg Date: 2014-03-30 22:25:25 Summary: Fix for installing a repository into Galaxy that contains a tool to be loaded into the tool panel where another version in the tool's lineage chain is already loaded into the tool panel. An additional entry will no longer be made. Instead, a single entry will exist for the entire lineage chain. Affected #: 5 files diff -r f83f4d9965283e8dc8640698262da52080081496 -r f86fcd8dd3c350b6cfe3d0c5e0a809bab5e62469 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -63,7 +63,9 @@ from galaxy.web.form_builder import SelectField from galaxy.model.item_attrs import Dictifiable from galaxy.model import Workflow -from tool_shed.util import shed_util_common +from tool_shed.util import common_util +from tool_shed.util import encoding_util +from tool_shed.util import shed_util_common as suc from .loader import load_tool, template_macro_params from .wrappers import ( ToolParameterValueWrapper, @@ -227,9 +229,10 @@ return shed_config_dict return default - def __add_tool_to_tool_panel( self, tool_id, panel_component, section=False ): + def __add_tool_to_tool_panel( self, tool, panel_component, section=False ): # See if a version of this tool is already loaded into the tool panel. The value of panel_component # will be a ToolSection (if the value of section=True) or self.tool_panel (if section=False). + tool_id = str( tool.id ) tool = self.tools_by_id[ tool_id ] if section: panel_dict = panel_component.elems @@ -244,7 +247,14 @@ if loaded_version_key in panel_dict: already_loaded = True break - if not already_loaded: + if already_loaded: + if tool.lineage_ids.index( tool_id ) > tool.lineage_ids.index( lineage_id ): + key = 'tool_%s' % tool.id + index = panel_dict.keys().index( loaded_version_key ) + del panel_dict[ loaded_version_key ] + panel_dict.insert( index, key, tool ) + log.debug( "Loaded tool id: %s, version: %s into tool panel." % ( tool.id, tool.version ) ) + else: inserted = False key = 'tool_%s' % tool.id # The value of panel_component is the in-memory tool panel dictionary. @@ -253,22 +263,62 @@ panel_dict.insert( index, key, tool ) inserted = True if not inserted: - # If the tool is not defined in integrated_tool_panel.xml, append it to the tool panel. - panel_dict[ key ] = tool - log.debug( "Loaded tool id: %s, version: %s into tool panel." % ( tool.id, tool.version ) ) - elif tool.lineage_ids.index( tool_id ) > tool.lineage_ids.index( lineage_id ): - key = 'tool_%s' % tool.id - index = panel_dict.keys().index( loaded_version_key ) - del panel_dict[ loaded_version_key ] - panel_dict.insert( index, key, tool ) - log.debug( "Loaded tool id: %s, version: %s into tool panel." % ( tool.id, tool.version ) ) + if tool.guid is None or \ + tool.tool_shed is None or \ + tool.repository_name is None or \ + tool.repository_owner is None or \ + tool.installed_changeset_revision is None: + # We have a tool that was not installed from the Tool Shed, but is also not yet defined in + # integrated_tool_panel.xml, so append it to the tool panel. + panel_dict[ key ] = tool + log.debug( "Loaded tool id: %s, version: %s into tool panel.." % ( tool.id, tool.version ) ) + else: + # We are in the process of installing the tool. The version lineage chain will not yet be + # built on the Galaxy side, so we have to get it from the Tool Shed. + tool_shed_url = suc.get_url_from_tool_shed( self.app, str( tool.tool_shed ) ) + # get_versions_of_tool( self, trans, name, owner, changeset_revision, encoded_guid ) + params = '?name=%s&owner=%s&changeset_revision=%s&encoded_guid=%s' % \ + ( str( tool.repository_name ), + str( tool.repository_owner ), + str( tool.installed_changeset_revision ), + encoding_util.tool_shed_encode( tool.guid ) ) + url = suc.url_join( tool_shed_url, 'repository/get_versions_of_tool%s' % params ) + try: + # For backward compatibility - some versions of the Tool Shed will not have the + # get_versions_of_tool() method. + raw_text = common_util.tool_shed_get( self.app, tool_shed_url, url ) + tool_lineage_ids = json.loads( raw_text ) + except Exception, e: + log.exception( "Error with url\n%s\n%s" % ( str( url ), str( e ) ) ) + tool_lineage_ids = [] + for lineage_id in tool_lineage_ids: + if lineage_id in self.tools_by_id: + loaded_version_key = 'tool_%s' % lineage_id + if loaded_version_key in panel_dict: + if not already_loaded: + already_loaded = True + # Even though a version of the tool is loaded in the tool panel, we need to find out if it + # is the most recent version. + key = 'tool_%s' % tool.guid + index = panel_dict.keys().index( loaded_version_key ) + if loaded_version_key > key: + index = panel_dict.keys().index( loaded_version_key ) + del panel_dict[ loaded_version_key ] + panel_dict.insert( index, key, tool ) + log.debug( "Loaded tool id: %s, version: %s into tool panel..." % ( tool.id, tool.version ) ) + inserted = True + break + if not already_loaded: + # If the tool is not defined in integrated_tool_panel.xml, append it to the tool panel. + panel_dict[ key ] = tool + log.debug( "Loaded tool id: %s, version: %s into tool panel...." % ( tool.id, tool.version ) ) def load_tool_panel( self ): for key, val in self.integrated_tool_panel.items(): if isinstance( val, Tool ): tool_id = key.replace( 'tool_', '', 1 ) if tool_id in self.tools_by_id: - self.__add_tool_to_tool_panel( tool_id, self.tool_panel, section=False ) + self.__add_tool_to_tool_panel( val, self.tool_panel, section=False ) elif isinstance( val, Workflow ): workflow_id = key.replace( 'workflow_', '', 1 ) if workflow_id in self.workflows_by_id: @@ -288,7 +338,7 @@ if isinstance( section_val, Tool ): tool_id = section_key.replace( 'tool_', '', 1 ) if tool_id in self.tools_by_id: - self.__add_tool_to_tool_panel( tool_id, section, section=True ) + self.__add_tool_to_tool_panel( section_val, section, section=True ) elif isinstance( section_val, Workflow ): workflow_id = section_key.replace( 'workflow_', '', 1 ) if workflow_id in self.workflows_by_id: @@ -431,8 +481,8 @@ def __get_tool_version( self, tool_id ): """Return a ToolVersion if one exists for the tool_id""" return self.app.install_model.context.query( self.app.install_model.ToolVersion ) \ - .filter( self.app.install_model.ToolVersion.table.c.tool_id == tool_id ) \ - .first() + .filter( self.app.install_model.ToolVersion.table.c.tool_id == tool_id ) \ + .first() def __get_tool_shed_repository( self, tool_shed, name, owner, installed_changeset_revision ): return self.app.install_model.context.query( self.app.install_model.ToolShedRepository ) \ @@ -546,14 +596,15 @@ tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id ) self.sa_session.add( tta ) self.sa_session.flush() - #if tool.id not in self.tools_by_id: - # Allow for the same tool to be loaded into multiple places in the tool panel. We have to handle the case where the tool is contained - # in a repository installed from the tool shed, and the Galaxy administrator has retrieved updates to the installed repository. In this - # case, the tool may have been updated, but the version was not changed, so the tool should always be reloaded here. We used to only load - # the tool if it's it was not found in self.tools_by_id, but performing that check did not enable this scenario. + # Allow for the same tool to be loaded into multiple places in the tool panel. We have to handle + # the case where the tool is contained in a repository installed from the tool shed, and the Galaxy + # administrator has retrieved updates to the installed repository. In this case, the tool may have + # been updated, but the version was not changed, so the tool should always be reloaded here. We used + # to only load the tool if it's it was not found in self.tools_by_id, but performing that check did + # not enable this scenario. self.tools_by_id[ tool.id ] = tool if load_panel_dict: - self.__add_tool_to_tool_panel( tool.id, panel_dict, section=isinstance( panel_dict, ToolSection ) ) + self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) ) # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. if key in integrated_panel_dict or index is None: integrated_panel_dict[ key ] = tool @@ -1020,8 +1071,8 @@ def tool_version( self ): """Return a ToolVersion if one exists for our id""" return self.app.install_model.context.query( self.app.install_model.ToolVersion ) \ - .filter( self.app.install_model.ToolVersion.table.c.tool_id == self.id ) \ - .first() + .filter( self.app.install_model.ToolVersion.table.c.tool_id == self.id ) \ + .first() @property def tool_versions( self ): @@ -1043,11 +1094,11 @@ def tool_shed_repository( self ): # If this tool is included in an installed tool shed repository, return it. if self.tool_shed: - return shed_util_common.get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( self.app, - self.tool_shed, - self.repository_name, - self.repository_owner, - self.installed_changeset_revision ) + return suc.get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( self.app, + self.tool_shed, + self.repository_name, + self.repository_owner, + self.installed_changeset_revision ) return None def __get_job_tool_configuration(self, job_params=None): @@ -1359,7 +1410,7 @@ lock = threading.Lock() lock.acquire( True ) try: - self.help.text = shed_util_common.set_image_paths( self.app, self.repository_id, self.help.text ) + self.help.text = suc.set_image_paths( self.app, self.repository_id, self.help.text ) except Exception, e: log.exception( "Exception in parse_help, so images may not be properly displayed:\n%s" % str( e ) ) finally: diff -r f83f4d9965283e8dc8640698262da52080081496 -r f86fcd8dd3c350b6cfe3d0c5e0a809bab5e62469 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -405,10 +405,6 @@ repo_information_dict = json.from_json_string( raw_text ) return repo_information_dict - def get_versions_of_tool( self, app, guid ): - tool_version = tool_util.get_tool_version( app, guid ) - return tool_version.get_version_ids( app, reverse=True ) - @web.expose @web.require_admin def import_workflow( self, trans, workflow_name, repository_id, **kwd ): @@ -1973,7 +1969,8 @@ tool_config = os.path.join( shed_config_dict.get( 'tool_path' ), tool_config ) tool = trans.app.toolbox.load_tool( os.path.abspath( tool_config ), guid=tool_metadata[ 'guid' ] ) if tool: - tool_lineage = self.get_versions_of_tool( trans.app, tool.id ) + tool_version = tool_util.get_tool_version( trans.app, str( tool.id ) ) + tool_lineage = tool_version.get_version_ids( trans.app, reverse=True ) break return trans.fill_template( "/admin/tool_shed_repository/view_tool_metadata.mako", repository=repository, diff -r f83f4d9965283e8dc8640698262da52080081496 -r f86fcd8dd3c350b6cfe3d0c5e0a809bab5e62469 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -89,12 +89,13 @@ def browse_categories( self, trans, **kwd ): # The request came from the tool shed. if 'f-free-text-search' in kwd: - # Trick to enable searching repository name, description from the CategoryGrid. What we've done is rendered the search box for the - # RepositoryGrid on the grid.mako template for the CategoryGrid. See ~/templates/webapps/tool_shed/category/grid.mako. Since we - # are searching repositories and not categories, redirect to browse_repositories(). + # Trick to enable searching repository name, description from the CategoryGrid. + # What we've done is rendered the search box for the RepositoryGrid on the grid.mako + # template for the CategoryGrid. See ~/templates/webapps/tool_shed/category/grid.mako. + # Since we are searching repositories and not categories, redirect to browse_repositories(). if 'id' in kwd and 'f-free-text-search' in kwd and kwd[ 'id' ] == kwd[ 'f-free-text-search' ]: - # The value of 'id' has been set to the search string, which is a repository name. We'll try to get the desired encoded repository - # id to pass on. + # The value of 'id' has been set to the search string, which is a repository name. + # We'll try to get the desired encoded repository id to pass on. try: repository_name = kwd[ 'id' ] repository = suc.get_repository_by_name( trans.app, repository_name ) @@ -1906,41 +1907,35 @@ readme_files_dict=readme_files_dict, repo_info_dict=repo_info_dict ) - def get_versions_of_tool( self, trans, repository, repository_metadata, guid ): + @web.json + def get_versions_of_tool( self, trans, name, owner, changeset_revision, encoded_guid ): + #def get_versions_of_tool( self, trans, repository_id, repository_metadata, guid ): """ - Return the tool lineage in descendant order for the received guid contained in the received - repsitory_metadata.tool_versions. + Return the tool version lineage chain in descending order for the received guid contained + in the repsitory_metadata.tool_versions column associated with the received changeset_revision + whose associated repository is associated with the received name and owner. """ - encoded_id = trans.security.encode_id( repository.id ) - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( suc.get_configured_ui(), repo_dir ) - # Initialize the tool lineage - tool_guid_lineage = [ guid ] - # Get all ancestor guids of the received guid. - current_child_guid = guid - for changeset in suc.reversed_upper_bounded_changelog( repo, repository_metadata.changeset_revision ): - ctx = repo.changectx( changeset ) - rm = suc.get_repository_metadata_by_changeset_revision( trans, encoded_id, str( ctx ) ) - if rm: - parent_guid = rm.tool_versions.get( current_child_guid, None ) - if parent_guid: - tool_guid_lineage.append( parent_guid ) - current_child_guid = parent_guid - # Get all descendant guids of the received guid. - current_parent_guid = guid - for changeset in suc.reversed_lower_upper_bounded_changelog( repo, - repository_metadata.changeset_revision, - repository.tip( trans.app ) ): - ctx = repo.changectx( changeset ) - rm = suc.get_repository_metadata_by_changeset_revision( trans, encoded_id, str( ctx ) ) - if rm: - tool_versions = rm.tool_versions - for child_guid, parent_guid in tool_versions.items(): - if parent_guid == current_parent_guid: - tool_guid_lineage.insert( 0, child_guid ) - current_parent_guid = child_guid - break - return tool_guid_lineage + guid = encoding_util.tool_shed_decode( encoded_guid ) + repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + version_lineage = [] + if repository: + repository_id = trans.security.encode_id( repository.id ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_id, + changeset_revision ) + if not repository_metadata: + repo_dir = repository.repo_path( trans.app ) + repo = hg.repository( suc.get_configured_ui(), repo_dir ) + updated_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, + repo, + changeset_revision ) + if updated_changeset_revision: + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_id, + updated_changeset_revision ) + if repository_metadata: + version_lineage = tool_util.get_version_lineage_for_tool( trans, repository_id, repository_metadata, guid ) + return version_lineage @web.expose def help( self, trans, **kwd ): @@ -3284,29 +3279,35 @@ guid = tool_metadata_dict[ 'guid' ] full_path_to_tool_config = os.path.abspath( relative_path_to_tool_config ) full_path_to_dir, tool_config_filename = os.path.split( full_path_to_tool_config ) - can_use_disk_file = tool_util.can_use_tool_config_disk_file( trans, repository, repo, full_path_to_tool_config, changeset_revision ) + can_use_disk_file = tool_util.can_use_tool_config_disk_file( trans, + repository, + repo, + full_path_to_tool_config, + changeset_revision ) if can_use_disk_file: trans.app.config.tool_data_path = work_dir - tool, valid, message, sample_files = tool_util.handle_sample_files_and_load_tool_from_disk( trans, - repo_files_dir, - repository_id, - full_path_to_tool_config, - work_dir ) + tool, valid, message, sample_files = \ + tool_util.handle_sample_files_and_load_tool_from_disk( trans, + repo_files_dir, + repository_id, + full_path_to_tool_config, + work_dir ) if message: status = 'error' else: - tool, message, sample_files = tool_util.handle_sample_files_and_load_tool_from_tmp_config( trans, - repo, - repository_id, - changeset_revision, - tool_config_filename, - work_dir ) + tool, message, sample_files = \ + tool_util.handle_sample_files_and_load_tool_from_tmp_config( trans, + repo, + repository_id, + changeset_revision, + tool_config_filename, + work_dir ) if message: status = 'error' suc.remove_dir( work_dir ) break if guid: - tool_lineage = self.get_versions_of_tool( trans, repository, repository_metadata, guid ) + tool_lineage = tool_util.get_version_lineage_for_tool( trans, repository_id, repository_metadata, guid ) else: repository_metadata_id = None metadata = None diff -r f83f4d9965283e8dc8640698262da52080081496 -r f86fcd8dd3c350b6cfe3d0c5e0a809bab5e62469 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -1180,9 +1180,9 @@ def get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ): """Get metadata for a specified repository change set from the database.""" - # Make sure there are no duplicate records, and return the single unique record for the changeset_revision. Duplicate records were somehow - # created in the past. The cause of this issue has been resolved, but we'll leave this method as is for a while longer to ensure all duplicate - # records are removed. + # Make sure there are no duplicate records, and return the single unique record for the changeset_revision. + # Duplicate records were somehow created in the past. The cause of this issue has been resolved, but we'll + # leave this method as is for a while longer to ensure all duplicate records are removed. all_metadata_records = trans.sa_session.query( trans.model.RepositoryMetadata ) \ .filter( and_( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ), trans.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) ) \ diff -r f83f4d9965283e8dc8640698262da52080081496 -r f86fcd8dd3c350b6cfe3d0c5e0a809bab5e62469 lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -524,6 +524,42 @@ app.install_model.ToolVersionAssociation.table.c.tool_id == tool_version.id ) ) \ .first() +def get_version_lineage_for_tool( trans, repository_id, repository_metadata, guid ): + """ + Return the tool version lineage chain in descendant order for the received guid contained in the received + repsitory_metadata.tool_versions. + """ + repository = suc.get_repository_by_id( trans, repository_id ) + repo_dir = repository.repo_path( trans.app ) + repo = hg.repository( suc.get_configured_ui(), repo_dir ) + # Initialize the tool lineage + version_lineage = [ guid ] + # Get all ancestor guids of the received guid. + current_child_guid = guid + for changeset in suc.reversed_upper_bounded_changelog( repo, repository_metadata.changeset_revision ): + ctx = repo.changectx( changeset ) + rm = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, str( ctx ) ) + if rm: + parent_guid = rm.tool_versions.get( current_child_guid, None ) + if parent_guid: + version_lineage.append( parent_guid ) + current_child_guid = parent_guid + # Get all descendant guids of the received guid. + current_parent_guid = guid + for changeset in suc.reversed_lower_upper_bounded_changelog( repo, + repository_metadata.changeset_revision, + repository.tip( trans.app ) ): + ctx = repo.changectx( changeset ) + rm = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, str( ctx ) ) + if rm: + tool_versions = rm.tool_versions + for child_guid, parent_guid in tool_versions.items(): + if parent_guid == current_parent_guid: + version_lineage.insert( 0, child_guid ) + current_parent_guid = child_guid + break + return version_lineage + def handle_missing_data_table_entry( app, relative_install_dir, tool_path, repository_tools_tups ): """ Inspect each tool to see if any have input parameters that are dynamically generated select lists that require entries in the Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.