1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/e6464387ed3f/ changeset: e6464387ed3f user: greg date: 2012-02-03 21:13:28 summary: Add the ability to chain Galaxy tools into a lineage of tool versions. This enables workflows and the rerun button on history items to work when a new version of a tool is loaded into the Galaxy tool panel. Version information for tools included in tool shed repositories is maintained for each changeset revision in the tool shed, and this information can be retrieved from the Administrative "Manage installed tool shed repositories" menu item for installed tool shed repositories that contain tools. This new functionality eliminates the need for the tool_id_guid_map table and associated features, so they have all been removed. affected #: 17 files diff -r f87dff286e86368fb33032ac567a5c9368dc990b -r e6464387ed3fce7f235cf5c9af1281da14a32811 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2687,16 +2687,46 @@ def includes_workflows( self ): return 'workflows' in self.metadata -class ToolIdGuidMap( object ): - def __init__( self, id=None, create_time=None, tool_id=None, tool_version=None, tool_shed=None, repository_owner=None, repository_name=None, guid=None ): +class ToolVersion( object ): + def __init__( self, id=None, create_time=None, tool_id=None, tool_shed_repository=None ): self.id = id self.create_time = create_time self.tool_id = tool_id - self.tool_version = tool_version - self.tool_shed = tool_shed - self.repository_owner = repository_owner - self.repository_name = repository_name - self.guid = guid + self.tool_shed_repository = tool_shed_repository + def get_versions( self, app ): + sa_session = app.model.context.current + tool_versions = [] + # Prepend ancestors. + def __ancestors( tool_version ): + # Should we handle multiple parents at each level? + previous_tva = tool_version.previous_version + if previous_tva: + parent_version = previous_tva[0].parent_version + if parent_version not in tool_versions: + tool_versions.insert( 0, parent_version ) + __ancestors( parent_version ) + # Append descendants. + def __descendants( tool_version ): + # Should we handle multiple child siblings at each level? + next_tva = sa_session.query( app.model.ToolVersionAssociation ) \ + .filter( app.model.ToolVersionAssociation.table.c.parent_id == tool_version.id ) \ + .first() + if next_tva: + current_version = next_tva.tool_version + if current_version not in tool_versions: + tool_versions.append( current_version ) + __descendants( current_version ) + __ancestors( self ) + __descendants( self ) + return tool_versions + def get_version_ids( self, app ): + return [ tool_version.tool_id for tool_version in self.get_versions( app ) ] + +class ToolVersionAssociation( object ): + def __init__( self, id=None, tool_id=None, parent_id=None ): + self.id = id + self.tool_id = tool_id + self.parent_id = parent_id ## ---- Utility methods ------------------------------------------------------- diff -r f87dff286e86368fb33032ac567a5c9368dc990b -r e6464387ed3fce7f235cf5c9af1281da14a32811 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -382,16 +382,17 @@ Column( "uninstalled", Boolean, default=False ), Column( "dist_to_shed", Boolean, default=False ) ) -ToolIdGuidMap.table = Table( "tool_id_guid_map", metadata, +ToolVersion.table = Table( "tool_version", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "tool_id", String( 255 ) ), - Column( "tool_version", TEXT ), - Column( "tool_shed", TrimmedString( 255 ) ), - Column( "repository_owner", TrimmedString( 255 ) ), - Column( "repository_name", TrimmedString( 255 ) ), - Column( "guid", TEXT, index=True, unique=True ) ) + Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True, nullable=True ) ) + +ToolVersionAssociation.table = Table( "tool_version_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "tool_id", Integer, ForeignKey( "tool_version.id" ), index=True, nullable=False ), + Column( "parent_id", Integer, ForeignKey( "tool_version.id" ), index=True, nullable=False ) ) Job.table = Table( "job", metadata, Column( "id", Integer, primary_key=True ), @@ -1619,9 +1620,20 @@ ratings=relation( PageRatingAssociation, order_by=PageRatingAssociation.table.c.id, backref="pages" ) ) ) -assign_mapper( context, ToolShedRepository, ToolShedRepository.table ) +assign_mapper( context, ToolShedRepository, ToolShedRepository.table, + properties=dict( tool_versions=relation( ToolVersion, + primaryjoin=( ToolShedRepository.table.c.id == ToolVersion.table.c.tool_shed_repository_id ), + backref='tool_shed_repository' ) ) ) -assign_mapper( context, ToolIdGuidMap, ToolIdGuidMap.table ) +assign_mapper( context, ToolVersion, ToolVersion.table ) + +assign_mapper( context, ToolVersionAssociation, ToolVersionAssociation.table, + properties=dict( tool_version=relation( ToolVersion, + primaryjoin=( ToolVersionAssociation.table.c.tool_id == ToolVersion.table.c.id ), + backref='current_version' ), + parent_version=relation( ToolVersion, + primaryjoin=( ToolVersionAssociation.table.c.parent_id == ToolVersion.table.c.id ), + backref='previous_version' ) ) ) # Set up proxy so that # Page.users_shared_with diff -r f87dff286e86368fb33032ac567a5c9368dc990b -r e6464387ed3fce7f235cf5c9af1281da14a32811 lib/galaxy/model/migrate/versions/0091_add_tool_version_tables.py --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0091_add_tool_version_tables.py @@ -0,0 +1,119 @@ +""" +Migration script to create the tool_version and tool_version_association tables and drop the tool_id_guid_map table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * +from galaxy.util.json import from_json_string, to_json_string + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def nextval( table, col='id' ): + if migrate_engine.name == 'postgres': + return "nextval('%s_%s_seq')" % ( table, col ) + elif migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + return "null" + else: + raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name ) + +def localtimestamp(): + if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql': + return "LOCALTIMESTAMP" + elif migrate_engine.name == 'sqlite': + return "current_date || ' ' || current_time" + else: + raise Exception( 'Unable to convert data for unknown database type: %s' % db ) + +ToolVersion_table = Table( "tool_version", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_id", String( 255 ) ), + Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True, nullable=True ) ) + +ToolVersionAssociation_table = Table( "tool_version_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "tool_id", Integer, ForeignKey( "tool_version.id" ), index=True, nullable=False ), + Column( "parent_id", Integer, ForeignKey( "tool_version.id" ), index=True, nullable=False ) ) + +def upgrade(): + print __doc__ + + ToolIdGuidMap_table = Table( "tool_id_guid_map", metadata, autoload=True ) + + metadata.reflect() + # Create the tables. + try: + ToolVersion_table.create() + except Exception, e: + log.debug( "Creating tool_version table failed: %s" % str( e ) ) + try: + ToolVersionAssociation_table.create() + except Exception, e: + log.debug( "Creating tool_version_association table failed: %s" % str( e ) ) + # Populate the tool table with tools included in installed tool shed repositories. + cmd = "SELECT id, metadata FROM tool_shed_repository" + result = db_session.execute( cmd ) + count = 0 + for row in result: + if row[1]: + tool_shed_repository_id = row[0] + repository_metadata = from_json_string( str( row[1] ) ) + # Create a new row in the tool table for each tool included in repository. We will NOT + # handle tool_version_associaions because we do not have the information we need to do so. + tools = repository_metadata.get( 'tools', [] ) + for tool_dict in tools: + cmd = "INSERT INTO tool_version VALUES (%s, %s, %s, '%s', %s)" % \ + ( nextval( 'tool_version' ), localtimestamp(), localtimestamp(), tool_dict[ 'guid' ], tool_shed_repository_id ) + db_session.execute( cmd ) + count += 1 + print "Added %d rows to the new tool_version table." % count + # Drop the tool_id_guid_map table since the 2 new tables render it unnecessary. + try: + ToolIdGuidMap_table.drop() + except Exception, e: + log.debug( "Dropping tool_id_guid_map table failed: %s" % str( e ) ) + +def downgrade(): + + ToolIdGuidMap_table = Table( "tool_id_guid_map", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_id", String( 255 ) ), + Column( "tool_version", TEXT ), + Column( "tool_shed", TrimmedString( 255 ) ), + Column( "repository_owner", TrimmedString( 255 ) ), + Column( "repository_name", TrimmedString( 255 ) ), + Column( "guid", TEXT, index=True, unique=True ) ) + + metadata.reflect() + try: + ToolVersionAssociation_table.drop() + except Exception, e: + log.debug( "Dropping tool_version_association table failed: %s" % str( e ) ) + try: + ToolVersion_table.drop() + except Exception, e: + log.debug( "Dropping tool_version table failed: %s" % str( e ) ) + try: + ToolIdGuidMap_table.create() + except Exception, e: + log.debug( "Creating tool_id_guid_map table failed: %s" % str( e ) ) diff -r f87dff286e86368fb33032ac567a5c9368dc990b -r e6464387ed3fce7f235cf5c9af1281da14a32811 lib/galaxy/tool_shed/install_manager.py --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -4,7 +4,9 @@ shed. Tools included in tool_shed_install.xml that have already been installed will not be re-installed. """ +import urllib2 from galaxy.tools import ToolSection +from galaxy.util.json import from_json_string, to_json_string from galaxy.util.shed_util import * log = logging.getLogger( __name__ ) @@ -53,7 +55,7 @@ changeset_revision = elem.get( 'changeset_revision' ) # Install path is of the form: <tool path>/<tool shed>/repos/<repository owner>/<repository name>/<installed changeset revision> clone_dir = os.path.join( self.tool_path, self.tool_shed, 'repos', self.repository_owner, name, changeset_revision ) - if self.__isinstalled( elem, clone_dir ): + if self.__isinstalled( clone_dir ): log.debug( "Skipping automatic install of repository '%s' because it has already been installed in location '%s'" % ( name, clone_dir ) ) else: if section_name and section_id: @@ -81,47 +83,59 @@ if returncode == 0: returncode, tmp_name = update_repository( current_working_dir, relative_install_dir, changeset_revision ) if returncode == 0: - metadata_dict = load_repository_contents( app=self.app, - repository_name=name, - description=description, - owner=self.repository_owner, - changeset_revision=changeset_revision, - tool_path=self.tool_path, - repository_clone_url=repository_clone_url, - relative_install_dir=relative_install_dir, - current_working_dir=current_working_dir, - tmp_name=tmp_name, - tool_shed=self.tool_shed, - tool_section=tool_section, - shed_tool_conf=self.install_tool_config, - new_install=True, - dist_to_shed=True ) - # Add a new record to the tool_id_guid_map table for each tool in the repository if one doesn't already exist. + tool_shed_repository, metadata_dict = load_repository_contents( app=self.app, + repository_name=name, + description=description, + owner=self.repository_owner, + changeset_revision=changeset_revision, + tool_path=self.tool_path, + repository_clone_url=repository_clone_url, + relative_install_dir=relative_install_dir, + current_working_dir=current_working_dir, + tmp_name=tmp_name, + tool_shed=self.tool_shed, + tool_section=tool_section, + shed_tool_conf=self.install_tool_config, + new_install=True, + dist_to_shed=True ) if 'tools' in metadata_dict: - tools_mapped = 0 - for tool_dict in metadata_dict[ 'tools' ]: - flush_needed = False - tool_id = tool_dict[ 'id' ] - tool_version = tool_dict[ 'version' ] - guid = tool_dict[ 'guid' ] - tool_id_guid_map = get_tool_id_guid_map( self.app, tool_id, tool_version, self.tool_shed, self.repository_owner, name ) - if tool_id_guid_map: - if tool_id_guid_map.guid != guid: - tool_id_guid_map.guid = guid - flush_needed = True - else: - tool_id_guid_map = self.app.model.ToolIdGuidMap( tool_id=tool_id, - tool_version=tool_version, - tool_shed=self.tool_shed, - repository_owner=self.repository_owner, - repository_name=name, - guid=guid ) - flush_needed = True - if flush_needed: - self.sa_session.add( tool_id_guid_map ) - self.sa_session.flush() - tools_mapped += 1 - log.debug( "Mapped tool ids to guids for %d tools included in repository '%s'." % ( tools_mapped, name ) ) + # Get the tool_versions from the tool shed for each tool in the installed change set. + url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + ( tool_shed_url, name, self.repository_owner, changeset_revision ) + response = urllib2.urlopen( url ) + text = response.read() + response.close() + if text: + tool_versions_dict = from_json_string( text ) + handle_tool_versions( self.app, tool_versions_dict, tool_shed_repository ) + else: + # Set the tool versions since they seem to be missing for this repository in the tool shed. + for tool_dict in metadata_dict[ 'tools' ]: + flush_needed = False + tool_id = tool_dict[ 'guid' ] + old_tool_id = tool_dict[ 'id' ] + tool_version = tool_dict[ 'version' ] + tool_version_using_old_id = get_tool_version( self.app, old_tool_id ) + tool_version_using_guid = get_tool_version( self.app, tool_id ) + if not tool_version_using_old_id: + tool_version_using_old_id = self.app.model.ToolVersion( tool_id=old_tool_id, + tool_shed_repository=tool_shed_repository ) + self.sa_session.add( tool_version_using_old_id ) + self.sa_session.flush() + if not tool_version_using_guid: + tool_version_using_guid = self.app.model.ToolVersion( tool_id=tool_id, + tool_shed_repository=tool_shed_repository ) + self.sa_session.add( tool_version_using_guid ) + self.sa_session.flush() + # Associate the two versions as parent / child. + tool_version_association = get_tool_version_association( self.app, + tool_version_using_old_id, + tool_version_using_guid ) + if not tool_version_association: + tool_version_association = self.app.model.ToolVersionAssociation( tool_id=tool_version_using_guid.id, + parent_id=tool_version_using_old_id.id ) + self.sa_session.add( tool_version_association ) + self.sa_session.flush() else: tmp_stderr = open( tmp_name, 'rb' ) log.debug( "Error updating repository '%s': %s" % ( name, tmp_stderr.read() ) ) @@ -154,26 +168,11 @@ # The tool shed from which the repository was originally # installed must no longer be configured in tool_sheds_conf.xml. return None - def __isinstalled( self, repository_elem, clone_dir ): - name = repository_elem.get( 'name' ) - installed = False - for tool_elem in repository_elem: - tool_config = tool_elem.get( 'file' ) - tool_id = tool_elem.get( 'id' ) - tool_version = tool_elem.get( 'version' ) - tigm = get_tool_id_guid_map( self.app, tool_id, tool_version, self.tool_shed, self.repository_owner, name ) - if tigm: - # A record exists in the tool_id_guid_map table, so see if the repository is installed. - if os.path.exists( clone_dir ): - installed = True - break - if not installed: - full_path = os.path.abspath( clone_dir ) - # We may have a repository that contains no tools. - if os.path.exists( full_path ): - for root, dirs, files in os.walk( full_path ): - if '.hg' in dirs: - # Assume that the repository has been installed if we find a .hg directory. - installed = True - break - return installed + def __isinstalled( self, clone_dir ): + full_path = os.path.abspath( clone_dir ) + if os.path.exists( full_path ): + for root, dirs, files in os.walk( full_path ): + if '.hg' in dirs: + # Assume that the repository has been installed if we find a .hg directory. + return True + return False diff -r f87dff286e86368fb33032ac567a5c9368dc990b -r e6464387ed3fce7f235cf5c9af1281da14a32811 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -120,36 +120,32 @@ return tool else: return tool - # Handle the case where the tool was used when the tool was included in the Galaxy distribution, - # but now the tool is contained in an installed tool shed repository. In this case, the original - # tool id can be mapped to the new tool id, which is the tool's guid in the tool shed repository. - # This scenarios can occur in workflows and in a history item when the rerun icon is clicked. - # The weakness here is that workflows currently handle only tool ids and not versions. - tool_id_guid_map = self.__get_tool_id_guid_map( tool_id, tool_version=tool_version ) - if tool_id_guid_map: - guid = tool_id_guid_map.guid - if guid in self.tools_by_id: - return self.tools_by_id[ guid ] - # Handle the case where a proprietary tool was initially developed and hosted in a local Galaxy - # instance, but the developer later uploaded the tool to a Galaxy tool shed, removed the original - # tool from the local Galaxy instance and installed the tool's repository from the tool shed. - for k, tool in self.tools_by_id.items(): - if tool_id == tool.old_id: - if tool_version and tool.version == tool_version: - return tool - else: - return tool + # Handle the case where the received tool_id has a tool_version. In this case, one of the following + # conditions is true. + # 1. The tool was used when it was included in the Galaxy distribution, but now the tool is contained + # in an installed tool shed repository. In this case, the original tool id can be mapped to the new + # tool id, which is the tool's guid in the tool shed repository. This scenarios can occur in + # workflows and in a history item when the rerun icon is clicked. The weakness here is that workflows + # currently handle only tool ids and not versions. + # 2. A proprietary tool was initially developed and hosted in a local Galaxy instance, but the developer + # later uploaded the tool to a Galaxy tool shed, removed the original tool from the local Galaxy + # instance and installed the tool's repository from the tool shed. + tv = self.__get_tool_version( tool_id ) + if tv: + tool_version_ids = tv.get_version_ids( self.app ) + for tool_version_id in tool_version_ids: + if tool_version_id in self.tools_by_id: + tool = self.tools_by_id[ tool_version_id ] + if tool_version and tool.version == tool_version: + return tool + else: + return tool return None - def __get_tool_id_guid_map( self, tool_id, tool_version=None ): - if tool_version: - return self.sa_session.query( self.app.model.ToolIdGuidMap ) \ - .filter( and_( self.app.model.ToolIdGuidMap.table.c.tool_id == tool_id, - self.app.model.ToolIdGuidMap.table.c.tool_version == tool_version ) ) \ - .first() - else: - return self.sa_session.query( self.app.model.ToolIdGuidMap ) \ - .filter( self.app.model.ToolIdGuidMap.table.c.tool_id == tool_id ) \ - .first() + def __get_tool_version( self, tool_id ): + """Return a ToolVersion if one exists for our tool_id""" + return self.sa_session.query( self.app.model.ToolVersion ) \ + .filter( self.app.model.ToolVersion.table.c.tool_id == tool_id ) \ + .first() def __get_tool_shed_repository( self, tool_shed, name, owner, installed_changeset_revision ): return self.sa_session.query( self.app.model.ToolShedRepository ) \ .filter( and_( self.app.model.ToolShedRepository.table.c.tool_shed == tool_shed, @@ -160,7 +156,10 @@ def load_tool_tag_set( self, elem, panel_dict, tool_path, guid=None, section=None ): try: path = elem.get( "file" ) - if guid is not None: + if guid is None: + tool_shed_repository = None + can_load = True + else: # The tool is contained in an installed tool shed repository, so load # the tool only if the repository has not been marked deleted. tool_shed = elem.find( "tool_shed" ).text @@ -218,8 +217,6 @@ # If there is not yet a tool_shed_repository record, we're in the process of installing # a new repository, so any included tools can be loaded into the tool panel. can_load = True - else: - can_load = True if can_load: tool = self.load_tool( os.path.join( tool_path, path ), guid=guid ) if guid is not None: @@ -230,6 +227,11 @@ tool.guid = guid tool.old_id = elem.find( "id" ).text tool.version = elem.find( "version" ).text + # Make sure the tool has a tool_version. + if not self.__get_tool_version( tool.id ): + tool_version = self.app.model.ToolVersion( tool_id=tool.id, tool_shed_repository=tool_shed_repository ) + self.sa_session.add( tool_version ) + self.sa_session.flush() if self.app.config.get_bool( 'enable_tool_tags', False ): tag_names = elem.get( "tags", "" ).split( "," ) for tag_name in tag_names: @@ -542,13 +544,30 @@ # Parse XML element containing configuration self.parse( root, guid=guid ) self.external_runJob_script = app.config.drmaa_external_runjob_script - @property def sa_session( self ): - """ - Returns a SQLAlchemy session - """ + """Returns a SQLAlchemy session""" return self.app.model.context + @property + def tool_version( self ): + """Return a ToolVersion if one exists for our id""" + return self.sa_session.query( self.app.model.ToolVersion ) \ + .filter( self.app.model.ToolVersion.table.c.tool_id == self.id ) \ + .first() + @property + def tool_versions( self ): + # If we have versions, return them. + tool_version = self.tool_version + if tool_version: + return tool_version.get_versions( self.app ) + return [] + @property + def tool_version_ids( self ): + # If we have versions, return a list of their tool_ids. + tool_version = self.tool_version + if tool_version: + return tool_version.get_version_ids( self.app ) + return [] def parse( self, root, guid=None ): """ Read tool configuration from the element `root` and fill in `self`. diff -r f87dff286e86368fb33032ac567a5c9368dc990b -r e6464387ed3fce7f235cf5c9af1281da14a32811 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -110,6 +110,7 @@ dist_to_shed=dist_to_shed ) sa_session.add( tool_shed_repository ) sa_session.flush() + return tool_shed_repository def generate_datatypes_metadata( datatypes_config, metadata_dict ): """ Update the received metadata_dict with changes that have been applied @@ -358,15 +359,18 @@ tool = app.toolbox.load_tool( os.path.abspath( relative_path ), guid=guid ) repository_tools_tups.append( ( relative_path, guid, tool ) ) return repository_tools_tups -def get_tool_id_guid_map( app, tool_id, version, tool_shed, repository_owner, repository_name ): +def get_tool_version( app, tool_id ): # This method is used by the InstallManager, which does not have access to trans. sa_session = app.model.context.current - return sa_session.query( app.model.ToolIdGuidMap ) \ - .filter( and_( app.model.ToolIdGuidMap.table.c.tool_id == tool_id, - app.model.ToolIdGuidMap.table.c.tool_version == version, - app.model.ToolIdGuidMap.table.c.tool_shed == tool_shed, - app.model.ToolIdGuidMap.table.c.repository_owner == repository_owner, - app.model.ToolIdGuidMap.table.c.repository_name == repository_name ) ) \ + return sa_session.query( app.model.ToolVersion ) \ + .filter( app.model.ToolVersion.table.c.tool_id == tool_id ) \ + .first() +def get_tool_version_association( app, parent_tool_version, tool_version ): + """Return a ToolVersionAssociation if one exists that associates the two received tool_versions""" + sa_session = app.model.context.current + return sa_session.query( app.model.ToolVersionAssociation ) \ + .filter( and_( app.model.ToolVersionAssociation.table.c.parent_id == parent_tool_version.id, + app.model.ToolVersionAssociation.table.c.tool_id == tool_version.id ) ) \ .first() def get_url_from_repository_tool_shed( app, repository ): """ @@ -472,6 +476,33 @@ error = tmp_stderr.read() tmp_stderr.close() log.debug( 'Problem installing dependencies for tool "%s"\n%s' % ( repository_tool.name, error ) ) +def handle_tool_versions( app, tool_versions, tool_shed_repository ): + """ + This method is used by the InstallManager, which does not have access to trans. Using + the tool_versions dictionary retrieved from the tool shed, create the parent / child pairs + of tool versions. The tool_versions dictionary contains { tool id : parent tool id } pairs. + """ + sa_session = app.model.context.current + for tool_guid, parent_id in tool_versions.items(): + tool_version_using_tool_guid = get_tool_version( app, tool_guid ) + tool_version_using_parent_id = get_tool_version( app, parent_id ) + if not tool_version_using_tool_guid: + tool_version_using_tool_guid = app.model.ToolVersion( tool_id=tool_guid, tool_shed_repository=tool_shed_repository ) + sa_session.add( tool_version_using_tool_guid ) + sa_session.flush() + if not tool_version_using_parent_id: + tool_version_using_parent_id = app.model.ToolVersion( tool_id=parent_id, tool_shed_repository=tool_shed_repository ) + sa_session.add( tool_version_using_parent_id ) + sa_session.flush() + # Associate the two versions as parent / child. + tool_version_association = get_tool_version_association( app, + tool_version_using_parent_id, + tool_version_using_tool_guid ) + if not tool_version_association: + tool_version_association = app.model.ToolVersionAssociation( tool_id=tool_version_using_tool_guid.id, + parent_id=tool_version_using_parent_id.id ) + sa_session.add( tool_version_association ) + sa_session.flush() def load_datatype_items( app, repository, relative_install_dir, deactivate=False ): # Load proprietary datatypes. metadata = repository.metadata @@ -638,13 +669,13 @@ # deleted, undelete it. It is imperative that this happens before the call to alter_tool_panel() below because # tools will not be properly loaded if the repository is marked deleted. log.debug( "Adding new row (or updating an existing row) for repository '%s' in the tool_shed_repository table." % repository_name ) - create_or_update_tool_shed_repository( app, - repository_name, - description, - changeset_revision, - repository_clone_url, - metadata_dict, - dist_to_shed=dist_to_shed ) + tool_shed_repository = create_or_update_tool_shed_repository( app, + repository_name, + description, + changeset_revision, + repository_clone_url, + metadata_dict, + dist_to_shed=dist_to_shed ) if 'tools' in metadata_dict: repository_tools_tups = get_repository_tools_tups( app, metadata_dict ) if repository_tools_tups: @@ -696,7 +727,7 @@ if display_path: # Load proprietary datatype display applications app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) - return metadata_dict + return tool_shed_repository, metadata_dict def alter_tool_panel( app, repository_name, repository_clone_url, changeset_revision, repository_tools_tups, tool_section, shed_tool_conf, tool_path, owner, new_install=True, deactivate=False, uninstall=False ): """ diff -r f87dff286e86368fb33032ac567a5c9368dc990b -r e6464387ed3fce7f235cf5c9af1281da14a32811 lib/galaxy/web/controllers/admin_toolshed.py --- a/lib/galaxy/web/controllers/admin_toolshed.py +++ b/lib/galaxy/web/controllers/admin_toolshed.py @@ -1,58 +1,11 @@ +import urllib2 from galaxy.web.controllers.admin import * +from galaxy.util.json import from_json_string, to_json_string from galaxy.util.shed_util import * from galaxy import tools log = logging.getLogger( __name__ ) -class ToolIdGuidMapGrid( grids.Grid ): - class ToolIdColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool_id_guid_map ): - return tool_id_guid_map.tool_id - class ToolVersionColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool_id_guid_map ): - return tool_id_guid_map.tool_version - class ToolGuidColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool_id_guid_map ): - return tool_id_guid_map.guid - class ToolShedColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool_id_guid_map ): - return tool_id_guid_map.tool_shed - class RepositoryNameColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool_id_guid_map ): - return tool_id_guid_map.repository_name - class RepositoryOwnerColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool_id_guid_map ): - return tool_id_guid_map.repository_owner - # Grid definition - title = "Map tool id to guid" - model_class = model.ToolIdGuidMap - template='/admin/tool_shed_repository/grid.mako' - default_sort_key = "tool_id" - columns = [ - ToolIdColumn( "Tool id" ), - ToolVersionColumn( "Version" ), - ToolGuidColumn( "Guid" ), - ToolShedColumn( "Tool shed" ), - RepositoryNameColumn( "Repository name" ), - RepositoryOwnerColumn( "Repository owner" ) - ] - columns.append( grids.MulticolFilterColumn( "Search repository name", - cols_to_filter=[ columns[0], columns[2], columns[4], columns[5] ], - key="free-text-search", - visible=False, - filterable="standard" ) ) - global_actions = [ - grids.GridAction( "Manage installed tool shed repositories", dict( controller='admin_toolshed', action='browse_repositories' ) ) - ] - operations = [] - standard_filters = [] - default_filter = {} - num_rows_per_page = 50 - preserve_state = False - use_paging = True - def build_initial_query( self, trans, **kwd ): - return trans.sa_session.query( self.model_class ) - class RepositoryListGrid( grids.Grid ): class NameColumn( grids.TextColumn ): def get_value( self, trans, grid, tool_shed_repository ): @@ -96,9 +49,7 @@ key="free-text-search", visible=False, filterable="standard" ) ) - global_actions = [ - grids.GridAction( "View tool id guid map", dict( controller='admin_toolshed', action='browse_tool_id_guid_map' ) ) - ] + global_actions = [] operations = [ grids.GridOperation( "Get updates", allow_multiple=False, condition=( lambda item: not item.deleted ), @@ -122,14 +73,9 @@ class AdminToolshed( AdminGalaxy ): repository_list_grid = RepositoryListGrid() - tool_id_guid_map_grid = ToolIdGuidMapGrid() @web.expose @web.require_admin - def browse_tool_id_guid_map( self, trans, **kwd ): - return self.tool_id_guid_map_grid( trans, **kwd ) - @web.expose - @web.require_admin def browse_repository( self, trans, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) @@ -267,21 +213,38 @@ if returncode == 0: owner = get_repository_owner( clean_repository_clone_url( repository_clone_url ) ) tool_shed = clean_tool_shed_url( tool_shed_url ) - metadata_dict = load_repository_contents( app=trans.app, - repository_name=name, - description=description, - owner=owner, - changeset_revision=changeset_revision, - tool_path=tool_path, - repository_clone_url=repository_clone_url, - relative_install_dir=relative_install_dir, - current_working_dir=current_working_dir, - tmp_name=tmp_name, - tool_shed=tool_shed, - tool_section=tool_section, - shed_tool_conf=shed_tool_conf, - new_install=True, - dist_to_shed=False ) + tool_shed_repository, metadata_dict = load_repository_contents( app=trans.app, + repository_name=name, + description=description, + owner=owner, + changeset_revision=changeset_revision, + tool_path=tool_path, + repository_clone_url=repository_clone_url, + relative_install_dir=relative_install_dir, + current_working_dir=current_working_dir, + tmp_name=tmp_name, + tool_shed=tool_shed, + tool_section=tool_section, + shed_tool_conf=shed_tool_conf, + new_install=True, + dist_to_shed=False ) + + + if 'tools' in metadata_dict: + # Get the tool_versions from the tool shed for each tool in the installed change set. + url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + ( tool_shed_url, name, owner, changeset_revision ) + response = urllib2.urlopen( url ) + text = response.read() + response.close() + if text: + tool_versions_dict = from_json_string( text ) + handle_tool_versions( trans.app, tool_versions_dict, tool_shed_repository ) + else: + message += "Version information for the tools included in the <b>%s</b> repository is missing. " % name + message += "Reset all of this repository's metadata in the tool shed, then set the installed tool versions " + message += "from the installed repository's <b>Repository Actions</b> menu. " + status = 'error' installed_repository_names.append( name ) else: tmp_stderr = open( tmp_name, 'rb' ) @@ -293,7 +256,7 @@ message += '%s<br/>' % tmp_stderr.read() tmp_stderr.close() status = 'error' - if installed_repository_names: + if installed_repository_names: installed_repository_names.sort() num_repositories_installed = len( installed_repository_names ) if tool_section: @@ -332,6 +295,35 @@ status=status ) @web.expose @web.require_admin + def set_tool_versions( self, trans, **kwd ): + # Get the tool_versions from the tool shed for each tool in the installed change set. + repository = get_repository( trans, kwd[ 'id' ] ) + tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) + url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + ( tool_shed_url, repository.name, repository.owner, repository.changeset_revision ) + response = urllib2.urlopen( url ) + text = response.read() + response.close() + if text: + tool_versions_dict = from_json_string( text ) + handle_tool_versions( trans.app, tool_versions_dict, repository ) + message = "Tool versions have been set for all included tools." + status = 'done' + else: + message = "Version information for the tools included in the <b>%s</b> repository is missing. " % repository.name + message += "Reset all of this reppository's metadata in the tool shed, then set the installed tool versions " + message ++ "from the installed repository's <b>Repository Actions</b> menu. " + status = 'error' + shed_tool_conf, tool_path, relative_install_dir = self.__get_tool_path_and_relative_install_dir( trans, repository ) + repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) ) + return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako', + repository=repository, + description=repository.description, + repo_files_dir=repo_files_dir, + message=message, + status=status ) + @web.expose + @web.require_admin def deactivate_or_uninstall_repository( self, trans, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) @@ -392,17 +384,6 @@ log.debug( "Removed repository installation directory: %s" % str( relative_install_dir ) ) except Exception, e: log.debug( "Error removing repository installation directory %s: %s" % ( str( relative_install_dir ), str( e ) ) ) - # If the repository was installed by the InstallManager, remove - # all appropriate rows from the tool_id_guid_map database table. - if repository.dist_to_shed: - count = 0 - for tool_id_guid_map in trans.sa_session.query( trans.model.ToolIdGuidMap ) \ - .filter( and_( trans.model.ToolIdGuidMap.table.c.tool_shed==repository.tool_shed, - trans.model.ToolIdGuidMap.table.c.repository_owner==repository.owner, - trans.model.ToolIdGuidMap.table.c.repository_name==repository.name ) ): - trans.sa_session.delete( tool_id_guid_map ) - count += 1 - log.debug( "Removed %d rows from the tool_id_guid_map database table." % count ) repository.uninstalled = True repository.deleted = True trans.sa_session.add( repository ) @@ -490,21 +471,21 @@ elem.attrib[ 'version' ] = '' tool_section = tools.ToolSection( elem ) trans.app.toolbox.tool_panel[ section_key ] = tool_section - metadata_dict = load_repository_contents( app=trans.app, - repository_name=repository.name, - description=repository.description, - owner=repository.owner, - changeset_revision=repository.installed_changeset_revision, - tool_path=tool_path, - repository_clone_url=repository_clone_url, - relative_install_dir=relative_install_dir, - current_working_dir=current_working_dir, - tmp_name=tmp_name, - tool_shed=repository.tool_shed, - tool_section=tool_section, - shed_tool_conf=shed_tool_conf, - new_install=True, - dist_to_shed=False ) + tool_shed_repository, metadata_dict = load_repository_contents( app=trans.app, + repository_name=repository.name, + description=repository.description, + owner=repository.owner, + changeset_revision=repository.installed_changeset_revision, + tool_path=tool_path, + repository_clone_url=repository_clone_url, + relative_install_dir=relative_install_dir, + current_working_dir=current_working_dir, + tmp_name=tmp_name, + tool_shed=repository.tool_shed, + tool_section=tool_section, + shed_tool_conf=shed_tool_conf, + new_install=True, + dist_to_shed=False ) repository.uninstalled = False repository.deleted = False trans.sa_session.add( repository ) @@ -518,6 +499,22 @@ metadata = repository.metadata repository_tools_tups = get_repository_tools_tups( trans.app, metadata ) guids_to_activate = [ repository_tool_tup[1] for repository_tool_tup in repository_tools_tups ] + # Undelete the tool_version for each guid. + for guid_to_activate in guids_to_activate: + tool_version = get_tool_version( trans.app, guid_to_activate ) + if tool_version: + if tool_version.deleted: + # This should not happen as we are currently not marking tool versions as deleted + # upon deactivation. We may decide to eliminate the tool_version.deleted column + # at some point, but we'll keep it for now in case we decide its useful. + tool_version.deleted = False + trans.sa_session.add( tool_version ) + trans.sa_session.flush() + else: + # We're somehow missing a tool_version, so create a new one. + tool_version = trans.model.ToolVersion( tool_id=guid_to_activate, tool_shed_repository=repository ) + trans.sa_session.add( tool_version ) + trans.sa_session.flush() tool_panel_section = metadata[ 'tool_panel_section' ] original_section_id = tool_panel_section[ 'id' ] if original_section_id in [ '' ]: @@ -653,8 +650,8 @@ # Send a request to the relevant tool shed to see if there are any updates. repository = get_repository( trans, kwd[ 'id' ] ) tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) - url = '%s/repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ - ( tool_shed_url, url_for( '', qualified=True ), repository.name, repository.owner, repository.changeset_revision ) + url = '%/srepository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + ( tool_shed_url, url_for( '/', qualified=True ), repository.name, repository.owner, repository.changeset_revision ) return trans.response.send_redirect( url ) @web.expose @web.require_admin @@ -684,20 +681,20 @@ # Update the repository metadata. repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name ) tool_shed = clean_tool_shed_url( tool_shed_url ) - metadata_dict = load_repository_contents( app=trans.app, - name=name, - description=repository.description, - owner=owner, - changeset_revision=changeset_revision, - tool_path=tool_path, - repository_clone_url=repository_clone_url, - relative_install_dir=relative_install_dir, - current_working_dir=current_working_dir, - tmp_name=tmp_name, - tool_shed=tool_shed, - tool_section=None, - shed_tool_conf=None, - new_install=False ) + tool_shed_repository, metadata_dict = load_repository_contents( app=trans.app, + name=name, + description=repository.description, + owner=owner, + changeset_revision=changeset_revision, + tool_path=tool_path, + repository_clone_url=repository_clone_url, + relative_install_dir=relative_install_dir, + current_working_dir=current_working_dir, + tmp_name=tmp_name, + tool_shed=tool_shed, + tool_section=None, + shed_tool_conf=None, + new_install=False ) # Update the repository changeset_revision in the database. repository.changeset_revision = latest_changeset_revision repository.update_available = False diff -r f87dff286e86368fb33032ac567a5c9368dc990b -r e6464387ed3fce7f235cf5c9af1281da14a32811 lib/galaxy/web/controllers/workflow.py --- a/lib/galaxy/web/controllers/workflow.py +++ b/lib/galaxy/web/controllers/workflow.py @@ -1220,7 +1220,7 @@ for shed_name, shed_url in trans.app.tool_shed_registry.tool_sheds.items(): if shed_url.endswith( '/' ): shed_url = shed_url.rstrip( '/' ) - url = '%s/repository/find_tools?galaxy_url=%s&webapp=%s' % ( shed_url, url_for( '', qualified=True ), webapp ) + url = '%s/repository/find_tools?galaxy_url=%s&webapp=%s' % ( shed_url, url_for( '/', qualified=True ), webapp ) if missing_tool_tups: url += '&tool_id=' for missing_tool_tup in missing_tool_tups: diff -r f87dff286e86368fb33032ac567a5c9368dc990b -r e6464387ed3fce7f235cf5c9af1281da14a32811 lib/galaxy/webapps/community/controllers/common.py --- a/lib/galaxy/webapps/community/controllers/common.py +++ b/lib/galaxy/webapps/community/controllers/common.py @@ -488,6 +488,7 @@ repo = hg.repository( get_configured_ui(), repo_dir ) if len( repo ) == 1: message, status = set_repository_metadata( trans, id, repository.tip, **kwd ) + add_repository_metadata_tool_versions( trans, id, [ repository.tip ] ) else: # The list of changeset_revisions refers to repository_metadata records that have been # created or updated. When the following loop completes, we'll delete all repository_metadata @@ -543,6 +544,7 @@ ancestor_changeset_revision = None ancestor_metadata_dict = None clean_repository_metadata( trans, id, changeset_revisions ) + add_repository_metadata_tool_versions( trans, id, changeset_revisions ) def clean_repository_metadata( trans, id, changeset_revisions ): # Delete all repository_metadata reecords associated with the repository # that have a changeset_revision that is not in changeset_revisions. @@ -551,6 +553,47 @@ if repository_metadata.changeset_revision not in changeset_revisions: trans.sa_session.delete( repository_metadata ) trans.sa_session.flush() +def add_repository_metadata_tool_versions( trans, id, changeset_revisions ): + # If a repository includes tools, build a dictionary of { 'tool id' : 'parent tool id' } + # pairs for each tool in each changeset revision. + for index, changeset_revision in enumerate( changeset_revisions ): + tool_versions_dict = {} + repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + metadata = repository_metadata.metadata + tool_dicts = metadata.get( 'tools', [] ) + if index == 0: + # The first changset_revision is a special case because it will have no ancestor + # changeset_revisions in which to match tools. The parent tool id for tools in + # the first changeset_revision will be the "old_id" in the tool config. + for tool_dict in tool_dicts: + tool_versions_dict[ tool_dict[ 'guid' ] ] = tool_dict[ 'id' ] + else: + for tool_dict in tool_dicts: + # We have at least 2 changeset revisions to compare tool guids and tool ids. + parent_id = get_parent_id( trans, id, tool_dict[ 'id' ], tool_dict[ 'version' ], tool_dict[ 'guid' ], changeset_revisions[ 0:index ] ) + tool_versions_dict[ tool_dict[ 'guid' ] ] = parent_id + if tool_versions_dict: + repository_metadata.tool_versions = tool_versions_dict + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() +def get_parent_id( trans, id, old_id, version, guid, changeset_revisions ): + parent_id = None + # Compare from most recent to oldest. + changeset_revisions.reverse() + for changeset_revision in changeset_revisions: + repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + metadata = repository_metadata.metadata + tools_dicts = metadata.get( 'tools', [] ) + for tool_dict in tools_dicts: + if tool_dict[ 'guid' ] == guid: + # The tool has not changed between the compared changeset revisions. + continue + if tool_dict[ 'id' ] == old_id and tool_dict[ 'version' ] != version: + # The tool version is different, so we've found the parent. + return tool_dict[ 'guid' ] + if parent_id is None: + # The tool did not change through all of the changeset revisions. + return old_id def create_or_update_repository_metadata( trans, id, repository, changeset_revision, metadata_dict ): repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) if repository_metadata: diff -r f87dff286e86368fb33032ac567a5c9368dc990b -r e6464387ed3fce7f235cf5c9af1281da14a32811 lib/galaxy/webapps/community/controllers/repository.py --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -438,7 +438,7 @@ galaxy_url = trans.get_cookie( name='toolshedgalaxyurl' ) encoded_repo_info_dict, includes_tools = self.__encode_repo_info_dict( trans, webapp, util.listify( item_id ) ) url = '%sadmin_toolshed/install_repository?tool_shed_url=%s&webapp=%s&repo_info_dict=%s&includes_tools=%s' % \ - ( galaxy_url, url_for( '', qualified=True ), webapp, encoded_repo_info_dict, str( includes_tools ) ) + ( galaxy_url, url_for( '/', qualified=True ), webapp, encoded_repo_info_dict, str( includes_tools ) ) return trans.response.send_redirect( url ) else: # This can only occur when there is a multi-select grid with check boxes and an operation, @@ -514,7 +514,7 @@ galaxy_url = trans.get_cookie( name='toolshedgalaxyurl' ) encoded_repo_info_dict, includes_tools = self.__encode_repo_info_dict( trans, webapp, util.listify( item_id ) ) url = '%sadmin_toolshed/install_repository?tool_shed_url=%s&webapp=%s&repo_info_dict=%s&includes_tools=%s' % \ - ( galaxy_url, url_for( '', qualified=True ), webapp, encoded_repo_info_dict, str( includes_tools ) ) + ( galaxy_url, url_for( '/', qualified=True ), webapp, encoded_repo_info_dict, str( includes_tools ) ) return trans.response.send_redirect( url ) else: # This can only occur when there is a multi-select grid with check boxes and an operation, @@ -770,9 +770,19 @@ encoded_repo_info_dict = encode( repo_info_dict ) # Redirect back to local Galaxy to perform install. url = '%sadmin_toolshed/install_repository?tool_shed_url=%s&repo_info_dict=%s&includes_tools=%s' % \ - ( galaxy_url, url_for( '', qualified=True ), encoded_repo_info_dict, str( includes_tools ) ) + ( galaxy_url, url_for( '/', qualified=True ), encoded_repo_info_dict, str( includes_tools ) ) return trans.response.send_redirect( url ) @web.expose + def get_tool_versions( self, trans, **kwd ): + name = kwd[ 'name' ] + owner = kwd[ 'owner' ] + changeset_revision = kwd[ 'changeset_revision' ] + repository = get_repository_by_name_and_owner( trans, name, owner ) + repository_metadata = get_repository_metadata_by_changeset_revision( trans, trans.security.encode_id( repository.id ), changeset_revision ) + if repository_metadata.tool_versions: + return to_json_string( repository_metadata.tool_versions ) + return '' + @web.expose def check_for_updates( self, trans, **kwd ): # Handle a request from a local Galaxy instance. If the request originated with the # Galaxy instances' UpdateManager, the value of 'webapp' will be 'update_manager'. @@ -792,7 +802,7 @@ no_update = 'false' else: # Start building up the url to redirect back to the calling Galaxy instance. - url = '%sadmin_toolshed/update_to_changeset_revision?tool_shed_url=%s' % ( galaxy_url, url_for( '', qualified=True ) ) + url = '%sadmin_toolshed/update_to_changeset_revision?tool_shed_url=%s' % ( galaxy_url, url_for( '/', qualified=True ) ) url += '&name=%s&owner=%s&changeset_revision=%s&latest_changeset_revision=' % \ ( repository.name, repository.user.username, changeset_revision ) if changeset_revision == repository.tip: diff -r f87dff286e86368fb33032ac567a5c9368dc990b -r e6464387ed3fce7f235cf5c9af1281da14a32811 lib/galaxy/webapps/community/controllers/workflow.py --- a/lib/galaxy/webapps/community/controllers/workflow.py +++ b/lib/galaxy/webapps/community/controllers/workflow.py @@ -402,8 +402,8 @@ to_file.write( to_json_string( workflow_data ) ) return open( tmp_fname ) galaxy_url = trans.get_cookie( name='toolshedgalaxyurl' ) - url = '%s/workflow/import_workflow?tool_shed_url=%s&repository_metadata_id=%s&workflow_name=%s&webapp=%s' % \ - ( galaxy_url, url_for( '', qualified=True ), repository_metadata_id, encode( workflow_name ), webapp ) + url = '%sworkflow/import_workflow?tool_shed_url=%s&repository_metadata_id=%s&workflow_name=%s&webapp=%s' % \ + ( galaxy_url, url_for( '/', qualified=True ), repository_metadata_id, encode( workflow_name ), webapp ) return trans.response.send_redirect( url ) return trans.response.send_redirect( web.url_for( controller='workflow', action='view_workflow', diff -r f87dff286e86368fb33032ac567a5c9368dc990b -r e6464387ed3fce7f235cf5c9af1281da14a32811 lib/galaxy/webapps/community/model/__init__.py --- a/lib/galaxy/webapps/community/model/__init__.py +++ b/lib/galaxy/webapps/community/model/__init__.py @@ -162,10 +162,11 @@ fp.close() class RepositoryMetadata( object ): - def __init__( self, repository_id=None, changeset_revision=None, metadata=None, malicious=False ): + def __init__( self, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False ): self.repository_id = repository_id self.changeset_revision = changeset_revision self.metadata = metadata or dict() + self.tool_versions = tool_versions or dict() self.malicious = malicious class ItemRatingAssociation( object ): diff -r f87dff286e86368fb33032ac567a5c9368dc990b -r e6464387ed3fce7f235cf5c9af1281da14a32811 lib/galaxy/webapps/community/model/mapping.py --- a/lib/galaxy/webapps/community/model/mapping.py +++ b/lib/galaxy/webapps/community/model/mapping.py @@ -119,6 +119,7 @@ Column( "repository_id", Integer, ForeignKey( "repository.id" ), index=True ), Column( "changeset_revision", TrimmedString( 255 ), index=True ), Column( "metadata", JSONType, nullable=True ), + Column( "tool_versions", JSONType, nullable=True ), Column( "malicious", Boolean, default=False ) ) RepositoryRatingAssociation.table = Table( "repository_rating_association", metadata, diff -r f87dff286e86368fb33032ac567a5c9368dc990b -r e6464387ed3fce7f235cf5c9af1281da14a32811 lib/galaxy/webapps/community/model/migrate/versions/0011_add_tool_versions_column.py --- /dev/null +++ b/lib/galaxy/webapps/community/model/migrate/versions/0011_add_tool_versions_column.py @@ -0,0 +1,46 @@ +""" +Migration script to add the new_repo_alert column to the galaxy_user table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True ) + c = Column( "tool_versions", JSONType, nullable=True ) + try: + # Create + c.create( RepositoryMetadata_table ) + assert c is RepositoryMetadata_table.c.tool_versions + except Exception, e: + print "Adding tool_versions column to the repository_metadata table failed: %s" % str( e ) + +def downgrade(): + metadata.reflect() + # Drop new_repo_alert column from galaxy_user table. + RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True ) + try: + RepositoryMetadata_table.c.tool_versions.drop() + except Exception, e: + print "Dropping column tool_versions from the repository_metadata table failed: %s" % str( e ) diff -r f87dff286e86368fb33032ac567a5c9368dc990b -r e6464387ed3fce7f235cf5c9af1281da14a32811 lib/galaxy/workflow/modules.py --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -194,17 +194,19 @@ @classmethod def from_workflow_step( Class, trans, step ): tool_id = step.tool_id - install_tool_id = None if trans.app.toolbox and tool_id not in trans.app.toolbox.tools_by_id: - # Handle the case where the tool was used when the tool was included in the Galaxy distribution, - # but now the tool is contained in an installed tool shed repository. In this case, the original - # tool id can be mapped to the new tool id, which is the tool's guid in the tool shed repository. - tool_id_guid_map = trans.sa_session.query( trans.model.ToolIdGuidMap ) \ - .filter( trans.model.ToolIdGuidMap.table.c.tool_id == tool_id ) \ - .first() - if tool_id_guid_map: - install_tool_id = tool_id_guid_map.guid - if ( trans.app.toolbox and tool_id in trans.app.toolbox.tools_by_id ) or install_tool_id: + # See if we have access to a different version of the tool. + # TODO: If workflows are ever enhanced to use tool version + # in addition to tool id, enhance the selection process here + # to retrieve the correct version of the tool. + tool_version = self.__get_tool_version( trans, tool_id ) + if tool_version: + tool_version_ids = tool_version.get_version_ids( trans.app ) + for tool_version_id in tool_version_ids: + if tool_version_id in trans.app.toolbox.tools_by_id: + tool_id = tool_version_id + break + if ( trans.app.toolbox and tool_id in trans.app.toolbox.tools_by_id ): module = Class( trans, tool_id ) module.state = DefaultToolState() module.state.inputs = module.tool.params_from_strings( step.tool_inputs, trans.app, ignore_errors=True ) @@ -217,6 +219,11 @@ module.post_job_actions = pjadict return module return None + def __get_tool_version( self, trans, tool_id ): + # Return a ToolVersion if one exists for tool_id. + return trans.sa_session.query( trans.app.model.ToolVersion ) \ + .filter( trans.app.model.ToolVersion.table.c.tool_id == tool_id ) \ + .first() def save_to_step( self, step ): step.type = self.type step.tool_id = self.tool_id diff -r f87dff286e86368fb33032ac567a5c9368dc990b -r e6464387ed3fce7f235cf5c9af1281da14a32811 templates/admin/tool_shed_repository/deactivate_or_uninstall_repository.mako --- a/templates/admin/tool_shed_repository/deactivate_or_uninstall_repository.mako +++ b/templates/admin/tool_shed_repository/deactivate_or_uninstall_repository.mako @@ -84,11 +84,6 @@ <div class="toolParamHelp" style="clear: both;"> * The repository record's uninstalled column in the tool_shed_repository database table will be set to True. </div> - %if repository.dist_to_shed: - <div class="toolParamHelp" style="clear: both;"> - * All records associated with this repository will be eliminated from the tool_id_guid_map database table. - </div> - %endif <div style="clear: both"></div></div><div class="form-row"> diff -r f87dff286e86368fb33032ac567a5c9368dc990b -r e6464387ed3fce7f235cf5c9af1281da14a32811 templates/admin/tool_shed_repository/manage_repository.mako --- a/templates/admin/tool_shed_repository/manage_repository.mako +++ b/templates/admin/tool_shed_repository/manage_repository.mako @@ -8,6 +8,9 @@ <div popupmenu="repository-${repository.id}-popup"><a class="action-button" href="${h.url_for( controller='admin_toolshed', action='browse_repository', id=trans.security.encode_id( repository.id ) )}">Browse repository</a><a class="action-button" href="${h.url_for( controller='admin_toolshed', action='check_for_updates', id=trans.security.encode_id( repository.id ) )}">Get updates</a> + %if repository.includes_tools: + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='set_tool_versions', id=trans.security.encode_id( repository.id ) )}">Set tool versions</a> + %endif <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='deactivate_or_uninstall_repository', id=trans.security.encode_id( repository.id ) )}">Deactivate or Uninstall</a></div></ul> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.