1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/0c804033ae5a/ changeset: 0c804033ae5a user: greg date: 2011-12-14 16:18:43 summary: 1. Add a new UpdateManager for use with tool shed repositories installed into a local Galaxy instance. The UpdateManager will poll all appropriate tool sheds to see if updates are available for each of the installed repositories. Polling occurs when the Galaxy server is started. In addition, a config setting tells the UpdateManager to poll after the configured number of hours have passed. If updates are available for an installed repository, a table column is updated, and the repository name is highlighted in red, alerting the Galaxy admin that updates are available for that repository. 2. Add a new ToolIdGuidMap grid that displays all of the mappings between tool ids whose tools used to be in the distribution and guids, which is the new tool id for tools that are installed with repositories from tool sheds. 3. Add a new column named installed_changeset_revsion to the tool_shed_repository table. This column is set when the repository is installed and remains static thereafter. 4. Move several tool shed related components to a new ~/lib/galaxy/tool_shed directory in the code base. affected #: 15 files diff -r 4a39bc2094875a9878f07f27f7307976e05e8b87 -r 0c804033ae5a766f1d12a2030b73fe306c27f660 lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -3,7 +3,7 @@ from galaxy import config, jobs, util, tools, web import galaxy.tools.search import galaxy.tools.data -import galaxy.tools.tool_shed_registry +import galaxy.tool_shed.tool_shed_registry from galaxy.web import security import galaxy.model import galaxy.datatypes.registry @@ -28,7 +28,7 @@ galaxy.model.set_datatypes_registry( self.datatypes_registry ) # Set up the tool sheds registry if os.path.isfile( self.config.tool_sheds_config ): - self.tool_shed_registry = galaxy.tools.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config ) + self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config ) else: self.tool_shed_registry = None # Determine the database url @@ -61,8 +61,13 @@ # If enabled, check for tools missing from the distribution because they # have been moved to the tool shed and install all such discovered tools. if self.config.get_bool( 'enable_tool_shed_install', False ): - from tools import install_manager + from tool_shed import install_manager self.install_manager = install_manager.InstallManager( self, self.config.tool_shed_install_config, self.config.install_tool_config ) + # If enabled, poll respective tool sheds to see if updates are + # available for any installed tool shed repositories. + if self.config.get_bool( 'enable_tool_shed_check', False ): + from tool_shed import update_manager + self.update_manager = update_manager.UpdateManager( self ) # Load datatype converters self.datatypes_registry.load_datatype_converters( self.toolbox ) # Load history import/export tools diff -r 4a39bc2094875a9878f07f27f7307976e05e8b87 -r 0c804033ae5a766f1d12a2030b73fe306c27f660 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -47,12 +47,12 @@ self.enable_openid = string_as_bool( kwargs.get( 'enable_openid', False ) ) self.enable_quotas = string_as_bool( kwargs.get( 'enable_quotas', False ) ) self.tool_sheds_config = kwargs.get( 'tool_sheds_config_file', 'tool_sheds_conf.xml' ) - self.enable_unique_workflow_defaults = string_as_bool( kwargs.get ( 'enable_unique_workflow_defaults', False ) ) + self.enable_unique_workflow_defaults = string_as_bool( kwargs.get( 'enable_unique_workflow_defaults', False ) ) self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root ) self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "tool-data" ), os.getcwd() ) self.len_file_path = kwargs.get( "len_file_path", resolve_path(os.path.join(self.tool_data_path, 'shared','ucsc','chrom'), self.root) ) self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root ) - self.enable_tool_shed_install = string_as_bool( kwargs.get ( 'enable_tool_shed_install', False ) ) + self.enable_tool_shed_install = string_as_bool( kwargs.get( 'enable_tool_shed_install', False ) ) self.tool_shed_install_config = resolve_path( kwargs.get( "tool_shed_install_config_file", "tool_shed_install.xml" ), self.root ) self.install_tool_config = resolve_path( kwargs.get( "install_tool_config_file", "shed_tool_conf.xml" ), self.root ) if 'tool_config_file' in kwargs: @@ -63,6 +63,13 @@ tcf = 'tool_conf.xml' self.tool_configs = [ resolve_path( p, self.root ) for p in listify( tcf ) ] self.tool_data_table_config_path = resolve_path( kwargs.get( 'tool_data_table_config_path', 'tool_data_table_conf.xml' ), self.root ) + self.enable_tool_shed_check = string_as_bool( kwargs.get( 'enable_tool_shed_check', False ) ) + try: + self.hours_between_check = int( kwargs.get( 'hours_between_check', 12 ) ) + if self.hours_between_check < 1 or self.hours_between_check > 24: + self.hours_between_check = 12 + except: + self.hours_between_check = 12 self.tool_secret = kwargs.get( "tool_secret", "" ) self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" ) self.set_metadata_externally = string_as_bool( kwargs.get( "set_metadata_externally", "False" ) ) diff -r 4a39bc2094875a9878f07f27f7307976e05e8b87 -r 0c804033ae5a766f1d12a2030b73fe306c27f660 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2660,7 +2660,7 @@ pass class ToolShedRepository( object ): - def __init__( self, id=None, create_time=None, tool_shed=None, name=None, description=None, owner=None, + def __init__( self, id=None, create_time=None, tool_shed=None, name=None, description=None, owner=None, installed_changeset_revision=None, changeset_revision=None, metadata=None, includes_datatypes=False, update_available=False, deleted=False ): self.id = id self.create_time = create_time @@ -2668,6 +2668,7 @@ self.name = name self.description = description self.owner = owner + self.installed_changeset_revision = installed_changeset_revision self.changeset_revision = changeset_revision self.metadata = metadata self.includes_datatypes = includes_datatypes diff -r 4a39bc2094875a9878f07f27f7307976e05e8b87 -r 0c804033ae5a766f1d12a2030b73fe306c27f660 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -372,6 +372,7 @@ Column( "name", TrimmedString( 255 ), index=True ), Column( "description" , TEXT ), Column( "owner", TrimmedString( 255 ), index=True ), + Column( "installed_changeset_revision", TrimmedString( 255 ) ), Column( "changeset_revision", TrimmedString( 255 ), index=True ), Column( "metadata", JSONType, nullable=True ), Column( "includes_datatypes", Boolean, index=True, default=False ), diff -r 4a39bc2094875a9878f07f27f7307976e05e8b87 -r 0c804033ae5a766f1d12a2030b73fe306c27f660 lib/galaxy/model/migrate/versions/0088_add_installed_changeset_revison_column.py --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0088_add_installed_changeset_revison_column.py @@ -0,0 +1,63 @@ +""" +Migration script to add the installed_changeset_revision column to the tool_shed_repository table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + col = Column( "installed_changeset_revision", TrimmedString( 255 ) ) + try: + col.create( ToolShedRepository_table ) + assert col is ToolShedRepository_table.c.installed_changeset_revision + except Exception, e: + print "Adding installed_changeset_revision column to the tool_shed_repository table failed: %s" % str( e ) + log.debug( "Adding installed_changeset_revision column to the tool_shed_repository table failed: %s" % str( e ) ) + # Update each row by setting the value of installed_changeset_revison to be the value of changeset_revision. + # This will be problematic if the value of changeset_revision was updated to something other than the value + # that it was when the repository was installed (because the install path determined in real time will attempt to + # find the repository using the updated changeset_revison instead of the required installed_changeset_revision), + # but at the time this script was written, this scenario is extremely unlikely. + cmd = "SELECT id AS id, " \ + + "installed_changeset_revision AS installed_changeset_revision, " \ + + "changeset_revision AS changeset_revision " \ + + "FROM tool_shed_repository;" + tool_shed_repositories = db_session.execute( cmd ).fetchall() + update_count = 0 + for row in tool_shed_repositories: + cmd = "UPDATE tool_shed_repository " \ + + "SET installed_changeset_revision = '%s' " % row.changeset_revision \ + + "WHERE changeset_revision = '%s';" % row.changeset_revision + db_session.execute( cmd ) + update_count += 1 + print "Updated the installed_changeset_revision column for ", update_count, " rows in the tool_shed_repository table. " +def downgrade(): + metadata.reflect() + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + try: + ToolShedRepository_table.c.installed_changeset_revision.drop() + except Exception, e: + print "Dropping column installed_changeset_revision from the tool_shed_repository table failed: %s" % str( e ) + log.debug( "Dropping column installed_changeset_revision from the tool_shed_repository table failed: %s" % str( e ) ) diff -r 4a39bc2094875a9878f07f27f7307976e05e8b87 -r 0c804033ae5a766f1d12a2030b73fe306c27f660 lib/galaxy/tool_shed/__init__.py --- /dev/null +++ b/lib/galaxy/tool_shed/__init__.py @@ -0,0 +1,3 @@ +""" +Classes encapsulating the relationships between Galaxy and Galaxy tool sheds. +""" \ No newline at end of file diff -r 4a39bc2094875a9878f07f27f7307976e05e8b87 -r 0c804033ae5a766f1d12a2030b73fe306c27f660 lib/galaxy/tool_shed/install_manager.py --- /dev/null +++ b/lib/galaxy/tool_shed/install_manager.py @@ -0,0 +1,160 @@ +""" +Manage automatic installation of tools configured in tool_shed_install.xml, all of which were +at some point included in the Galaxy distribution, but are now hosted in the main Galaxy tool +shed. Tools included in tool_shed_install.xml that have already been installed will not be +re-installed. +""" +from galaxy.util.shed_util import * + +log = logging.getLogger( __name__ ) + +class InstallManager( object ): + def __init__( self, app, tool_shed_install_config, install_tool_config ): + """ + Check tool settings in tool_shed_install_config and install all tools that are + not already installed. The tool panel configuration file is the received + shed_tool_config, which defaults to shed_tool_conf.xml. + """ + self.app = app + self.sa_session = self.app.model.context.current + self.install_tool_config = install_tool_config + # Parse shed_tool_config to get the install location (tool_path). + tree = util.parse_xml( install_tool_config ) + root = tree.getroot() + self.tool_path = root.get( 'tool_path' ) + self.app.toolbox.shed_tool_confs[ install_tool_config ] = self.tool_path + # Parse tool_shed_install_config to check each of the tools. + log.debug( "Parsing tool shed install configuration %s" % tool_shed_install_config ) + self.tool_shed_install_config = tool_shed_install_config + tree = util.parse_xml( tool_shed_install_config ) + root = tree.getroot() + self.tool_shed = clean_tool_shed_url( root.get( 'name' ) ) + log.debug( "Repositories will be installed from tool shed '%s' into configured tool_path location '%s'" % ( str( self.tool_shed ), str( self.tool_path ) ) ) + self.repository_owner = 'devteam' + for elem in root: + if elem.tag == 'repository': + self.install_repository( elem ) + elif elem.tag == 'section': + self.install_section( elem ) + def install_repository( self, elem, section_name='', section_id='' ): + # Install a single repository into the tool config. If outside of any sections, the entry looks something like: + # <repository name="cut_wrapper" description="Galaxy wrapper for the Cut tool" changeset_revision="f3ed6cfe6402"> + # <tool id="Cut1" version="1.0.1" /> + # </repository> + name = elem.get( 'name' ) + description = elem.get( 'description' ) + changeset_revision = elem.get( 'changeset_revision' ) + # Install path is of the form: <tool path>/<tool shed>/repos/<repository owner>/<repository name>/<changeset revision> + clone_dir = os.path.join( self.tool_path, self.tool_shed, 'repos', self.repository_owner, name, changeset_revision ) + if self.__isinstalled( elem, clone_dir ): + log.debug( "Skipping automatic install of repository '%s' because it has already been installed in location '%s'" % ( name, clone_dir ) ) + else: + if section_name and section_id: + section_key = 'section_%s' % str( section_id ) + if section_key in self.app.toolbox.tool_panel: + # Appending a tool to an existing section in self.app.toolbox.tool_panel + log.debug( "Appending to tool panel section: %s" % section_name ) + tool_section = self.app.toolbox.tool_panel[ section_key ] + else: + # Appending a new section to self.app.toolbox.tool_panel + log.debug( "Loading new tool panel section: %s" % section_name ) + new_section_elem = Element( 'section' ) + new_section_elem.attrib[ 'name' ] = section_name + new_section_elem.attrib[ 'id' ] = section_id + tool_section = ToolSection( new_section_elem ) + self.app.toolbox.tool_panel[ section_key ] = tool_section + else: + tool_section = None + current_working_dir = os.getcwd() + tool_shed_url = self.__get_url_from_tool_shed( self.tool_shed ) + repository_clone_url = os.path.join( tool_shed_url, 'repos', self.repository_owner, name ) + relative_install_dir = os.path.join( clone_dir, name ) + returncode, tmp_name = clone_repository( name, clone_dir, current_working_dir, repository_clone_url ) + if returncode == 0: + returncode, tmp_name = update_repository( current_working_dir, relative_install_dir, changeset_revision ) + if returncode == 0: + metadata_dict = load_repository_contents( self.app, + name, + description, + self.repository_owner, + changeset_revision, + repository_clone_url, + self.install_tool_config, + self.tool_path, + tool_section, + relative_install_dir, + current_working_dir, + tmp_name ) + # Add a new record to the tool_id_guid_map table for each + # tool in the repository if one doesn't already exist. + if 'tools' in metadata_dict: + tools_mapped = 0 + for tool_dict in metadata_dict[ 'tools' ]: + flush_needed = False + tool_id = tool_dict[ 'id' ] + tool_version = tool_dict[ 'version' ] + guid = tool_dict[ 'guid' ] + tool_id_guid_map = get_tool_id_guid_map( self.app, tool_id, tool_version, self.tool_shed, self.repository_owner, name ) + if tool_id_guid_map: + if tool_id_guid_map.guid != guid: + tool_id_guid_map.guid = guid + flush_needed = True + else: + tool_id_guid_map = self.app.model.ToolIdGuidMap( tool_id=tool_id, + tool_version=tool_version, + tool_shed=self.tool_shed, + repository_owner=self.repository_owner, + repository_name=name, + guid=guid ) + flush_needed = True + if flush_needed: + self.sa_session.add( tool_id_guid_map ) + self.sa_session.flush() + tools_mapped += 1 + log.debug( "Mapped tool ids to guids for %d tools included in repository '%s'." % ( tools_mapped, name ) ) + else: + tmp_stderr = open( tmp_name, 'rb' ) + log.debug( "Error updating repository '%s': %s" % ( name, tmp_stderr.read() ) ) + tmp_stderr.close() + else: + tmp_stderr = open( tmp_name, 'rb' ) + log.debug( "Error cloning repository '%s': %s" % ( name, tmp_stderr.read() ) ) + tmp_stderr.close() + def install_section( self, elem ): + # Install 1 or more repositories into a section in the tool config. An entry looks something like: + # <section name="EMBOSS" id="EMBOSSLite"> + # <repository name="emboss_5" description="Galaxy wrappers for EMBOSS version 5 tools" changeset_revision="bdd88ae5d0ac"> + # <tool file="emboss_5/emboss_antigenic.xml" id="EMBOSS: antigenic1" version="5.0.0" /> + # ... + # </repository> + # </section> + section_name = elem.get( 'name' ) + section_id = elem.get( 'id' ) + for repository_elem in elem: + self.install_repository( repository_elem, section_name=section_name, section_id=section_id ) + def __get_url_from_tool_shed( self, tool_shed ): + # The value of tool_shed is something like: toolshed.g2.bx.psu.edu + # We need the URL to this tool shed, which is something like: + # http://toolshed.g2.bx.psu.edu/ + for shed_name, shed_url in self.app.tool_shed_registry.tool_sheds.items(): + if shed_url.find( tool_shed ) >= 0: + if shed_url.endswith( '/' ): + shed_url = shed_url.rstrip( '/' ) + return shed_url + # The tool shed from which the repository was originally + # installed must no longer be configured in tool_sheds_conf.xml. + return None + def __isinstalled( self, repository_elem, clone_dir ): + name = repository_elem.get( 'name' ) + installed = False + for tool_elem in repository_elem: + tool_config = tool_elem.get( 'file' ) + tool_id = tool_elem.get( 'id' ) + tool_version = tool_elem.get( 'version' ) + tigm = get_tool_id_guid_map( self.app, tool_id, tool_version, self.tool_shed, self.repository_owner, name ) + if tigm: + # A record exists in the tool_id_guid_map table, so see if the repository is installed. + if os.path.exists( clone_dir ): + installed = True + break + return installed diff -r 4a39bc2094875a9878f07f27f7307976e05e8b87 -r 0c804033ae5a766f1d12a2030b73fe306c27f660 lib/galaxy/tool_shed/tool_shed_registry.py --- /dev/null +++ b/lib/galaxy/tool_shed/tool_shed_registry.py @@ -0,0 +1,31 @@ +import sys, logging +from galaxy.util import parse_xml +from galaxy.util.odict import odict + +log = logging.getLogger( __name__ ) + +if sys.version_info[:2] == ( 2, 4 ): + from galaxy import eggs + eggs.require( 'ElementTree' ) + from elementtree import ElementTree +else: + from xml.etree import ElementTree + +class Registry( object ): + def __init__( self, root_dir=None, config=None ): + self.tool_sheds = odict() + if root_dir and config: + # Parse datatypes_conf.xml + tree = parse_xml( config ) + root = tree.getroot() + # Load datatypes and converters from config + log.debug( 'Loading references to tool sheds from %s' % config ) + for elem in root.findall( 'tool_shed' ): + try: + name = elem.get( 'name', None ) + url = elem.get( 'url', None ) + if name and url: + self.tool_sheds[ name ] = url + log.debug( 'Loaded reference to tool shed: %s' % name ) + except Exception, e: + log.warning( 'Error loading reference to tool shed "%s", problem: %s' % ( name, str( e ) ) ) diff -r 4a39bc2094875a9878f07f27f7307976e05e8b87 -r 0c804033ae5a766f1d12a2030b73fe306c27f660 lib/galaxy/tool_shed/update_manager.py --- /dev/null +++ b/lib/galaxy/tool_shed/update_manager.py @@ -0,0 +1,67 @@ +""" +Determine if installed tool shed repositories have updates available in their respective tool sheds. +""" +import threading, urllib2, logging +from galaxy.util import string_as_bool +from galaxy.util.shed_util import * + +log = logging.getLogger( __name__ ) + +class UpdateManager( object ): + def __init__( self, app ): + """ + Check tool settings in tool_shed_install_config and install all tools that are + not already installed. The tool panel configuration file is the received + shed_tool_config, which defaults to shed_tool_conf.xml. + """ + self.app = app + self.sa_session = self.app.model.context.current + # Ideally only one Galaxy server process + # should be able to check for repository updates. + self.running = True + self.sleeper = Sleeper() + self.restarter = threading.Thread( target=self.__restarter ) + self.restarter.start() + self.seconds_to_sleep = app.config.hours_between_check * 3600 + def __restarter( self ): + log.info( 'Update manager restarter starting up...' ) + while self.running: + flush_needed = False + for repository in self.sa_session.query( self.app.model.ToolShedRepository ) \ + .filter( and_( self.app.model.ToolShedRepository.table.c.update_available == False, + self.app.model.ToolShedRepository.table.c.deleted == False ) ): + if self.check_for_update( repository ): + repository.update_available = True + self.sa_session.add( repository ) + flush_needed = True + if flush_needed: + self.sa_session.flush() + self.sleeper.sleep( self.seconds_to_sleep ) + log.info( 'Transfer job restarter shutting down...' ) + def check_for_update( self, repository ): + tool_shed_url = get_url_from_repository_tool_shed( self.app, repository ) + url = '%s/repository/check_for_updates?name=%s&owner=%s&changeset_revision=%s&webapp=update_manager' % \ + ( tool_shed_url, repository.name, repository.owner, repository.changeset_revision ) + response = urllib2.urlopen( url ) + text = response.read() + response.close() + return string_as_bool( text ) + def shutdown( self ): + self.running = False + self.sleeper.wake() + +class Sleeper( object ): + """ + Provides a 'sleep' method that sleeps for a number of seconds *unless* + the notify method is called (from a different thread). + """ + def __init__( self ): + self.condition = threading.Condition() + def sleep( self, seconds ): + self.condition.acquire() + self.condition.wait( seconds ) + self.condition.release() + def wake( self ): + self.condition.acquire() + self.condition.notify() + self.condition.release() diff -r 4a39bc2094875a9878f07f27f7307976e05e8b87 -r 0c804033ae5a766f1d12a2030b73fe306c27f660 lib/galaxy/tools/install_manager.py --- a/lib/galaxy/tools/install_manager.py +++ /dev/null @@ -1,160 +0,0 @@ -""" -Manage automatic installation of tools configured in tool_shed_install.xml, all of which were -at some point included in the Galaxy distribution, but are now hosted in the main Galaxy tool -shed. Tools included in tool_shed_install.xml that have already been installed will not be -re-installed. -""" -from galaxy.util.shed_util import * - -log = logging.getLogger( __name__ ) - -class InstallManager( object ): - def __init__( self, app, tool_shed_install_config, install_tool_config ): - """ - Check tool settings in tool_shed_install_config and install all tools that are - not already installed. The tool panel configuration file is the received - shed_tool_config, which defaults to shed_tool_conf.xml. - """ - self.app = app - self.sa_session = self.app.model.context.current - self.install_tool_config = install_tool_config - # Parse shed_tool_config to get the install location (tool_path). - tree = util.parse_xml( install_tool_config ) - root = tree.getroot() - self.tool_path = root.get( 'tool_path' ) - self.app.toolbox.shed_tool_confs[ install_tool_config ] = self.tool_path - # Parse tool_shed_install_config to check each of the tools. - log.debug( "Parsing tool shed install configuration %s" % tool_shed_install_config ) - self.tool_shed_install_config = tool_shed_install_config - tree = util.parse_xml( tool_shed_install_config ) - root = tree.getroot() - self.tool_shed = clean_tool_shed_url( root.get( 'name' ) ) - log.debug( "Repositories will be installed from tool shed '%s' into configured tool_path location '%s'" % ( str( self.tool_shed ), str( self.tool_path ) ) ) - self.repository_owner = 'devteam' - for elem in root: - if elem.tag == 'repository': - self.install_repository( elem ) - elif elem.tag == 'section': - self.install_section( elem ) - def install_repository( self, elem, section_name='', section_id='' ): - # Install a single repository into the tool config. If outside of any sections, the entry looks something like: - # <repository name="cut_wrapper" description="Galaxy wrapper for the Cut tool" changeset_revision="f3ed6cfe6402"> - # <tool id="Cut1" version="1.0.1" /> - # </repository> - name = elem.get( 'name' ) - description = elem.get( 'description' ) - changeset_revision = elem.get( 'changeset_revision' ) - # Install path is of the form: <tool path>/<tool shed>/repos/<repository owner>/<repository name>/<changeset revision> - clone_dir = os.path.join( self.tool_path, self.tool_shed, 'repos', self.repository_owner, name, changeset_revision ) - if self.__isinstalled( elem, clone_dir ): - log.debug( "Skipping automatic install of repository '%s' because it has already been installed in location '%s'" % ( name, clone_dir ) ) - else: - if section_name and section_id: - section_key = 'section_%s' % str( section_id ) - if section_key in self.app.toolbox.tool_panel: - # Appending a tool to an existing section in self.app.toolbox.tool_panel - log.debug( "Appending to tool panel section: %s" % section_name ) - tool_section = self.app.toolbox.tool_panel[ section_key ] - else: - # Appending a new section to self.app.toolbox.tool_panel - log.debug( "Loading new tool panel section: %s" % section_name ) - new_section_elem = Element( 'section' ) - new_section_elem.attrib[ 'name' ] = section_name - new_section_elem.attrib[ 'id' ] = section_id - tool_section = ToolSection( new_section_elem ) - self.app.toolbox.tool_panel[ section_key ] = tool_section - else: - tool_section = None - current_working_dir = os.getcwd() - tool_shed_url = self.__get_url_from_tool_shed( self.tool_shed ) - repository_clone_url = os.path.join( tool_shed_url, 'repos', self.repository_owner, name ) - relative_install_dir = os.path.join( clone_dir, name ) - returncode, tmp_name = clone_repository( name, clone_dir, current_working_dir, repository_clone_url ) - if returncode == 0: - returncode, tmp_name = update_repository( current_working_dir, relative_install_dir, changeset_revision ) - if returncode == 0: - metadata_dict = load_repository_contents( self.app, - name, - description, - self.repository_owner, - changeset_revision, - repository_clone_url, - self.install_tool_config, - self.tool_path, - tool_section, - relative_install_dir, - current_working_dir, - tmp_name ) - # Add a new record to the tool_id_guid_map table for each - # tool in the repository if one doesn't already exist. - if 'tools' in metadata_dict: - tools_mapped = 0 - for tool_dict in metadata_dict[ 'tools' ]: - flush_needed = False - tool_id = tool_dict[ 'id' ] - tool_version = tool_dict[ 'version' ] - guid = tool_dict[ 'guid' ] - tool_id_guid_map = get_tool_id_guid_map( self.app, tool_id, tool_version, self.tool_shed, self.repository_owner, name ) - if tool_id_guid_map: - if tool_id_guid_map.guid != guid: - tool_id_guid_map.guid = guid - flush_needed = True - else: - tool_id_guid_map = self.app.model.ToolIdGuidMap( tool_id=tool_id, - tool_version=tool_version, - tool_shed=self.tool_shed, - repository_owner=self.repository_owner, - repository_name=name, - guid=guid ) - flush_needed = True - if flush_needed: - self.sa_session.add( tool_id_guid_map ) - self.sa_session.flush() - tools_mapped += 1 - log.debug( "Mapped tool ids to guids for %d tools included in repository '%s'." % ( tools_mapped, name ) ) - else: - tmp_stderr = open( tmp_name, 'rb' ) - log.debug( "Error updating repository '%s': %s" % ( name, tmp_stderr.read() ) ) - tmp_stderr.close() - else: - tmp_stderr = open( tmp_name, 'rb' ) - log.debug( "Error cloning repository '%s': %s" % ( name, tmp_stderr.read() ) ) - tmp_stderr.close() - def install_section( self, elem ): - # Install 1 or more repositories into a section in the tool config. An entry looks something like: - # <section name="EMBOSS" id="EMBOSSLite"> - # <repository name="emboss_5" description="Galaxy wrappers for EMBOSS version 5 tools" changeset_revision="bdd88ae5d0ac"> - # <tool file="emboss_5/emboss_antigenic.xml" id="EMBOSS: antigenic1" version="5.0.0" /> - # ... - # </repository> - # </section> - section_name = elem.get( 'name' ) - section_id = elem.get( 'id' ) - for repository_elem in elem: - self.install_repository( repository_elem, section_name=section_name, section_id=section_id ) - def __get_url_from_tool_shed( self, tool_shed ): - # The value of tool_shed is something like: toolshed.g2.bx.psu.edu - # We need the URL to this tool shed, which is something like: - # http://toolshed.g2.bx.psu.edu/ - for shed_name, shed_url in self.app.tool_shed_registry.tool_sheds.items(): - if shed_url.find( tool_shed ) >= 0: - if shed_url.endswith( '/' ): - shed_url = shed_url.rstrip( '/' ) - return shed_url - # The tool shed from which the repository was originally - # installed must no longer be configured in tool_sheds_conf.xml. - return None - def __isinstalled( self, repository_elem, clone_dir ): - name = repository_elem.get( 'name' ) - installed = False - for tool_elem in repository_elem: - tool_config = tool_elem.get( 'file' ) - tool_id = tool_elem.get( 'id' ) - tool_version = tool_elem.get( 'version' ) - tigm = get_tool_id_guid_map( self.app, tool_id, tool_version, self.tool_shed, self.repository_owner, name ) - if tigm: - # A record exists in the tool_id_guid_map table, so see if the repository is installed. - if os.path.exists( clone_dir ): - installed = True - break - return installed diff -r 4a39bc2094875a9878f07f27f7307976e05e8b87 -r 0c804033ae5a766f1d12a2030b73fe306c27f660 lib/galaxy/tools/tool_shed_registry.py --- a/lib/galaxy/tools/tool_shed_registry.py +++ /dev/null @@ -1,31 +0,0 @@ -import sys, logging -from galaxy.util import parse_xml -from galaxy.util.odict import odict - -log = logging.getLogger( __name__ ) - -if sys.version_info[:2] == ( 2, 4 ): - from galaxy import eggs - eggs.require( 'ElementTree' ) - from elementtree import ElementTree -else: - from xml.etree import ElementTree - -class Registry( object ): - def __init__( self, root_dir=None, config=None ): - self.tool_sheds = odict() - if root_dir and config: - # Parse datatypes_conf.xml - tree = parse_xml( config ) - root = tree.getroot() - # Load datatypes and converters from config - log.debug( 'Loading references to tool sheds from %s' % config ) - for elem in root.findall( 'tool_shed' ): - try: - name = elem.get( 'name', None ) - url = elem.get( 'url', None ) - if name and url: - self.tool_sheds[ name ] = url - log.debug( 'Loaded reference to tool shed: %s' % name ) - except Exception, e: - log.warning( 'Error loading reference to tool shed "%s", problem: %s' % ( name, str( e ) ) ) diff -r 4a39bc2094875a9878f07f27f7307976e05e8b87 -r 0c804033ae5a766f1d12a2030b73fe306c27f660 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -65,7 +65,8 @@ return tool_shed_url.rstrip( '/' ) def clone_repository( name, clone_dir, current_working_dir, repository_clone_url ): log.debug( "Installing repository '%s'" % name ) - os.makedirs( clone_dir ) + if not os.path.exists( clone_dir ): + os.makedirs( clone_dir ) log.debug( 'Cloning %s' % repository_clone_url ) cmd = 'hg clone %s' % repository_clone_url tmp_name = tempfile.NamedTemporaryFile().name @@ -88,15 +89,18 @@ tool_shed_repository = get_repository_by_shed_name_owner_changeset_revision( app, tool_shed, name, owner, changeset_revision ) if tool_shed_repository: if tool_shed_repository.deleted: + tool_shed_repository.description = description + tool_shed_repository.changeset_revision = changeset_revision + tool_shed_repository.metadata = metadata_dict + tool_shed_repository.includes_datatypes = includes_datatypes tool_shed_repository.deleted = False - # Reset includes_datatypes in case metadata changed since last installed. - tool_shed_repository.includes_datatypes = includes_datatypes flush_needed = True else: tool_shed_repository = app.model.ToolShedRepository( tool_shed=tool_shed, name=name, description=description, owner=owner, + installed_changeset_revision=changeset_revision, changeset_revision=changeset_revision, metadata=metadata_dict, includes_datatypes=includes_datatypes ) @@ -318,6 +322,20 @@ app.model.ToolIdGuidMap.table.c.repository_owner == repository_owner, app.model.ToolIdGuidMap.table.c.repository_name == repository_name ) ) \ .first() +def get_url_from_repository_tool_shed( app, repository ): + """ + This method is used by the UpdateManager, which does not have access to trans. + The stored value of repository.tool_shed is something like: toolshed.g2.bx.psu.edu + We need the URL to this tool shed, which is something like: http://toolshed.g2.bx.psu.edu/ + """ + for shed_name, shed_url in app.tool_shed_registry.tool_sheds.items(): + if shed_url.find( repository.tool_shed ) >= 0: + if shed_url.endswith( '/' ): + shed_url = shed_url.rstrip( '/' ) + return shed_url + # The tool shed from which the repository was originally + # installed must no longer be configured in tool_sheds_conf.xml. + return None def handle_missing_data_table_entry( app, tool_path, sample_files, repository_tools_tups ): """ Inspect each tool to see if any have input parameters that are dynamically @@ -532,7 +550,19 @@ if level and ( not elem.tail or not elem.tail.strip() ): elem.tail = i + pad return elem -def update_repository( current_working_dir, relative_install_dir, changeset_revision ): +def pull_repository( current_working_dir, repo_files_dir, name ): + # Pull the latest possible contents to the repository. + log.debug( "Pulling latest updates to the repository named '%s'" % name ) + cmd = 'hg pull' + tmp_name = tempfile.NamedTemporaryFile().name + tmp_stderr = open( tmp_name, 'wb' ) + os.chdir( repo_files_dir ) + proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + os.chdir( current_working_dir ) + tmp_stderr.close() + return returncode, tmp_name +def update_repository( current_working_dir, repo_files_dir, changeset_revision ): # Update the cloned repository to changeset_revision. It is imperative that the # installed repository is updated to the desired changeset_revision before metadata # is set because the process for setting metadata uses the repository files on disk. @@ -540,7 +570,7 @@ cmd = 'hg update -r %s' % changeset_revision tmp_name = tempfile.NamedTemporaryFile().name tmp_stderr = open( tmp_name, 'wb' ) - os.chdir( relative_install_dir ) + os.chdir( repo_files_dir ) proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) returncode = proc.wait() os.chdir( current_working_dir ) diff -r 4a39bc2094875a9878f07f27f7307976e05e8b87 -r 0c804033ae5a766f1d12a2030b73fe306c27f660 lib/galaxy/web/controllers/admin_toolshed.py --- a/lib/galaxy/web/controllers/admin_toolshed.py +++ b/lib/galaxy/web/controllers/admin_toolshed.py @@ -3,6 +3,55 @@ log = logging.getLogger( __name__ ) +class ToolIdGuidMapGrid( grids.Grid ): + class ToolIdColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_id_guid_map ): + return tool_id_guid_map.tool_id + class ToolVersionColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_id_guid_map ): + return tool_id_guid_map.tool_version + class ToolGuidColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_id_guid_map ): + return tool_id_guid_map.guid + class ToolShedColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_id_guid_map ): + return tool_id_guid_map.tool_shed + class RepositoryNameColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_id_guid_map ): + return tool_id_guid_map.repository_name + class RepositoryOwnerColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_id_guid_map ): + return tool_id_guid_map.repository_owner + # Grid definition + title = "Map tool id to guid" + model_class = model.ToolIdGuidMap + template='/admin/tool_shed_repository/grid.mako' + default_sort_key = "tool_id" + columns = [ + ToolIdColumn( "Tool id" ), + ToolVersionColumn( "Version" ), + ToolGuidColumn( "Guid" ), + ToolShedColumn( "Tool shed" ), + RepositoryNameColumn( "Repository name" ), + RepositoryOwnerColumn( "Repository owner" ) + ] + columns.append( grids.MulticolFilterColumn( "Search repository name", + cols_to_filter=[ columns[0], columns[2], columns[4], columns[5] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + global_actions = [ + grids.GridAction( "Manage installed tool shed repositories", dict( controller='admin_toolshed', action='browse_repositories' ) ) + ] + operations = [] + standard_filters = [] + default_filter = {} + num_rows_per_page = 50 + preserve_state = False + use_paging = True + def build_initial_query( self, trans, **kwd ): + return trans.sa_session.query( self.model_class ) + class RepositoryListGrid( grids.Grid ): class NameColumn( grids.TextColumn ): def get_value( self, trans, grid, tool_shed_repository ): @@ -46,6 +95,9 @@ key="free-text-search", visible=False, filterable="standard" ) ) + global_actions = [ + grids.GridAction( "View tool id guid map", dict( controller='admin_toolshed', action='browse_tool_id_guid_map' ) ) + ] operations = [ grids.GridOperation( "Get updates", allow_multiple=False, condition=( lambda item: not item.deleted ), @@ -62,9 +114,14 @@ class AdminToolshed( AdminGalaxy ): repository_list_grid = RepositoryListGrid() + tool_id_guid_map_grid = ToolIdGuidMapGrid() @web.expose @web.require_admin + def browse_tool_id_guid_map( self, trans, **kwd ): + return self.tool_id_guid_map_grid( trans, **kwd ) + @web.expose + @web.require_admin def browse_repository( self, trans, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) @@ -272,7 +329,7 @@ def check_for_updates( self, trans, **kwd ): # Send a request to the relevant tool shed to see if there are any updates. repository = get_repository( trans, kwd[ 'id' ] ) - tool_shed_url = get_url_from_repository_tool_shed( trans, repository ) + tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) url = '%s/repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ ( tool_shed_url, url_for( '', qualified=True ), repository.name, repository.owner, repository.changeset_revision ) return trans.response.send_redirect( url ) @@ -296,30 +353,14 @@ current_working_dir = os.getcwd() relative_install_dir = self.__get_relative_install_dir( trans, repository ) if relative_install_dir: - # Update the cloned repository to changeset_revision. repo_files_dir = os.path.join( relative_install_dir, name ) - log.debug( "Updating cloned repository named '%s' from revision '%s' to revision '%s'..." % \ - ( name, changeset_revision, latest_changeset_revision ) ) - cmd = 'hg pull' - tmp_name = tempfile.NamedTemporaryFile().name - tmp_stderr = open( tmp_name, 'wb' ) - os.chdir( repo_files_dir ) - proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) - returncode = proc.wait() - os.chdir( current_working_dir ) - tmp_stderr.close() + returncode, tmp_name = pull_repository( current_working_dir, repo_files_dir, name ) if returncode == 0: - cmd = 'hg update -r %s' % latest_changeset_revision - tmp_name = tempfile.NamedTemporaryFile().name - tmp_stderr = open( tmp_name, 'wb' ) - os.chdir( repo_files_dir ) - proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) - returncode = proc.wait() - os.chdir( current_working_dir ) - tmp_stderr.close() + returncode, tmp_name = update_repository( current_working_dir, repo_files_dir, latest_changeset_revision ) if returncode == 0: # Update the repository changeset_revision in the database. repository.changeset_revision = latest_changeset_revision + repository.update_available = False trans.sa_session.add( repository ) trans.sa_session.flush() message = "The cloned repository named '%s' has been updated to change set revision '%s'." % \ @@ -370,7 +411,7 @@ def __get_relative_install_dir( self, trans, repository ): # Get the directory where the repository is install. tool_shed = clean_tool_shed_url( repository.tool_shed ) - partial_install_dir = '%s/repos/%s/%s/%s' % ( tool_shed, repository.owner, repository.name, repository.changeset_revision ) + partial_install_dir = '%s/repos/%s/%s/%s' % ( tool_shed, repository.owner, repository.name, repository.installed_changeset_revision ) # Get the relative tool installation paths from each of the shed tool configs. shed_tool_confs = trans.app.toolbox.shed_tool_confs relative_install_dir = None @@ -396,7 +437,7 @@ return '%s/repos%s/%s' % ( tool_shed_url, repo_path, changeset_revision ) def __generate_clone_url( self, trans, repository ): """Generate the URL for cloning a repository.""" - tool_shed_url = get_url_from_repository_tool_shed( trans, repository ) + tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) return '%s/repos/%s/%s' % ( tool_shed_url, repository.owner, repository.name ) ## ---- Utility methods ------------------------------------------------------- @@ -426,23 +467,3 @@ def get_repository( trans, id ): """Get a tool_shed_repository from the database via id""" return trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( id ) ) -def get_repository_by_name_owner_changeset_revision( trans, name, owner, changeset_revision ): - """Get a repository from the database via name owner and changeset_revision""" - return trans.sa_session.query( trans.model.ToolShedRepository ) \ - .filter( and_( trans.model.ToolShedRepository.table.c.name == name, - trans.model.ToolShedRepository.table.c.owner == owner, - trans.model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ - .first() -def get_url_from_repository_tool_shed( trans, repository ): - # The stored value of repository.tool_shed is something like: - # toolshed.g2.bx.psu.edu - # We need the URL to this tool shed, which is something like: - # http://toolshed.g2.bx.psu.edu/ - for shed_name, shed_url in trans.app.tool_shed_registry.tool_sheds.items(): - if shed_url.find( repository.tool_shed ) >= 0: - if shed_url.endswith( '/' ): - shed_url = shed_url.rstrip( '/' ) - return shed_url - # The tool shed from which the repository was originally - # installed must no longer be configured in tool_sheds_conf.xml. - return None diff -r 4a39bc2094875a9878f07f27f7307976e05e8b87 -r 0c804033ae5a766f1d12a2030b73fe306c27f660 lib/galaxy/webapps/community/controllers/repository.py --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -764,22 +764,32 @@ return trans.response.send_redirect( url ) @web.expose def check_for_updates( self, trans, **kwd ): + # Handle a request from a local Galaxy instance. If the request originated with the + # Galaxy instances' UpdateManager, the value of 'webapp' will be 'update_manager'. params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) - galaxy_url = kwd[ 'galaxy_url' ] + # If the request originated with the UpdateManager, it will not include a galaxy_url. + galaxy_url = kwd.get( 'galaxy_url', '' ) name = params.get( 'name', None ) owner = params.get( 'owner', None ) changeset_revision = params.get( 'changeset_revision', None ) webapp = params.get( 'webapp', 'community' ) - # Start building up the url to redirect back to the calling Galaxy instance. - url = '%s/admin_toolshed/update_to_changeset_revision?tool_shed_url=%s' % ( galaxy_url, url_for( '', qualified=True ) ) repository = get_repository_by_name_and_owner( trans, name, owner ) - url += '&name=%s&owner=%s&changeset_revision=%s&latest_changeset_revision=' % \ - ( repository.name, repository.user.username, changeset_revision ) + from_update_manager = webapp == 'update_manager' + if from_update_manager: + update = 'true' + no_update = 'false' + else: + # Start building up the url to redirect back to the calling Galaxy instance. + url = '%s/admin_toolshed/update_to_changeset_revision?tool_shed_url=%s' % ( galaxy_url, url_for( '', qualified=True ) ) + url += '&name=%s&owner=%s&changeset_revision=%s&latest_changeset_revision=' % \ + ( repository.name, repository.user.username, changeset_revision ) if changeset_revision == repository.tip: # If changeset_revision is the repository tip, then # we know there are no additional updates for the tools. + if from_update_manager: + return no_update url += repository.tip else: repository_metadata = get_repository_metadata_by_changeset_revision( trans, @@ -788,6 +798,8 @@ if repository_metadata: # If changeset_revision is in the repository_metadata table for this # repository, then we know there are no additional updates for the tools. + if from_update_manager: + return no_update url += changeset_revision else: # The changeset_revision column in the repository_metadata table has been @@ -836,15 +848,21 @@ if tool_guids == metadata_tool_guids: # We've found the repository_metadata record whose changeset_revision # value has been updated. + if from_update_manager: + return update url += repository_metadata.changeset_revision found = True break if not found: # There must be a problem in the data, so we'll just send back the received changeset_revision. log.debug( "Possible data corruption - updated repository_metadata cannot be found for repository id %d." % repository.id ) + if from_update_manager: + return no_update url += changeset_revision else: # There are not tools in the changeset_revision, so no tool updates are possible. + if from_update_manager: + return no_update url += changeset_revision return trans.response.send_redirect( url ) @web.expose diff -r 4a39bc2094875a9878f07f27f7307976e05e8b87 -r 0c804033ae5a766f1d12a2030b73fe306c27f660 universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -147,6 +147,13 @@ # if appropriate or use a different file name for the setting. #install_tool_config_file = shed_tool_conf.xml +# Enable automatic polling of relative tool sheds to see if any updates +# are available for installed repositories. Ideally only one Galaxy +# server process should be able to check for repository updates. The +# setting for hours_between_check should be an integer between 1 and 24. +#enable_tool_shed_check = False +#hours_between_check = 12 + # Directory where data used by tools is located, see the samples in that # directory and the wiki for help: # http://wiki.g2.bx.psu.edu/Admin/Data%20Integration Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.