1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/185232cef732/ Changeset: 185232cef732 User: greg Date: 2013-08-22 16:12:08 Summary: Automatically eliminate tool entries from non-shed-related tool panel configs for tools contained in tool shed repositories that were migrated out of the Galaxy distribution. Affected #: 5 files diff -r 097abeb6969e93265135a8d6f3e566b5e913ca8f -r 185232cef732d9b79aad686ca0aa556bb7c63614 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -3,7 +3,9 @@ All of the tools were at some point included in the Galaxy distribution, but are now hosted in the main Galaxy tool shed. """ import os +import shutil import tempfile +import threading from galaxy import util from galaxy.tools import ToolSection from galaxy.util.json import from_json_string @@ -63,7 +65,9 @@ if tool_panel_configs: # The missing_tool_configs_dict contents are something like: # {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]} - tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, tool_panel_configs, latest_migration_script_number ) + tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, + tool_panel_configs, + latest_migration_script_number ) else: # It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in @@ -79,7 +83,9 @@ file_names = ', '.join( self.proprietary_tool_confs ) if missing_tool_configs_dict: for repository_elem in root: - self.install_repository( repository_elem, install_dependencies ) + # Make sure we have a valid repository tag. + if self.__is_valid_repository_tag( repository_elem ): + self.install_repository( repository_elem, install_dependencies ) else: message = "\nNo tools associated with migration stage %s are defined in your " % str( latest_migration_script_number ) message += "file%s named %s,\nso no repositories will be installed on disk.\n" % ( plural, file_names ) @@ -89,6 +95,43 @@ message += "Try again later.\n" print message + def filter_and_persist_proprietary_tool_panel_configs( self, tool_configs_to_filter ): + """Eliminate all entries in all non-shed-related tool panel configs for all tool config file names in the received tool_configs_to_filter.""" + for proprietary_tool_conf in self.proprietary_tool_confs: + persist_required = False + tree, error_message = xml_util.parse_xml( proprietary_tool_conf ) + if tree: + root = tree.getroot() + for elem in root: + if elem.tag == 'tool': + # Tools outside of sections. + file_path = elem.get( 'file', None ) + if file_path: + file_name = suc.strip_path( file_path ) + if file_name in tool_configs_to_filter: + root.remove( elem ) + persist_required = True + elif elem.tag == 'section': + # Tools contained in a section. + for section_elem in elem: + if section_elem.tag == 'tool': + file_path = section_elem.get( 'file', None ) + if file_path: + file_name = suc.strip_path( file_path ) + + if file_name in tool_configs_to_filter: + elem.remove( section_elem ) + persist_required = True + if persist_required: + fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-toolshed-fapptpc" ) + tmp_filename = fh.name + fh.close() + fh = open( tmp_filename, 'wb' ) + tree.write( tmp_filename ) + fh.close() + shutil.move( tmp_filename, os.path.abspath( proprietary_tool_conf ) ) + os.chmod( proprietary_tool_conf, 0644 ) + def get_guid( self, repository_clone_url, relative_install_dir, tool_config ): if self.shed_config_dict.get( 'tool_path' ): relative_install_dir = os.path.join( self.shed_config_dict['tool_path'], relative_install_dir ) @@ -108,9 +151,11 @@ return suc.generate_tool_guid( repository_clone_url, tool ) def get_proprietary_tool_panel_elems( self, latest_tool_migration_script_number ): - # Parse each config in self.proprietary_tool_confs (the default is tool_conf.xml) and generate a list of Elements that are - # either ToolSection elements or Tool elements. These will be used to generate new entries in the migrated_tools_conf.xml - # file for the installed tools. + """ + Parse each config in self.proprietary_tool_confs (the default is tool_conf.xml) and generate a list of Elements that are + either ToolSection elements or Tool elements. These will be used to generate new entries in the migrated_tools_conf.xml + file for the installed tools. + """ tools_xml_file_path = os.path.abspath( os.path.join( 'scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number ) ) # Parse the XML and load the file attributes for later checking against the integrated elements from self.proprietary_tool_confs. migrated_tool_configs = [] @@ -183,7 +228,13 @@ return is_displayed, tool_sections def handle_repository_contents( self, tool_shed_repository, repository_clone_url, relative_install_dir, repository_elem, install_dependencies ): - """Generate the metadata for the installed tool shed repository, among other things.""" + """ + Generate the metadata for the installed tool shed repository, among other things. If the installed tool_shed_repository contains tools + that are loaded into the Galaxy tool panel, this method will automatically eliminate all entries for each of the tools defined in the + received repository_elem from all non-shed-related tool panel configuration files since the entries are automatically added to the reserved + migrated_tools_conf.xml file as part of the migration process. + """ + tool_configs_to_filter = [] tool_panel_dict_for_display = odict() if self.tool_path: repo_install_dir = os.path.join( self.tool_path, relative_install_dir ) @@ -197,11 +248,29 @@ is_displayed, tool_sections = self.get_containing_tool_sections( tool_config ) if is_displayed: tool_panel_dict_for_tool_config = tool_util.generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=tool_sections ) + # The tool-panel_dict has the following structure. + # {<Tool guid> : [{ tool_config : <tool_config_file>, id: <ToolSection id>, version : <ToolSection version>, name : <TooSection name>}]} for k, v in tool_panel_dict_for_tool_config.items(): tool_panel_dict_for_display[ k ] = v + for tool_panel_dict in v: + # Keep track of tool config file names associated with entries that have been made to the migrated_tools_conf.xml file so + # they can be eliminated from all non-shed-related tool panel configs. + tool_config_file = tool_panel_dict.get( 'tool_config', None ) + if tool_config_file: + if tool_config_file not in tool_configs_to_filter: + tool_configs_to_filter.append( tool_config_file ) else: print 'The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).' \ % ( guid, tool_config, ", ".join( self.proprietary_tool_confs or [] ) ) + if tool_configs_to_filter: + lock = threading.Lock() + lock.acquire( True ) + try: + self.filter_and_persist_proprietary_tool_panel_configs( tool_configs_to_filter ) + except Exception, e: + log.exception( "Exception attempting to filter and persist non-shed-related tool panel configs:\n%s" % str( e ) ) + finally: + lock.release() metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=self.app, repository=tool_shed_repository, changeset_revision=tool_shed_repository.changeset_revision, @@ -384,3 +453,13 @@ # Assume that the repository has been installed if we find a .hg directory. return True return False + + def __is_valid_repository_tag( self, elem ): + # <repository name="emboss_datatypes" description="Datatypes for Emboss tools" changeset_revision="a89163f31369" /> + if elem.tag != 'repository': + return False + if not elem.get( 'name' ): + return False + if not elem.get( 'changeset_revision' ): + return False + return True diff -r 097abeb6969e93265135a8d6f3e566b5e913ca8f -r 185232cef732d9b79aad686ca0aa556bb7c63614 lib/tool_shed/galaxy_install/migrate/check.py --- a/lib/tool_shed/galaxy_install/migrate/check.py +++ b/lib/tool_shed/galaxy_install/migrate/check.py @@ -1,13 +1,17 @@ -import sys, os, logging, subprocess +import logging +import os +import subprocess +import sys from galaxy import eggs -import pkg_resources -pkg_resources.require( "SQLAlchemy" ) -pkg_resources.require( "decorator" ) -pkg_resources.require( "Tempita " ) -pkg_resources.require( "sqlalchemy-migrate" ) - -from migrate.versioning import repository, schema -from sqlalchemy import * +eggs.require( "decorator" ) +eggs.require( "Tempita" ) +eggs.require( "SQLAlchemy" ) +eggs.require( "sqlalchemy_migrate" ) +from migrate.versioning import repository +from migrate.versioning import schema +from sqlalchemy import create_engine +from sqlalchemy import MetaData +from sqlalchemy import Table from galaxy.util.odict import odict from galaxy.model.orm import dialect_to_egg from tool_shed.util import common_util @@ -52,7 +56,9 @@ if tool_panel_configs: # The missing_tool_configs_dict contents are something like: # {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]} - tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_script_number ) + tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, + tool_panel_configs, + latest_tool_migration_script_number ) else: # It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in @@ -136,8 +142,8 @@ msg += "Tool dependencies can be installed after the repositories have been installed as well.\n\n" msg += "After the installation process finishes, you can start your Galaxy server. As part of this installation process,\n" msg += "entries for each of the following tool config files will be added to the file named ./migrated_tool_conf.xml, so these\n" - msg += "tools will continue to be loaded into your tool panel. Because of this, existing entries for these files should be\n" - msg += "removed from your file%s named %s, but only after the installation process finishes.\n\n" % ( plural, tool_panel_config_file_names ) + msg += "tools will continue to be loaded into your tool panel. Because of this, existing entries for these tools have been\n" + msg += "removed from your file%s named %s.\n\n" % ( plural, tool_panel_config_file_names ) for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items(): msg += "%s\n" % missing_tool_config msg += "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n" diff -r 097abeb6969e93265135a8d6f3e566b5e913ca8f -r 185232cef732d9b79aad686ca0aa556bb7c63614 lib/tool_shed/galaxy_install/migrate/common.py --- a/lib/tool_shed/galaxy_install/migrate/common.py +++ b/lib/tool_shed/galaxy_install/migrate/common.py @@ -1,8 +1,10 @@ -import sys, os, ConfigParser +import ConfigParser +import os +import sys import galaxy.config import galaxy.datatypes.registry from galaxy import tools -from galaxy.tools.data import * +from galaxy.tools.data import ToolDataTableManager from galaxy.web import security import galaxy.model.mapping import galaxy.tools.search @@ -10,8 +12,10 @@ import tool_shed.tool_shed_registry from tool_shed.galaxy_install import install_manager + class MigrateToolsApplication( object ): """Encapsulates the state of a basic Galaxy Universe application in order to initiate the Install Manager""" + def __init__( self, tools_migration_config ): install_dependencies = 'install_dependencies' in sys.argv galaxy_config_file = 'universe_wsgi.ini' @@ -23,7 +27,7 @@ if not os.path.exists( galaxy_config_file ): print "Galaxy config file does not exist (hint: use '-c config.ini' for non-standard locations): %s" % galaxy_config_file sys.exit( 1 ) - config_parser = ConfigParser.ConfigParser( { 'here':os.getcwd() } ) + config_parser = ConfigParser.ConfigParser( { 'here' : os.getcwd() } ) config_parser.read( galaxy_config_file ) galaxy_config_dict = {} for key, value in config_parser.items( "app:main" ): @@ -77,8 +81,10 @@ tools_migration_config ), migrated_tools_config=self.config.migrated_tools_config, install_dependencies=install_dependencies ) + @property def sa_session( self ): return self.model.context.current + def shutdown( self ): self.object_store.shutdown() diff -r 097abeb6969e93265135a8d6f3e566b5e913ca8f -r 185232cef732d9b79aad686ca0aa556bb7c63614 lib/tool_shed/util/common_util.py --- a/lib/tool_shed/util/common_util.py +++ b/lib/tool_shed/util/common_util.py @@ -76,7 +76,7 @@ return missing_tool_configs_dict def get_non_shed_tool_panel_configs( app ): - # Get the non-shed related tool panel configs - there can be more than one, and the default is tool_conf.xml. + """Get the non-shed related tool panel configs - there can be more than one, and the default is tool_conf.xml.""" config_filenames = [] for config_filename in app.config.tool_configs: # Any config file that includes a tool_path attribute in the root tag set like the following is shed-related. @@ -98,13 +98,13 @@ shed_url = shed_url.rstrip( '/' ) return shed_url return None - + def tool_shed_get( app, tool_shed_url, uri ): """Make contact with the tool shed via the uri provided.""" registry = app.tool_shed_registry urlopener = urllib2.build_opener() password_mgr = registry.password_manager_for_url( tool_shed_url ) - if ( password_mgr is not None ): + if password_mgr is not None: auth_handler = urllib2.HTTPBasicAuthHandler( password_mgr ) urlopener.add_handler( auth_handler ) response = urlopener.open( uri ) diff -r 097abeb6969e93265135a8d6f3e566b5e913ca8f -r 185232cef732d9b79aad686ca0aa556bb7c63614 scripts/migrate_tools/migrate_tools.py --- a/scripts/migrate_tools/migrate_tools.py +++ b/scripts/migrate_tools/migrate_tools.py @@ -1,4 +1,3 @@ - """ This script will start up its own web application which includes an InstallManager (~/lib/galaxy/tool_shed/install_manager.py). For each tool discovered missing, the tool shed repository that contains it will be installed on disk and a new entry will be @@ -8,11 +7,12 @@ layout of the local tool_conf.xml file. Entries will not be created in migrated_tools_conf.xml for tools included in the tool shed repository but not defined in tool_conf.xml. """ -import sys, os +import os +import sys new_path = [ os.path.join( os.getcwd(), "lib" ) ] # Remove scripts/ from the path. -new_path.extend( sys.path[1:] ) +new_path.extend( sys.path[ 1: ] ) sys.path = new_path from galaxy import eggs @@ -26,8 +26,8 @@ else: plural = 's' file_names = ', '.join( non_shed_tool_confs ) -msg = "\nThe installation process is finished. If any tools associated with this migration were defined in your file%s named\n" % plural -msg += "%s, then you should remove entries for them and start your Galaxy server.\n" % file_names +msg = "\nThe installation process is finished. All tools associated with this migration that were defined in your file%s named\n" % plural +msg += "%s, have been removed. You may now start your Galaxy server.\n" % file_names print msg app.shutdown() sys.exit( 0 ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.