1 new commit in galaxy-central:

https://bitbucket.org/galaxy/galaxy-central/changeset/d10de4954af2/ changeset: d10de4954af2 user: greg date: 2012-11-07 16:51:35 summary: Since tool migration scripts can be executed any number of times, make sure that no repositories are installed if not tools associated with the migration are defined in the tool_conf.xml file (or equivalent). This fix is associated only with the recently introduced Galaxy admin UI feature displaying the list of migration stages currently available. This is the way the migration process at server startup has always worked so no changes are needed in that scenario. affected #: 4 files

diff -r 6971bbc8bd8418baea51345b891e1aa647d14a88 -r d10de4954af2e94cb7c517ae6398d0247843168e lib/galaxy/tool_shed/common_util.py --- /dev/null +++ b/lib/galaxy/tool_shed/common_util.py @@ -0,0 +1,93 @@ +import os, urllib2 +from galaxy import util +from galaxy.util.odict import odict +from galaxy.tool_shed.encoding_util import * + +REPOSITORY_OWNER = 'devteam' + +def check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_script_number ): + # Get the 000x_tools.xml file associated with the current migrate_tools version number. + tools_xml_file_path = os.path.abspath( os.path.join( 'scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number ) ) + # Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config. + migrated_tool_configs_dict = odict() + tree = util.parse_xml( tools_xml_file_path ) + root = tree.getroot() + tool_shed = root.get( 'name' ) + tool_shed_url = get_tool_shed_url_from_tools_xml_file_path( app, tool_shed ) + # The default behavior is that the tool shed is down. + tool_shed_accessible = False + if tool_shed_url: + for elem in root: + if elem.tag == 'repository': + tool_dependencies = [] + tool_dependencies_dict = {} + repository_name = elem.get( 'name' ) + changeset_revision = elem.get( 'changeset_revision' ) + url = '%s/repository/get_tool_dependencies?name=%s&owner=%s&changeset_revision=%s&from_install_manager=True' % \ + ( tool_shed_url, repository_name, REPOSITORY_OWNER, changeset_revision ) + try: + response = urllib2.urlopen( url ) + text = response.read() + response.close() + tool_shed_accessible = True + except Exception, e: + # Tool shed may be unavailable - we have to set tool_shed_accessible since we're looping. + tool_shed_accessible = False + print "The URL\n%s\nraised the exception:\n%s\n" % ( url, str( e ) ) + if tool_shed_accessible: + if text: + tool_dependencies_dict = tool_shed_decode( text ) + for dependency_key, requirements_dict in tool_dependencies_dict.items(): + tool_dependency_name = requirements_dict[ 'name' ] + tool_dependency_version = requirements_dict[ 'version' ] + tool_dependency_type = requirements_dict[ 'type' ] + tool_dependency_readme = requirements_dict.get( 'readme', '' ) + tool_dependencies.append( ( tool_dependency_name, tool_dependency_version, tool_dependency_type, tool_dependency_readme ) ) + for tool_elem in elem.findall( 'tool' ): + migrated_tool_configs_dict[ tool_elem.get( 'file' ) ] = tool_dependencies + if tool_shed_accessible: + # Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names. + missing_tool_configs_dict = odict() + for tool_panel_config in tool_panel_configs: + tree = util.parse_xml( tool_panel_config ) + root = tree.getroot() + for elem in root: + if elem.tag == 'tool': + missing_tool_configs_dict = check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict ) + elif elem.tag == 'section': + for section_elem in elem: + if section_elem.tag == 'tool': + missing_tool_configs_dict = check_tool_tag_set( section_elem, migrated_tool_configs_dict, missing_tool_configs_dict ) + else: + exception_msg = '\n\nThe entry for the main Galaxy tool shed at %s is missing from the %s file. ' % ( tool_shed, app.config.tool_sheds_config ) + exception_msg += 'The entry for this tool shed must always be available in this file, so re-add it before attempting to start your Galaxy server.\n' + raise Exception( exception_msg ) + return tool_shed_accessible, missing_tool_configs_dict +def check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict ): + file_path = elem.get( 'file', None ) + if file_path: + path, name = os.path.split( file_path ) + if name in migrated_tool_configs_dict: + tool_dependencies = migrated_tool_configs_dict[ name ] + missing_tool_configs_dict[ name ] = tool_dependencies + return missing_tool_configs_dict +def get_non_shed_tool_panel_configs( app ): + # Get the non-shed related tool panel configs - there can be more than one, and the default is tool_conf.xml. + config_filenames = [] + for config_filename in app.config.tool_configs: + # Any config file that includes a tool_path attribute in the root tag set like the following is shed-related. + # <toolbox tool_path="../shed_tools"> + tree = util.parse_xml( config_filename ) + root = tree.getroot() + tool_path = root.get( 'tool_path', None ) + if tool_path is None: + config_filenames.append( config_filename ) + return config_filenames +def get_tool_shed_url_from_tools_xml_file_path( app, tool_shed ): + search_str = '://%s' % tool_shed + for shed_name, shed_url in app.tool_shed_registry.tool_sheds.items(): + if shed_url.find( search_str ) >= 0: + if shed_url.endswith( '/' ): + shed_url = shed_url.rstrip( '/' ) + return shed_url + return None

diff -r 6971bbc8bd8418baea51345b891e1aa647d14a88 -r d10de4954af2e94cb7c517ae6398d0247843168e lib/galaxy/tool_shed/install_manager.py --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -7,8 +7,7 @@ from galaxy.util.json import from_json_string, to_json_string from galaxy.util.shed_util import * from galaxy.util.odict import odict - -REPOSITORY_OWNER = 'devteam' +from galaxy.tool_shed.common_util import *

class InstallManager( object ): def __init__( self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config, install_dependencies ): @@ -19,10 +18,11 @@ self.app = app self.toolbox = self.app.toolbox self.migrated_tools_config = migrated_tools_config - # If install_dependencies is True, but tool_dependency_dir is not set, - # do not attempt to install, but print informative error message + # If install_dependencies is True but tool_dependency_dir is not set, do not attempt to install but print informative error message. if install_dependencies and app.config.tool_dependency_dir is None: - raise Exception( 'You are attempting to install tool dependencies, but do not have a value for "tool_dependency_dir" set in your ini file. Please set this to the path where you would like to install dependencies and rerun the migration script.' ) + message = 'You are attempting to install tool dependencies but do not have a value for "tool_dependency_dir" set in your universe_wsgi.ini ' + message += 'file. Set this location value to the path where you want tool dependencies installed and rerun the migration script.' + raise Exception( message ) # Get the local non-shed related tool panel configs (there can be more than one, and the default name is tool_conf.xml). self.proprietary_tool_confs = self.non_shed_tool_panel_configs self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems( latest_migration_script_number ) @@ -38,8 +38,39 @@ self.tool_shed = clean_tool_shed_url( root.get( 'name' ) ) self.repository_owner = REPOSITORY_OWNER index, self.shed_config_dict = get_shed_tool_conf_dict( app, self.migrated_tools_config ) - for repository_elem in root: - self.install_repository( repository_elem, install_dependencies ) + # Since tool migration scripts can be executed any number of times, we need to make sure the appropriate tools are defined in + # tool_conf.xml. If no tools associated with the migration stage are defined, no repositories will be installed on disk. + # The default behavior is that the tool shed is down. + tool_shed_accessible = False + tool_panel_configs = get_non_shed_tool_panel_configs( app ) + if tool_panel_configs: + # The missing_tool_configs_dict contents are something like: + # {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]} + tool_shed_accessible, missing_tool_configs_dict = check_for_missing_tools( app, tool_panel_configs, latest_migration_script_number ) + else: + # It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but + # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in + # the database. + tool_shed_accessible = True + missing_tool_configs_dict = odict() + if tool_shed_accessible: + if len( self.proprietary_tool_confs ) == 1: + plural = '' + file_names = self.proprietary_tool_confs[ 0 ] + else: + plural = 's' + file_names = ', '.join( self.proprietary_tool_confs ) + if missing_tool_configs_dict: + for repository_elem in root: + self.install_repository( repository_elem, install_dependencies ) + else: + message = "\nNo tools associated with migration stage %s are defined in your " % str( latest_migration_script_number ) + message += "file%s named %s,\nso no repositories will be installed on disk.\n" % ( plural, file_names ) + print message + else: + message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % str( latest_migration_script_number ) + message += "Try again later.\n" + print message def get_guid( self, repository_clone_url, relative_install_dir, tool_config ): if self.shed_config_dict.get( 'tool_path' ): relative_install_dir = os.path.join( self.shed_config_dict['tool_path'], relative_install_dir ) @@ -144,7 +175,8 @@ for k, v in tool_panel_dict_for_tool_config.items(): tool_panel_dict_for_display[ k ] = v else: - print 'The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).' % ( guid, tool_config, ", ".join( self.proprietary_tool_confs or [] ) ) + print 'The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).' \ + % ( guid, tool_config, ", ".join( self.proprietary_tool_confs or [] ) ) metadata_dict, invalid_file_tups = generate_metadata_for_changeset_revision( app=self.app, repository=tool_shed_repository, repository_clone_url=repository_clone_url, @@ -315,20 +347,9 @@ update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.INSTALLED ) @property def non_shed_tool_panel_configs( self ): - # Get the non-shed related tool panel config file names from the Galaxy config - the default is tool_conf.xml. - config_filenames = [] - for config_filename in self.app.config.tool_configs: - # Any config file that includes a tool_path attribute in the root tag set like the following is shed-related. - # <toolbox tool_path="../shed_tools"> - tree = util.parse_xml( config_filename ) - root = tree.getroot() - tool_path = root.get( 'tool_path', None ) - if tool_path is None: - config_filenames.append( config_filename ) - return config_filenames + return get_non_shed_tool_panel_configs( self.app ) def __get_url_from_tool_shed( self, tool_shed ): - # The value of tool_shed is something like: toolshed.g2.bx.psu.edu - # We need the URL to this tool shed, which is something like: + # The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is something like: # http://toolshed.g2.bx.psu.edu/ for shed_name, shed_url in self.app.tool_shed_registry.tool_sheds.items(): if shed_url.find( tool_shed ) >= 0:

diff -r 6971bbc8bd8418baea51345b891e1aa647d14a88 -r d10de4954af2e94cb7c517ae6398d0247843168e lib/galaxy/tool_shed/migrate/common.py --- a/lib/galaxy/tool_shed/migrate/common.py +++ b/lib/galaxy/tool_shed/migrate/common.py @@ -1,103 +1,13 @@ -import sys, os, ConfigParser, urllib2 +import sys, os, ConfigParser import galaxy.config import galaxy.datatypes.registry -from galaxy import util, tools +from galaxy import tools import galaxy.model.mapping import galaxy.tools.search from galaxy.objectstore import build_object_store_from_config +from galaxy.tool_shed.common_util import * import galaxy.tool_shed.tool_shed_registry from galaxy.tool_shed import install_manager -from galaxy.tool_shed.encoding_util import * -from galaxy.util.odict import odict - -REPOSITORY_OWNER = 'devteam' - -def check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_script_number ): - # Get the 000x_tools.xml file associated with the current migrate_tools version number. - tools_xml_file_path = os.path.abspath( os.path.join( 'scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number ) ) - # Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config. - migrated_tool_configs_dict = odict() - tree = util.parse_xml( tools_xml_file_path ) - root = tree.getroot() - tool_shed = root.get( 'name' ) - tool_shed_url = get_tool_shed_url_from_tools_xml_file_path( app, tool_shed ) - # The default behavior is that the tool shed is down. - tool_shed_accessible = False - if tool_shed_url: - for elem in root: - if elem.tag == 'repository': - tool_dependencies = [] - tool_dependencies_dict = {} - repository_name = elem.get( 'name' ) - changeset_revision = elem.get( 'changeset_revision' ) - url = '%s/repository/get_tool_dependencies?name=%s&owner=%s&changeset_revision=%s&from_install_manager=True' % \ - ( tool_shed_url, repository_name, REPOSITORY_OWNER, changeset_revision ) - try: - response = urllib2.urlopen( url ) - text = response.read() - response.close() - tool_shed_accessible = True - except Exception, e: - # Tool shed may be unavailable - we have to set tool_shed_accessible since we're looping. - tool_shed_accessible = False - print "The URL\n%s\nraised the exception:\n%s\n" % ( url, str( e ) ) - if tool_shed_accessible: - if text: - tool_dependencies_dict = tool_shed_decode( text ) - for dependency_key, requirements_dict in tool_dependencies_dict.items(): - tool_dependency_name = requirements_dict[ 'name' ] - tool_dependency_version = requirements_dict[ 'version' ] - tool_dependency_type = requirements_dict[ 'type' ] - tool_dependency_readme = requirements_dict.get( 'readme', '' ) - tool_dependencies.append( ( tool_dependency_name, tool_dependency_version, tool_dependency_type, tool_dependency_readme ) ) - for tool_elem in elem.findall( 'tool' ): - migrated_tool_configs_dict[ tool_elem.get( 'file' ) ] = tool_dependencies - if tool_shed_accessible: - # Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names. - missing_tool_configs_dict = odict() - for tool_panel_config in tool_panel_configs: - tree = util.parse_xml( tool_panel_config ) - root = tree.getroot() - for elem in root: - if elem.tag == 'tool': - missing_tool_configs_dict = check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict ) - elif elem.tag == 'section': - for section_elem in elem: - if section_elem.tag == 'tool': - missing_tool_configs_dict = check_tool_tag_set( section_elem, migrated_tool_configs_dict, missing_tool_configs_dict ) - else: - exception_msg = '\n\nThe entry for the main Galaxy tool shed at %s is missing from the %s file. ' % ( tool_shed, app.config.tool_sheds_config ) - exception_msg += 'The entry for this tool shed must always be available in this file, so re-add it before attempting to start your Galaxy server.\n' - raise Exception( exception_msg ) - return tool_shed_accessible, missing_tool_configs_dict -def check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict ): - file_path = elem.get( 'file', None ) - if file_path: - path, name = os.path.split( file_path ) - if name in migrated_tool_configs_dict: - tool_dependencies = migrated_tool_configs_dict[ name ] - missing_tool_configs_dict[ name ] = tool_dependencies - return missing_tool_configs_dict -def get_non_shed_tool_panel_configs( app ): - # Get the non-shed related tool panel configs - there can be more than one, and the default is tool_conf.xml. - config_filenames = [] - for config_filename in app.config.tool_configs: - # Any config file that includes a tool_path attribute in the root tag set like the following is shed-related. - # <toolbox tool_path="../shed_tools"> - tree = util.parse_xml( config_filename ) - root = tree.getroot() - tool_path = root.get( 'tool_path', None ) - if tool_path is None: - config_filenames.append( config_filename ) - return config_filenames -def get_tool_shed_url_from_tools_xml_file_path( app, tool_shed ): - search_str = '://%s' % tool_shed - for shed_name, shed_url in app.tool_shed_registry.tool_sheds.items(): - if shed_url.find( search_str ) >= 0: - if shed_url.endswith( '/' ): - shed_url = shed_url.rstrip( '/' ) - return shed_url - return None

class MigrateToolsApplication( object ): """Encapsulates the state of a basic Galaxy Universe application in order to initiate the Install Manager"""

diff -r 6971bbc8bd8418baea51345b891e1aa647d14a88 -r d10de4954af2e94cb7c517ae6398d0247843168e scripts/migrate_tools/migrate_tools.py --- a/scripts/migrate_tools/migrate_tools.py +++ b/scripts/migrate_tools/migrate_tools.py @@ -26,8 +26,8 @@ else: plural = 's' file_names = ', '.join( non_shed_tool_confs ) -msg = "\nThe installation process is finished. You should now remove entries for the installed tools from your file%s named\n" % plural -msg += "%s and start your Galaxy server.\n" % file_names +msg = "\nThe installation process is finished. If any tools associated with this migration were defined in your file%s named\n" % plural +msg += "%s, then you should remove entries for them and start your Galaxy server.\n" % file_names print msg app.shutdown() sys.exit( 0 )

Repository URL: https://bitbucket.org/galaxy/galaxy-central/

--

This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.