[galaxy-commits] commit/galaxy-central: greg: Handle unavailable main Galaxy tool shed when checking for migrated tools.

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/cda0af45c3d3/ changeset: cda0af45c3d3 user: greg date: 2012-10-01 21:04:52 summary: Handle unavailable main Galaxy tool shed when checking for migrated tools. affected #: 2 files diff -r 4ccb8691639233bb99b4f3e3ebb9b9eacf4c9461 -r cda0af45c3d3bd4fa9e62b37b5cc91f5272a0178 lib/galaxy/tool_shed/migrate/check.py --- a/lib/galaxy/tool_shed/migrate/check.py +++ b/lib/galaxy/tool_shed/migrate/check.py @@ -43,6 +43,8 @@ db_schema = schema.ControlledSchema( engine, migrate_repository ) latest_tool_migration_script_number = migrate_repository.versions.latest if latest_tool_migration_script_number != db_schema.version: + # The default behavior is that the tool shed is down. + tool_shed_accessible = False if app.new_installation: # New installations will not be missing tools, so we don't need to worry about them. missing_tool_configs_dict = odict() @@ -51,8 +53,12 @@ if tool_panel_configs: # The missing_tool_configs_dict contents are something like: # {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]} - missing_tool_configs_dict = check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_script_number ) + tool_shed_accessible, missing_tool_configs_dict = check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_script_number ) else: + # It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but + # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in + # the database. + tool_shed_accessible = True missing_tool_configs_dict = odict() have_tool_dependencies = False for k, v in missing_tool_configs_dict.items(): @@ -63,79 +69,82 @@ if os.path.abspath( os.path.join( os.getcwd(), 'universe_wsgi.ini' ) ) != galaxy_config_file: config_arg = ' -c %s' % galaxy_config_file.replace( os.path.abspath( os.getcwd() ), '.' ) if not app.config.running_functional_tests: - # Automatically update the value of the migrate_tools.version database table column. - cmd = 'sh manage_tools.sh%s upgrade' % config_arg - proc = subprocess.Popen( args=cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) - return_code = proc.wait() - output = proc.stdout.read( 32768 ) - if return_code != 0: - raise Exception( "Error attempting to update the value of migrate_tools.version: %s" % output ) - elif missing_tool_configs_dict: - if len( tool_panel_configs ) == 1: - plural = '' - tool_panel_config_file_names = tool_panel_configs[ 0 ] - else: - plural = 's' - tool_panel_config_file_names = ', '.join( tool_panel_configs ) - msg = "\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" - msg += "\n\nThe list of files at the end of this message refers to tools that are configured to load into the tool panel for\n" - msg += "this Galaxy instance, but have been removed from the Galaxy distribution. These tools and their dependencies can be\n" - msg += "automatically installed from the Galaxy tool shed at http://toolshed.g2.bx.psu.edu.\n\n" - msg += "To skip this process, attempt to start your Galaxy server again (e.g., sh run.sh or whatever you use). If you do this,\n" - msg += "be aware that these tools will no longer be available in your Galaxy tool panel, and entries for each of them should\n" - msg += "be removed from your file%s named %s.\n\n" % ( plural, tool_panel_config_file_names ) - msg += "CRITICAL NOTE IF YOU PLAN TO INSTALL\n" - msg += "The location in which the tool repositories will be installed is the value of the 'tool_path' attribute in the <tool>\n" - msg += 'tag of the file named ./migrated_tool_conf.xml (i.e., <toolbox tool_path="../shed_tools">). The default location\n' - msg += "setting is '../shed_tools', which may be problematic for some cluster environments, so make sure to change it before\n" - msg += "you execute the installation process if appropriate. The configured location must be outside of the Galaxy installation\n" - msg += "directory or it must be in a sub-directory protected by a properly configured .hgignore file if the directory is within\n" - msg += "the Galaxy installation directory hierarchy. This is because tool shed repositories will be installed using mercurial's\n" - msg += "clone feature, which creates .hg directories and associated mercurial repository files. Not having .hgignore properly\n" - msg += "configured could result in undesired behavior when modifying or updating your local Galaxy instance or the tool shed\n" - msg += "repositories if they are in directories that pose conflicts. See mercurial's .hgignore documentation at the following\n" - msg += "URL for details.\n\nhttp://mercurial.selenic.com/wiki/.hgignore\n\n" - if have_tool_dependencies: - msg += "The following tool dependencies can also optionally be installed (see the option flag in the command below). If you\n" - msg += "choose to install them (recommended), they will be installed within the location specified by the 'tool_dependency_dir'\n" - msg += "setting in your main Galaxy configuration file (e.g., uninverse_wsgi.ini).\n" - processed_tool_dependencies = [] + if tool_shed_accessible: + # Automatically update the value of the migrate_tools.version database table column. + cmd = 'sh manage_tools.sh%s upgrade' % config_arg + proc = subprocess.Popen( args=cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) + return_code = proc.wait() + output = proc.stdout.read( 32768 ) + if return_code != 0: + raise Exception( "Error attempting to update the value of migrate_tools.version: %s" % output ) + elif missing_tool_configs_dict: + if len( tool_panel_configs ) == 1: + plural = '' + tool_panel_config_file_names = tool_panel_configs[ 0 ] + else: + plural = 's' + tool_panel_config_file_names = ', '.join( tool_panel_configs ) + msg = "\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + msg += "\n\nThe list of files at the end of this message refers to tools that are configured to load into the tool panel for\n" + msg += "this Galaxy instance, but have been removed from the Galaxy distribution. These tools and their dependencies can be\n" + msg += "automatically installed from the Galaxy tool shed at http://toolshed.g2.bx.psu.edu.\n\n" + msg += "To skip this process, attempt to start your Galaxy server again (e.g., sh run.sh or whatever you use). If you do this,\n" + msg += "be aware that these tools will no longer be available in your Galaxy tool panel, and entries for each of them should\n" + msg += "be removed from your file%s named %s.\n\n" % ( plural, tool_panel_config_file_names ) + msg += "CRITICAL NOTE IF YOU PLAN TO INSTALL\n" + msg += "The location in which the tool repositories will be installed is the value of the 'tool_path' attribute in the <tool>\n" + msg += 'tag of the file named ./migrated_tool_conf.xml (i.e., <toolbox tool_path="../shed_tools">). The default location\n' + msg += "setting is '../shed_tools', which may be problematic for some cluster environments, so make sure to change it before\n" + msg += "you execute the installation process if appropriate. The configured location must be outside of the Galaxy installation\n" + msg += "directory or it must be in a sub-directory protected by a properly configured .hgignore file if the directory is within\n" + msg += "the Galaxy installation directory hierarchy. This is because tool shed repositories will be installed using mercurial's\n" + msg += "clone feature, which creates .hg directories and associated mercurial repository files. Not having .hgignore properly\n" + msg += "configured could result in undesired behavior when modifying or updating your local Galaxy instance or the tool shed\n" + msg += "repositories if they are in directories that pose conflicts. See mercurial's .hgignore documentation at the following\n" + msg += "URL for details.\n\nhttp://mercurial.selenic.com/wiki/.hgignore\n\n" + if have_tool_dependencies: + msg += "The following tool dependencies can also optionally be installed (see the option flag in the command below). If you\n" + msg += "choose to install them (recommended), they will be installed within the location specified by the 'tool_dependency_dir'\n" + msg += "setting in your main Galaxy configuration file (e.g., uninverse_wsgi.ini).\n" + processed_tool_dependencies = [] + for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items(): + for tool_dependencies_tup in tool_dependencies: + if tool_dependencies_tup not in processed_tool_dependencies: + msg += "------------------------------------\n" + msg += "Tool Dependency\n" + msg += "------------------------------------\n" + msg += "Name: %s, Version: %s, Type: %s\n" % ( tool_dependencies_tup[ 0 ], + tool_dependencies_tup[ 1 ], + tool_dependencies_tup[ 2 ] ) + if tool_dependencies_tup[ 3 ]: + msg += "Requirements and installation information:\n" + msg += "%s\n" % tool_dependencies_tup[ 3 ] + else: + msg += "\n" + msg += "------------------------------------\n" + processed_tool_dependencies.append( tool_dependencies_tup ) + msg += "\n" + msg += "%s" % output.replace( 'done', '' ) + msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" + msg += "sh ./scripts/migrate_tools/%04d_tools.sh\n" % latest_tool_migration_script_number + msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n" + if have_tool_dependencies: + msg += "The tool dependencies listed above will be installed along with the repositories if you add the 'install_dependencies'\n" + msg += "option to the above command like this:\n\n" + msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" + msg += "sh ./scripts/migrate_tools/%04d_tools.sh install_dependencies\n" % latest_tool_migration_script_number + msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n" + msg += "Tool dependencies can be installed after the repositories have been installed as well.\n\n" + msg += "After the installation process finishes, you can start your Galaxy server. As part of this installation process,\n" + msg += "entries for each of the following tool config files will be added to the file named ./migrated_tool_conf.xml, so these\n" + msg += "tools will continue to be loaded into your tool panel. Because of this, existing entries for these files should be\n" + msg += "removed from your file%s named %s, but only after the installation process finishes.\n\n" % ( plural, tool_panel_config_file_names ) for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items(): - for tool_dependencies_tup in tool_dependencies: - if tool_dependencies_tup not in processed_tool_dependencies: - msg += "------------------------------------\n" - msg += "Tool Dependency\n" - msg += "------------------------------------\n" - msg += "Name: %s, Version: %s, Type: %s\n" % ( tool_dependencies_tup[ 0 ], - tool_dependencies_tup[ 1 ], - tool_dependencies_tup[ 2 ] ) - if tool_dependencies_tup[ 3 ]: - msg += "Requirements and installation information:\n" - msg += "%s\n" % tool_dependencies_tup[ 3 ] - else: - msg += "\n" - msg += "------------------------------------\n" - processed_tool_dependencies.append( tool_dependencies_tup ) - msg += "\n" - msg += "%s" % output.replace( 'done', '' ) - msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" - msg += "sh ./scripts/migrate_tools/%04d_tools.sh\n" % latest_tool_migration_script_number - msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n" - if have_tool_dependencies: - msg += "The tool dependencies listed above will be installed along with the repositories if you add the 'install_dependencies'\n" - msg += "option to the above command like this:\n\n" - msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" - msg += "sh ./scripts/migrate_tools/%04d_tools.sh install_dependencies\n" % latest_tool_migration_script_number - msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n" - msg += "Tool dependencies can be installed after the repositories have been installed as well.\n\n" - msg += "After the installation process finishes, you can start your Galaxy server. As part of this installation process,\n" - msg += "entries for each of the following tool config files will be added to the file named ./migrated_tool_conf.xml, so these\n" - msg += "tools will continue to be loaded into your tool panel. Because of this, existing entries for these files should be\n" - msg += "removed from your file%s named %s, but only after the installation process finishes.\n\n" % ( plural, tool_panel_config_file_names ) - for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items(): - msg += "%s\n" % missing_tool_config - msg += "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n" - raise Exception( msg ) + msg += "%s\n" % missing_tool_config + msg += "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n" + raise Exception( msg ) + else: + log.debug( "The main Galaxy tool shed is not currently available, so skipped tool migration %s until next server startup" % db_schema.version ) else: log.info( "At migrate_tools version %d" % db_schema.version ) diff -r 4ccb8691639233bb99b4f3e3ebb9b9eacf4c9461 -r cda0af45c3d3bd4fa9e62b37b5cc91f5272a0178 lib/galaxy/tool_shed/migrate/common.py --- a/lib/galaxy/tool_shed/migrate/common.py +++ b/lib/galaxy/tool_shed/migrate/common.py @@ -21,6 +21,8 @@ root = tree.getroot() tool_shed = root.get( 'name' ) tool_shed_url = get_tool_shed_url_from_tools_xml_file_path( app, tool_shed ) + # The default behavior is that the tool shed is down. + tool_shed_accessible = False if tool_shed_url: for elem in root: if elem.tag == 'repository': @@ -30,36 +32,44 @@ changeset_revision = elem.get( 'changeset_revision' ) url = '%s/repository/get_tool_dependencies?name=%s&owner=%s&changeset_revision=%s&webapp=install_manager' % \ ( tool_shed_url, repository_name, REPOSITORY_OWNER, changeset_revision ) - response = urllib2.urlopen( url ) - text = response.read() - response.close() - if text: - tool_dependencies_dict = tool_shed_decode( text ) - for dependency_key, requirements_dict in tool_dependencies_dict.items(): - tool_dependency_name = requirements_dict[ 'name' ] - tool_dependency_version = requirements_dict[ 'version' ] - tool_dependency_type = requirements_dict[ 'type' ] - tool_dependency_readme = requirements_dict.get( 'readme', '' ) - tool_dependencies.append( ( tool_dependency_name, tool_dependency_version, tool_dependency_type, tool_dependency_readme ) ) - for tool_elem in elem.findall( 'tool' ): - migrated_tool_configs_dict[ tool_elem.get( 'file' ) ] = tool_dependencies - # Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names. - missing_tool_configs_dict = odict() - for tool_panel_config in tool_panel_configs: - tree = util.parse_xml( tool_panel_config ) - root = tree.getroot() - for elem in root: - if elem.tag == 'tool': - missing_tool_configs_dict = check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict ) - elif elem.tag == 'section': - for section_elem in elem: - if section_elem.tag == 'tool': - missing_tool_configs_dict = check_tool_tag_set( section_elem, migrated_tool_configs_dict, missing_tool_configs_dict ) + try: + response = urllib2.urlopen( url ) + text = response.read() + response.close() + tool_shed_accessible = True + except Exception, e: + # Tool shed may be unavailable - we have to set tool_shed_accessible since we're looping. + tool_shed_accessible = False + print "The URL\n%s\nraised the exception:\n%s\n" % ( url, str( e ) ) + if tool_shed_accessible: + if text: + tool_dependencies_dict = tool_shed_decode( text ) + for dependency_key, requirements_dict in tool_dependencies_dict.items(): + tool_dependency_name = requirements_dict[ 'name' ] + tool_dependency_version = requirements_dict[ 'version' ] + tool_dependency_type = requirements_dict[ 'type' ] + tool_dependency_readme = requirements_dict.get( 'readme', '' ) + tool_dependencies.append( ( tool_dependency_name, tool_dependency_version, tool_dependency_type, tool_dependency_readme ) ) + for tool_elem in elem.findall( 'tool' ): + migrated_tool_configs_dict[ tool_elem.get( 'file' ) ] = tool_dependencies + if tool_shed_accessible: + # Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names. + missing_tool_configs_dict = odict() + for tool_panel_config in tool_panel_configs: + tree = util.parse_xml( tool_panel_config ) + root = tree.getroot() + for elem in root: + if elem.tag == 'tool': + missing_tool_configs_dict = check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict ) + elif elem.tag == 'section': + for section_elem in elem: + if section_elem.tag == 'tool': + missing_tool_configs_dict = check_tool_tag_set( section_elem, migrated_tool_configs_dict, missing_tool_configs_dict ) else: exception_msg = '\n\nThe entry for the main Galaxy tool shed at %s is missing from the %s file. ' % ( tool_shed, app.config.tool_sheds_config ) exception_msg += 'The entry for this tool shed must always be available in this file, so re-add it before attempting to start your Galaxy server.\n' raise Exception( exception_msg ) - return missing_tool_configs_dict + return tool_shed_accessible, missing_tool_configs_dict def check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict ): file_path = elem.get( 'file', None ) if file_path: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.