1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6294a65d863a/ Changeset: 6294a65d863a User: greg Date: 2014-06-17 20:38:02 Summary: Eliminate the use of the repository_util.py module for Galaxy installs from the Tool Shed. Affected #: 16 files diff -r 95937a7e51f20da0ebf0853f9385286d2c70c4eb -r 6294a65d863a4875a9e096ca5e81c3f30e8b0bb6 lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -19,6 +19,7 @@ from galaxy.jobs import metrics as job_metrics from galaxy.web.base import pluginframework from galaxy.queue_worker import GalaxyQueueWorker +from tool_shed.galaxy_install import update_repository_manager import logging log = logging.getLogger( __name__ ) @@ -80,12 +81,8 @@ # Load Data Manager self.data_managers = DataManagers( self ) - # If enabled, poll respective tool sheds to see if updates are available for any installed tool shed repositories. - if self.config.get_bool( 'enable_tool_shed_check', False ): - from tool_shed.galaxy_install import update_repository_manager - self.update_repository_manager = update_repository_manager.UpdateRepositoryManager( self ) - else: - self.update_repository_manager = None + # Load the update repository manager. + self.update_repository_manager = update_repository_manager.UpdateRepositoryManager( self ) # Load proprietary datatype converters and display applications. self.installed_repository_manager.load_proprietary_converters_and_display_applications() # Load datatype display applications defined in local datatypes_conf.xml @@ -159,8 +156,7 @@ self.object_store.shutdown() if self.heartbeat: self.heartbeat.shutdown() - if self.update_repository_manager is not None: - self.update_repository_manager.shutdown() + self.update_repository_manager.shutdown() if self.control_worker: self.control_worker.shutdown() try: diff -r 95937a7e51f20da0ebf0853f9385286d2c70c4eb -r 6294a65d863a4875a9e096ca5e81c3f30e8b0bb6 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -105,7 +105,6 @@ self.shed_tool_data_path = self.tool_data_path self.tool_data_table_config_path = [ resolve_path( x, self.root ) for x in kwargs.get( 'tool_data_table_config_path', 'tool_data_table_conf.xml' ).split( ',' ) ] self.shed_tool_data_table_config = resolve_path( kwargs.get( 'shed_tool_data_table_config', 'shed_tool_data_table_conf.xml' ), self.root ) - self.enable_tool_shed_check = string_as_bool( kwargs.get( 'enable_tool_shed_check', False ) ) self.manage_dependency_relationships = string_as_bool( kwargs.get( 'manage_dependency_relationships', False ) ) self.running_functional_tests = string_as_bool( kwargs.get( 'running_functional_tests', False ) ) self.hours_between_check = kwargs.get( 'hours_between_check', 12 ) diff -r 95937a7e51f20da0ebf0853f9385286d2c70c4eb -r 6294a65d863a4875a9e096ca5e81c3f30e8b0bb6 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -24,11 +24,11 @@ from tool_shed.util import metadata_util from tool_shed.util import readme_util from tool_shed.util import repository_dependency_util +from tool_shed.util import repository_maintenance_util from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util from tool_shed.util import workflow_util from tool_shed.util import xml_util -from tool_shed.galaxy_install import repository_util from tool_shed.galaxy_install import install_manager from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager import tool_shed.galaxy_install.grids.admin_toolshed_grids as admin_toolshed_grids @@ -114,7 +114,8 @@ # repository dependencies and tool dependencies will be current. Only allow selecting a different section # in the tool panel if the repository was uninstalled and it contained tools that should be displayed in # the tool panel. - changeset_revision_dict = repository_util.get_update_to_changeset_revision_and_ctx_rev( trans.app, repository ) + changeset_revision_dict = \ + trans.app.update_repository_manager.get_update_to_changeset_revision_and_ctx_rev( repository ) current_changeset_revision = changeset_revision_dict.get( 'changeset_revision', None ) current_ctx_rev = changeset_revision_dict.get( 'ctx_rev', None ) if current_changeset_revision and current_ctx_rev: @@ -588,11 +589,10 @@ updated_metadata = encoding_util.tool_shed_decode( encoded_updated_metadata ) else: updated_metadata = None - repository = repository_util.update_repository_record( trans.app, - repository=repository, - updated_metadata_dict=updated_metadata, - updated_changeset_revision=updating_to_changeset_revision, - updated_ctx_rev=updating_to_ctx_rev ) + repository = trans.app.update_repository_manager.update_repository_record( repository=repository, + updated_metadata_dict=updated_metadata, + updated_changeset_revision=updating_to_changeset_revision, + updated_ctx_rev=updating_to_ctx_rev ) if install_tool_dependencies: tool_dependencies = tool_dependency_util.create_tool_dependency_objects( trans.app, repository, @@ -657,7 +657,7 @@ def manage_repositories( self, trans, **kwd ): message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) - tsridslist = repository_util.get_tool_shed_repository_ids( **kwd ) + tsridslist = common_util.get_tool_shed_repository_ids( **kwd ) if 'operation' in kwd: operation = kwd[ 'operation' ].lower() if not tsridslist: @@ -941,7 +941,7 @@ @web.expose @web.require_admin def monitor_repository_installation( self, trans, **kwd ): - tsridslist = repository_util.get_tool_shed_repository_ids( **kwd ) + tsridslist = common_util.get_tool_shed_repository_ids( **kwd ) if not tsridslist: tsridslist = suc.get_ids_of_tool_shed_repositories_being_installed( trans.app, as_string=False ) kwd[ 'tool_shed_repository_ids' ] = tsridslist @@ -1062,11 +1062,10 @@ decoded_updated_metadata = encoding_util.tool_shed_decode( encoded_updated_metadata ) # Now that the user has decided whether they will handle dependencies, we can update # the repository to the latest revision. - repository = repository_util.update_repository_record( trans.app, - repository=repository, - updated_metadata_dict=decoded_updated_metadata, - updated_changeset_revision=updating_to_changeset_revision, - updated_ctx_rev=updating_to_ctx_rev ) + repository = trans.app.update_repository_manager.update_repository_record( repository=repository, + updated_metadata_dict=decoded_updated_metadata, + updated_changeset_revision=updating_to_changeset_revision, + updated_ctx_rev=updating_to_ctx_rev ) install_repository_dependencies = CheckboxField.is_checked( install_repository_dependencies ) if includes_tool_dependencies: install_tool_dependencies = CheckboxField.is_checked( install_tool_dependencies ) @@ -1081,7 +1080,7 @@ tool_path=tool_path, tool_shed_url=tool_shed_url ) created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts = \ - repository_util.handle_tool_shed_repositories( trans.app, installation_dict, using_api=False ) + irm.handle_tool_shed_repositories( installation_dict, using_api=False ) if created_or_updated_tool_shed_repositories: installation_dict = dict( created_or_updated_tool_shed_repositories=created_or_updated_tool_shed_repositories, filtered_repo_info_dicts=filtered_repo_info_dicts, @@ -1367,16 +1366,17 @@ tool_dependencies = metadata.get( 'tool_dependencies', None ) else: tool_dependencies = None - repo_info_dict = repository_util.create_repo_info_dict( app=trans.app, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.changeset_revision, - ctx_rev=ctx_rev, - repository_owner=tool_shed_repository.owner, - repository_name=tool_shed_repository.name, - repository=None, - repository_metadata=None, - tool_dependencies=tool_dependencies, - repository_dependencies=repository_dependencies ) + repo_info_dict = \ + repository_maintenance_util.create_repo_info_dict( app=trans.app, + repository_clone_url=repository_clone_url, + changeset_revision=tool_shed_repository.changeset_revision, + ctx_rev=ctx_rev, + repository_owner=tool_shed_repository.owner, + repository_name=tool_shed_repository.name, + repository=None, + repository_metadata=None, + tool_dependencies=tool_dependencies, + repository_dependencies=repository_dependencies ) if repo_info_dict not in repo_info_dicts: repo_info_dicts.append( repo_info_dict ) # Make sure all tool_shed_repository records exist. @@ -1581,16 +1581,17 @@ repository_dependencies = \ repository_dependency_util.get_repository_dependencies_for_installed_tool_shed_repository( trans.app, tool_shed_repository ) - repo_info_dict = repository_util.create_repo_info_dict( app=trans.app, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.installed_changeset_revision, - ctx_rev=tool_shed_repository.ctx_rev, - repository_owner=tool_shed_repository.owner, - repository_name=tool_shed_repository.name, - repository=None, - repository_metadata=None, - tool_dependencies=tool_dependencies, - repository_dependencies=repository_dependencies ) + repo_info_dict = \ + repository_maintenance_util.create_repo_info_dict( app=trans.app, + repository_clone_url=repository_clone_url, + changeset_revision=tool_shed_repository.installed_changeset_revision, + ctx_rev=tool_shed_repository.ctx_rev, + repository_owner=tool_shed_repository.owner, + repository_name=tool_shed_repository.name, + repository=None, + repository_metadata=None, + tool_dependencies=tool_dependencies, + repository_dependencies=repository_dependencies ) dependencies_for_repository_dict = common_install_util.get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, @@ -2024,11 +2025,10 @@ return self.install_tool_dependencies_with_update( trans, **new_kwd ) # Updates received did not include any newly defined repository dependencies or newly defined # tool dependencies that need to be installed. - repository = repository_util.update_repository_record( trans.app, - repository=repository, - updated_metadata_dict=metadata_dict, - updated_changeset_revision=latest_changeset_revision, - updated_ctx_rev=latest_ctx_rev ) + repository = trans.app.update_repository_manager.update_repository_record( repository=repository, + updated_metadata_dict=metadata_dict, + updated_changeset_revision=latest_changeset_revision, + updated_ctx_rev=latest_ctx_rev ) message = "The installed repository named '%s' has been updated to change set revision '%s'. " % \ ( name, latest_changeset_revision ) else: diff -r 95937a7e51f20da0ebf0853f9385286d2c70c4eb -r 6294a65d863a4875a9e096ca5e81c3f30e8b0bb6 lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -1,3 +1,4 @@ +import json import logging import os import tarfile @@ -6,13 +7,11 @@ from galaxy import eggs from galaxy import util from galaxy import web -from galaxy.util import json from galaxy.web.base.controller import BaseAPIController from galaxy.web.base.controller import HTTPBadRequest from galaxy.web.framework.helpers import time_ago import tool_shed.repository_types.util as rt_util import tool_shed.util.shed_util_common as suc -from tool_shed.galaxy_install import repository_util from tool_shed.util import basic_util from tool_shed.util import encoding_util from tool_shed.util import hg_util @@ -192,7 +191,10 @@ includes_tools_for_display_in_tool_panel, \ has_repository_dependencies, \ has_repository_dependencies_only_if_compiling_contained_td = \ - repository_util.get_repo_info_dict( trans, encoded_repository_id, changeset_revision ) + repository_maintenance_util.get_repo_info_dict( trans.app, + trans.user, + encoded_repository_id, + changeset_revision ) return repository_dict, repository_metadata_dict, repo_info_dict else: log.debug( "Unable to locate repository_metadata record for repository id %s and changeset_revision %s" % \ @@ -465,7 +467,7 @@ results = handle_repository( trans, repository, results ) stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) results[ 'stop_time' ] = stop_time - return json.to_json_string( results, sort_keys=True, indent=4 ) + return json.dumps( results, sort_keys=True, indent=4 ) @web.expose_api def reset_metadata_on_repository( self, trans, payload, **kwd ): @@ -504,7 +506,7 @@ results = handle_repository( trans, start_time, repository ) stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) results[ 'stop_time' ] = stop_time - return json.to_json_string( results, sort_keys=True, indent=4 ) + return json.dumps( results, sort_keys=True, indent=4 ) @web.expose_api_anonymous def show( self, trans, id, **kwd ): diff -r 95937a7e51f20da0ebf0853f9385286d2c70c4eb -r 6294a65d863a4875a9e096ca5e81c3f30e8b0bb6 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -31,7 +31,7 @@ from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util from tool_shed.util import workflow_util -from tool_shed.galaxy_install import repository_util +from tool_shed.galaxy_install import install_manager from galaxy.webapps.tool_shed.util import ratings_util import galaxy.tools import tool_shed.grids.repository_grids as repository_grids @@ -1761,7 +1761,7 @@ repository_id, changeset_revision = tup repo_info_dict, cur_includes_tools, cur_includes_tool_dependencies, cur_includes_tools_for_display_in_tool_panel, \ cur_has_repository_dependencies, cur_has_repository_dependencies_only_if_compiling_contained_td = \ - repository_util.get_repo_info_dict( trans, repository_id, changeset_revision ) + repository_maintenance_util.get_repo_info_dict( trans.app, trans.user, repository_id, changeset_revision ) if cur_has_repository_dependencies and not has_repository_dependencies: has_repository_dependencies = True if cur_has_repository_dependencies_only_if_compiling_contained_td and not has_repository_dependencies_only_if_compiling_contained_td: @@ -1881,7 +1881,7 @@ """ repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) repository_id = trans.security.encode_id( repository.id ) - repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans, repository ) + repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans.user, repository ) repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if not repository_metadata: @@ -1892,16 +1892,16 @@ after_changeset_revision=changeset_revision ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) - repo_info_dict = repository_util.create_repo_info_dict( app=trans.app, - repository_clone_url=repository_clone_url, - changeset_revision=changeset_revision, - ctx_rev=str( ctx.rev() ), - repository_owner=repository.user.username, - repository_name=repository.name, - repository=repository, - repository_metadata=repository_metadata, - tool_dependencies=None, - repository_dependencies=None ) + repo_info_dict = repository_maintenance_util.create_repo_info_dict( app=trans.app, + repository_clone_url=repository_clone_url, + changeset_revision=changeset_revision, + ctx_rev=str( ctx.rev() ), + repository_owner=repository.user.username, + repository_name=repository.name, + repository=repository, + repository_metadata=repository_metadata, + tool_dependencies=None, + repository_dependencies=None ) includes_data_managers = False includes_datatypes = False includes_tools = False diff -r 95937a7e51f20da0ebf0853f9385286d2c70c4eb -r 6294a65d863a4875a9e096ca5e81c3f30e8b0bb6 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -20,14 +20,17 @@ from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import container_util +from tool_shed.util import data_manager_util +from tool_shed.util import datatype_util from tool_shed.util import encoding_util from tool_shed.util import hg_util +from tool_shed.util import metadata_util +from tool_shed.util import repository_dependency_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util from tool_shed.util import xml_util -from tool_shed.galaxy_install import repository_util from tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import StepManager @@ -488,6 +491,136 @@ repo_info_dicts = [ repo_info_dict ] return repository_revision_dict, repo_info_dicts + def handle_repository_contents( self, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir, + tool_shed=None, tool_section=None, shed_tool_conf=None, reinstalling=False ): + """ + Generate the metadata for the installed tool shed repository, among other things. This method is called from Galaxy + (never the tool shed) when an administrator is installing a new repository or reinstalling an uninstalled repository. + """ + install_model = self.app.install_model + shed_config_dict = self.app.toolbox.get_shed_config_dict_by_filename( shed_tool_conf ) + metadata_dict, invalid_file_tups = \ + metadata_util.generate_metadata_for_changeset_revision( app=self.app, + repository=tool_shed_repository, + changeset_revision=tool_shed_repository.changeset_revision, + repository_clone_url=repository_clone_url, + shed_config_dict=shed_config_dict, + relative_install_dir=relative_install_dir, + repository_files_dir=None, + resetting_all_metadata_on_repository=False, + updating_installed_repository=False, + persist=True ) + tool_shed_repository.metadata = metadata_dict + # Update the tool_shed_repository.tool_shed_status column in the database. + tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( self.app, tool_shed_repository ) + if tool_shed_status_dict: + tool_shed_repository.tool_shed_status = tool_shed_status_dict + install_model.context.add( tool_shed_repository ) + install_model.context.flush() + if 'tool_dependencies' in metadata_dict and not reinstalling: + tool_dependencies = tool_dependency_util.create_tool_dependency_objects( self.app, + tool_shed_repository, + relative_install_dir, + set_status=True ) + if 'sample_files' in metadata_dict: + sample_files = metadata_dict.get( 'sample_files', [] ) + tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files ) + tool_data_table_conf_filename, tool_data_table_elems = \ + tool_util.install_tool_data_tables( self.app, tool_shed_repository, tool_index_sample_files ) + if tool_data_table_elems: + self.app.tool_data_tables.add_new_entries_from_config_file( tool_data_table_conf_filename, + None, + self.app.config.shed_tool_data_table_config, + persist=True ) + if 'tools' in metadata_dict: + tool_panel_dict = tool_util.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) + sample_files = metadata_dict.get( 'sample_files', [] ) + tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files ) + tool_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=tool_path ) + sample_files_copied = [ str( s ) for s in tool_index_sample_files ] + repository_tools_tups = suc.get_repository_tools_tups( self.app, metadata_dict ) + if repository_tools_tups: + # Handle missing data table entries for tool parameters that are dynamically generated select lists. + repository_tools_tups = tool_util.handle_missing_data_table_entry( self.app, + relative_install_dir, + tool_path, + repository_tools_tups ) + # Handle missing index files for tool parameters that are dynamically generated select lists. + repository_tools_tups, sample_files_copied = tool_util.handle_missing_index_file( self.app, + tool_path, + sample_files, + repository_tools_tups, + sample_files_copied ) + # Copy remaining sample files included in the repository to the ~/tool-data directory of the + # local Galaxy instance. + tool_util.copy_sample_files( self.app, sample_files, tool_path=tool_path, sample_files_copied=sample_files_copied ) + tool_util.add_to_tool_panel( app=self.app, + repository_name=tool_shed_repository.name, + repository_clone_url=repository_clone_url, + changeset_revision=tool_shed_repository.installed_changeset_revision, + repository_tools_tups=repository_tools_tups, + owner=tool_shed_repository.owner, + shed_tool_conf=shed_tool_conf, + tool_panel_dict=tool_panel_dict, + new_install=True ) + if 'data_manager' in metadata_dict: + new_data_managers = data_manager_util.install_data_managers( self.app, + self.app.config.shed_data_manager_config_file, + metadata_dict, + shed_config_dict, + relative_install_dir, + tool_shed_repository, + repository_tools_tups ) + if 'datatypes' in metadata_dict: + tool_shed_repository.status = install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES + if not tool_shed_repository.includes_datatypes: + tool_shed_repository.includes_datatypes = True + install_model.context.add( tool_shed_repository ) + install_model.context.flush() + files_dir = relative_install_dir + if shed_config_dict.get( 'tool_path' ): + files_dir = os.path.join( shed_config_dict[ 'tool_path' ], files_dir ) + datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, files_dir ) + # Load data types required by tools. + converter_path, display_path = \ + datatype_util.alter_config_and_load_prorietary_datatypes( self.app, datatypes_config, files_dir, override=False ) + if converter_path or display_path: + # Create a dictionary of tool shed repository related information. + repository_dict = \ + datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, + name=tool_shed_repository.name, + owner=tool_shed_repository.owner, + installed_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dicts=metadata_dict.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) + if converter_path: + # Load proprietary datatype converters + self.app.datatypes_registry.load_datatype_converters( self.app.toolbox, installed_repository_dict=repository_dict ) + if display_path: + # Load proprietary datatype display applications + self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) + + def handle_tool_shed_repositories( self, installation_dict, using_api=False ): + # The following installation_dict entries are all required. + install_repository_dependencies = installation_dict[ 'install_repository_dependencies' ] + new_tool_panel_section_label = installation_dict[ 'new_tool_panel_section_label' ] + no_changes_checked = installation_dict[ 'no_changes_checked' ] + repo_info_dicts = installation_dict[ 'repo_info_dicts' ] + tool_panel_section_id = installation_dict[ 'tool_panel_section_id' ] + tool_path = installation_dict[ 'tool_path' ] + tool_shed_url = installation_dict[ 'tool_shed_url' ] + created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts = \ + repository_dependency_util.create_repository_dependency_objects( app=self.app, + tool_path=tool_path, + tool_shed_url=tool_shed_url, + repo_info_dicts=repo_info_dicts, + install_repository_dependencies=install_repository_dependencies, + no_changes_checked=no_changes_checked, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) + return created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts + def initiate_repository_installation( self, installation_dict ): install_model = self.app.install_model # The following installation_dict entries are all required. @@ -699,9 +832,9 @@ cloned_ok, error_message = hg_util.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev ) if cloned_ok: if reinstalling: - # Since we're reinstalling the repository we need to find the latest changeset revision to which it can be updated. - changeset_revision_dict = repository_util.get_update_to_changeset_revision_and_ctx_rev( self.app, - tool_shed_repository ) + # Since we're reinstalling the repository we need to find the latest changeset revision to + # which it can be updated. + changeset_revision_dict = self.app.update_repository_manager.get_update_to_changeset_revision_and_ctx_rev( tool_shed_repository ) current_changeset_revision = changeset_revision_dict.get( 'changeset_revision', None ) current_ctx_rev = changeset_revision_dict.get( 'ctx_rev', None ) if current_ctx_rev != ctx_rev: @@ -711,15 +844,14 @@ create=False ) hg_util.pull_repository( repo, repository_clone_url, current_changeset_revision ) hg_util.update_repository( repo, ctx_rev=current_ctx_rev ) - repository_util.handle_repository_contents( self.app, - tool_shed_repository=tool_shed_repository, - tool_path=tool_path, - repository_clone_url=repository_clone_url, - relative_install_dir=relative_install_dir, - tool_shed=tool_shed_repository.tool_shed, - tool_section=tool_section, - shed_tool_conf=shed_tool_conf, - reinstalling=reinstalling ) + self.handle_repository_contents( tool_shed_repository=tool_shed_repository, + tool_path=tool_path, + repository_clone_url=repository_clone_url, + relative_install_dir=relative_install_dir, + tool_shed=tool_shed_repository.tool_shed, + tool_section=tool_section, + shed_tool_conf=shed_tool_conf, + reinstalling=reinstalling ) install_model.context.refresh( tool_shed_repository ) metadata = tool_shed_repository.metadata if 'tools' in metadata: diff -r 95937a7e51f20da0ebf0853f9385286d2c70c4eb -r 6294a65d863a4875a9e096ca5e81c3f30e8b0bb6 lib/tool_shed/galaxy_install/repair_repository_manager.py --- a/lib/tool_shed/galaxy_install/repair_repository_manager.py +++ b/lib/tool_shed/galaxy_install/repair_repository_manager.py @@ -4,13 +4,13 @@ log = logging.getLogger( __name__ ) from tool_shed.galaxy_install import install_manager -from tool_shed.galaxy_install import repository_util from tool_shed.util import common_install_util from tool_shed.util import common_util from tool_shed.util import container_util from tool_shed.util import shed_util_common as suc from tool_shed.util import repository_dependency_util +from tool_shed.util import repository_maintenance_util from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util @@ -123,16 +123,16 @@ new_tool_panel_section_label=tool_panel_section_name ) else: tool_dependencies = None - repo_info_dict = repository_util.create_repo_info_dict( app=self.app, - repository_clone_url=repository_clone_url, - changeset_revision=repository.changeset_revision, - ctx_rev=repository.ctx_rev, - repository_owner=repository.owner, - repository_name=repository.name, - repository=None, - repository_metadata=None, - tool_dependencies=tool_dependencies, - repository_dependencies=repository_dependencies ) + repo_info_dict = repository_maintenance_util.create_repo_info_dict( app=self.app, + repository_clone_url=repository_clone_url, + changeset_revision=repository.changeset_revision, + ctx_rev=repository.ctx_rev, + repository_owner=repository.owner, + repository_name=repository.name, + repository=None, + repository_metadata=None, + tool_dependencies=tool_dependencies, + repository_dependencies=repository_dependencies ) return repo_info_dict, tool_panel_section_key def repair_tool_shed_repository( self, repository, repo_info_dict ): diff -r 95937a7e51f20da0ebf0853f9385286d2c70c4eb -r 6294a65d863a4875a9e096ca5e81c3f30e8b0bb6 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ /dev/null @@ -1,382 +0,0 @@ -import logging -import os -from galaxy import util -from galaxy import web -import tool_shed.util.shed_util_common as suc -from tool_shed.util import common_util -from tool_shed.util import data_manager_util -from tool_shed.util import datatype_util -from tool_shed.util import encoding_util -from tool_shed.util import hg_util -from tool_shed.util import repository_dependency_util -from tool_shed.util import metadata_util -from tool_shed.util import tool_dependency_util -from tool_shed.util import tool_util - -log = logging.getLogger( __name__ ) - -def create_repo_info_dict( app, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_name=None, - repository=None, repository_metadata=None, tool_dependencies=None, repository_dependencies=None ): - """ - Return a dictionary that includes all of the information needed to install a repository into a local - Galaxy instance. The dictionary will also contain the recursive list of repository dependencies defined - for the repository, as well as the defined tool dependencies. - - This method is called from Galaxy under four scenarios: - 1. During the tool shed repository installation process via the tool shed's get_repository_information() - method. In this case both the received repository and repository_metadata will be objects, but - tool_dependencies and repository_dependencies will be None. - 2. When getting updates for an install repository where the updates include newly defined repository - dependency definitions. This scenario is similar to 1. above. The tool shed's get_repository_information() - method is the caller, and both the received repository and repository_metadata will be objects, but - tool_dependencies and repository_dependencies will be None. - 3. When a tool shed repository that was uninstalled from a Galaxy instance is being reinstalled with no - updates available. In this case, both repository and repository_metadata will be None, but tool_dependencies - and repository_dependencies will be objects previously retrieved from the tool shed if the repository includes - definitions for them. - 4. When a tool shed repository that was uninstalled from a Galaxy instance is being reinstalled with updates - available. In this case, this method is reached via the tool shed's get_updated_repository_information() - method, and both repository and repository_metadata will be objects but tool_dependencies and - repository_dependencies will be None. - """ - repo_info_dict = {} - repository = suc.get_repository_by_name_and_owner( app, repository_name, repository_owner ) - if app.name == 'tool_shed': - # We're in the tool shed. - repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, - app.security.encode_id( repository.id ), - changeset_revision ) - if repository_metadata: - metadata = repository_metadata.metadata - if metadata: - tool_shed_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) - # Get a dictionary of all repositories upon which the contents of the received repository depends. - repository_dependencies = \ - repository_dependency_util.get_repository_dependencies_for_changeset_revision( app=app, - repository=repository, - repository_metadata=repository_metadata, - toolshed_base_url=tool_shed_url, - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None, - circular_repository_dependencies=None ) - tool_dependencies = metadata.get( 'tool_dependencies', {} ) - if tool_dependencies: - new_tool_dependencies = {} - for dependency_key, requirements_dict in tool_dependencies.items(): - if dependency_key in [ 'set_environment' ]: - new_set_environment_dict_list = [] - for set_environment_dict in requirements_dict: - set_environment_dict[ 'repository_name' ] = repository_name - set_environment_dict[ 'repository_owner' ] = repository_owner - set_environment_dict[ 'changeset_revision' ] = changeset_revision - new_set_environment_dict_list.append( set_environment_dict ) - new_tool_dependencies[ dependency_key ] = new_set_environment_dict_list - else: - requirements_dict[ 'repository_name' ] = repository_name - requirements_dict[ 'repository_owner' ] = repository_owner - requirements_dict[ 'changeset_revision' ] = changeset_revision - new_tool_dependencies[ dependency_key ] = requirements_dict - tool_dependencies = new_tool_dependencies - # Cast unicode to string, with the exception of description, since it is free text and can contain special characters. - repo_info_dict[ str( repository.name ) ] = ( repository.description, - str( repository_clone_url ), - str( changeset_revision ), - str( ctx_rev ), - str( repository_owner ), - repository_dependencies, - tool_dependencies ) - return repo_info_dict - -def get_prior_install_required_dict( trans, tsr_ids, repo_info_dicts ): - """ - Return a dictionary whose keys are the received tsr_ids and whose values are a list of tsr_ids, each of which is contained in the received list of tsr_ids - and whose associated repository must be installed prior to the repository associated with the tsr_id key. - """ - # Initialize the dictionary. - prior_install_required_dict = {} - for tsr_id in tsr_ids: - prior_install_required_dict[ tsr_id ] = [] - # Inspect the repository dependencies for each repository about to be installed and populate the dictionary. - for repo_info_dict in repo_info_dicts: - repository, repository_dependencies = suc.get_repository_and_repository_dependencies_from_repo_info_dict( trans.app, repo_info_dict ) - if repository: - encoded_repository_id = trans.security.encode_id( repository.id ) - if encoded_repository_id in tsr_ids: - # We've located the database table record for one of the repositories we're about to install, so find out if it has any repository - # dependencies that require prior installation. - prior_install_ids = suc.get_repository_ids_requiring_prior_import_or_install( trans.app, tsr_ids, repository_dependencies ) - prior_install_required_dict[ encoded_repository_id ] = prior_install_ids - return prior_install_required_dict - -def get_repo_info_dict( trans, repository_id, changeset_revision ): - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) - repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) - repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans, repository ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, - repository_id, - changeset_revision ) - if not repository_metadata: - # The received changeset_revision is no longer installable, so get the next changeset_revision - # in the repository's changelog. This generally occurs only with repositories of type - # repository_suite_definition or tool_dependency_definition. - next_downloadable_changeset_revision = \ - suc.get_next_downloadable_changeset_revision( repository,repo, changeset_revision ) - if next_downloadable_changeset_revision: - repository_metadata = \ - suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, next_downloadable_changeset_revision ) - if repository_metadata: - # For now, we'll always assume that we'll get repository_metadata, but if we discover our assumption - # is not valid we'll have to enhance the callers to handle repository_metadata values of None in the - # returned repo_info_dict. - metadata = repository_metadata.metadata - if 'tools' in metadata: - includes_tools = True - else: - includes_tools = False - includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel - repository_dependencies_dict = metadata.get( 'repository_dependencies', {} ) - repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] ) - has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \ - suc.get_repository_dependency_types( repository_dependencies ) - if 'tool_dependencies' in metadata: - includes_tool_dependencies = True - else: - includes_tool_dependencies = False - else: - # Here's where we may have to handle enhancements to the callers. See above comment. - includes_tools = False - has_repository_dependencies = False - has_repository_dependencies_only_if_compiling_contained_td = False - includes_tool_dependencies = False - includes_tools_for_display_in_tool_panel = False - ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) - repo_info_dict = create_repo_info_dict( app=trans.app, - repository_clone_url=repository_clone_url, - changeset_revision=changeset_revision, - ctx_rev=str( ctx.rev() ), - repository_owner=repository.user.username, - repository_name=repository.name, - repository=repository, - repository_metadata=repository_metadata, - tool_dependencies=None, - repository_dependencies=None ) - return repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, \ - has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td - -def get_tool_shed_repository_ids( as_string=False, **kwd ): - tsrid = kwd.get( 'tool_shed_repository_id', None ) - tsridslist = util.listify( kwd.get( 'tool_shed_repository_ids', None ) ) - if not tsridslist: - tsridslist = util.listify( kwd.get( 'id', None ) ) - if tsridslist: - if tsrid and tsrid not in tsridslist: - tsridslist.append( tsrid ) - if as_string: - return ','.join( tsridslist ) - return tsridslist - else: - tsridslist = util.listify( kwd.get( 'ordered_tsr_ids', None ) ) - if as_string: - return ','.join( tsridslist ) - return tsridslist - if as_string: - '' - return [] - -def get_update_to_changeset_revision_and_ctx_rev( app, repository ): - """Return the changeset revision hash to which the repository can be updated.""" - changeset_revision_dict = {} - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, str( repository.tool_shed ) ) - params = '?name=%s&owner=%s&changeset_revision=%s' % ( str( repository.name ), - str( repository.owner ), - str( repository.installed_changeset_revision ) ) - url = common_util.url_join( tool_shed_url, 'repository/get_changeset_revision_and_ctx_rev%s' % params ) - try: - encoded_update_dict = common_util.tool_shed_get( app, tool_shed_url, url ) - if encoded_update_dict: - update_dict = encoding_util.tool_shed_decode( encoded_update_dict ) - includes_data_managers = update_dict.get( 'includes_data_managers', False ) - includes_datatypes = update_dict.get( 'includes_datatypes', False ) - includes_tools = update_dict.get( 'includes_tools', False ) - includes_tools_for_display_in_tool_panel = update_dict.get( 'includes_tools_for_display_in_tool_panel', False ) - includes_tool_dependencies = update_dict.get( 'includes_tool_dependencies', False ) - includes_workflows = update_dict.get( 'includes_workflows', False ) - has_repository_dependencies = update_dict.get( 'has_repository_dependencies', False ) - has_repository_dependencies_only_if_compiling_contained_td = update_dict.get( 'has_repository_dependencies_only_if_compiling_contained_td', False ) - changeset_revision = update_dict.get( 'changeset_revision', None ) - ctx_rev = update_dict.get( 'ctx_rev', None ) - changeset_revision_dict[ 'includes_data_managers' ] = includes_data_managers - changeset_revision_dict[ 'includes_datatypes' ] = includes_datatypes - changeset_revision_dict[ 'includes_tools' ] = includes_tools - changeset_revision_dict[ 'includes_tools_for_display_in_tool_panel' ] = includes_tools_for_display_in_tool_panel - changeset_revision_dict[ 'includes_tool_dependencies' ] = includes_tool_dependencies - changeset_revision_dict[ 'includes_workflows' ] = includes_workflows - changeset_revision_dict[ 'has_repository_dependencies' ] = has_repository_dependencies - changeset_revision_dict[ 'has_repository_dependencies_only_if_compiling_contained_td' ] = has_repository_dependencies_only_if_compiling_contained_td - changeset_revision_dict[ 'changeset_revision' ] = changeset_revision - changeset_revision_dict[ 'ctx_rev' ] = ctx_rev - except Exception, e: - log.debug( "Error getting change set revision for update from the tool shed for repository '%s': %s" % ( repository.name, str( e ) ) ) - changeset_revision_dict[ 'includes_data_managers' ] = False - changeset_revision_dict[ 'includes_datatypes' ] = False - changeset_revision_dict[ 'includes_tools' ] = False - changeset_revision_dict[ 'includes_tools_for_display_in_tool_panel' ] = False - changeset_revision_dict[ 'includes_tool_dependencies' ] = False - changeset_revision_dict[ 'includes_workflows' ] = False - changeset_revision_dict[ 'has_repository_dependencies' ] = False - changeset_revision_dict[ 'has_repository_dependencies_only_if_compiling_contained_td' ] = False - changeset_revision_dict[ 'changeset_revision' ] = None - changeset_revision_dict[ 'ctx_rev' ] = None - return changeset_revision_dict - -def handle_repository_contents( app, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir, - tool_shed=None, tool_section=None, shed_tool_conf=None, reinstalling=False ): - """ - Generate the metadata for the installed tool shed repository, among other things. This method is called from Galaxy - (never the tool shed) when an administrator is installing a new repository or reinstalling an uninstalled repository. - """ - install_model = app.install_model - shed_config_dict = app.toolbox.get_shed_config_dict_by_filename( shed_tool_conf ) - metadata_dict, invalid_file_tups = \ - metadata_util.generate_metadata_for_changeset_revision( app=app, - repository=tool_shed_repository, - changeset_revision=tool_shed_repository.changeset_revision, - repository_clone_url=repository_clone_url, - shed_config_dict=shed_config_dict, - relative_install_dir=relative_install_dir, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=False, - persist=True ) - tool_shed_repository.metadata = metadata_dict - # Update the tool_shed_repository.tool_shed_status column in the database. - tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( app, tool_shed_repository ) - if tool_shed_status_dict: - tool_shed_repository.tool_shed_status = tool_shed_status_dict - install_model.context.add( tool_shed_repository ) - install_model.context.flush() - if 'tool_dependencies' in metadata_dict and not reinstalling: - tool_dependencies = tool_dependency_util.create_tool_dependency_objects( app, - tool_shed_repository, - relative_install_dir, - set_status=True ) - if 'sample_files' in metadata_dict: - sample_files = metadata_dict.get( 'sample_files', [] ) - tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files ) - tool_data_table_conf_filename, tool_data_table_elems = \ - tool_util.install_tool_data_tables( app, tool_shed_repository, tool_index_sample_files ) - if tool_data_table_elems: - app.tool_data_tables.add_new_entries_from_config_file( tool_data_table_conf_filename, - None, - app.config.shed_tool_data_table_config, - persist=True ) - if 'tools' in metadata_dict: - tool_panel_dict = tool_util.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) - sample_files = metadata_dict.get( 'sample_files', [] ) - tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files ) - tool_util.copy_sample_files( app, tool_index_sample_files, tool_path=tool_path ) - sample_files_copied = [ str( s ) for s in tool_index_sample_files ] - repository_tools_tups = suc.get_repository_tools_tups( app, metadata_dict ) - if repository_tools_tups: - # Handle missing data table entries for tool parameters that are dynamically generated select lists. - repository_tools_tups = tool_util.handle_missing_data_table_entry( app, - relative_install_dir, - tool_path, - repository_tools_tups ) - # Handle missing index files for tool parameters that are dynamically generated select lists. - repository_tools_tups, sample_files_copied = \ - tool_util.handle_missing_index_file( app, - tool_path, - sample_files, - repository_tools_tups, - sample_files_copied ) - # Copy remaining sample files included in the repository to the ~/tool-data directory of the - # local Galaxy instance. - tool_util.copy_sample_files( app, sample_files, tool_path=tool_path, sample_files_copied=sample_files_copied ) - tool_util.add_to_tool_panel( app=app, - repository_name=tool_shed_repository.name, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.installed_changeset_revision, - repository_tools_tups=repository_tools_tups, - owner=tool_shed_repository.owner, - shed_tool_conf=shed_tool_conf, - tool_panel_dict=tool_panel_dict, - new_install=True ) - if 'data_manager' in metadata_dict: - new_data_managers = data_manager_util.install_data_managers( app, - app.config.shed_data_manager_config_file, - metadata_dict, - shed_config_dict, - relative_install_dir, - tool_shed_repository, - repository_tools_tups ) - if 'datatypes' in metadata_dict: - tool_shed_repository.status = install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES - if not tool_shed_repository.includes_datatypes: - tool_shed_repository.includes_datatypes = True - install_model.context.add( tool_shed_repository ) - install_model.context.flush() - files_dir = relative_install_dir - if shed_config_dict.get( 'tool_path' ): - files_dir = os.path.join( shed_config_dict[ 'tool_path' ], files_dir ) - datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, files_dir ) - # Load data types required by tools. - converter_path, display_path = \ - datatype_util.alter_config_and_load_prorietary_datatypes( app, datatypes_config, files_dir, override=False ) - if converter_path or display_path: - # Create a dictionary of tool shed repository related information. - repository_dict = \ - datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, - name=tool_shed_repository.name, - owner=tool_shed_repository.owner, - installed_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dicts=metadata_dict.get( 'tools', [] ), - converter_path=converter_path, - display_path=display_path ) - if converter_path: - # Load proprietary datatype converters - app.datatypes_registry.load_datatype_converters( app.toolbox, installed_repository_dict=repository_dict ) - if display_path: - # Load proprietary datatype display applications - app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) - -def handle_tool_shed_repositories( app, installation_dict, using_api=False ): - # The following installation_dict entries are all required. - install_repository_dependencies = installation_dict[ 'install_repository_dependencies' ] - new_tool_panel_section_label = installation_dict[ 'new_tool_panel_section_label' ] - no_changes_checked = installation_dict[ 'no_changes_checked' ] - repo_info_dicts = installation_dict[ 'repo_info_dicts' ] - tool_panel_section_id = installation_dict[ 'tool_panel_section_id' ] - tool_path = installation_dict[ 'tool_path' ] - tool_shed_url = installation_dict[ 'tool_shed_url' ] - created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts = \ - repository_dependency_util.create_repository_dependency_objects( app=app, - tool_path=tool_path, - tool_shed_url=tool_shed_url, - repo_info_dicts=repo_info_dicts, - install_repository_dependencies=install_repository_dependencies, - no_changes_checked=no_changes_checked, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) - return created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts - -def update_repository_record( app, repository, updated_metadata_dict, updated_changeset_revision, updated_ctx_rev ): - """ - Update a tool_shed_repository database record with new information retrieved from the - Tool Shed. This happens when updating an installed repository to a new changeset revision. - """ - repository.metadata = updated_metadata_dict - # Update the repository.changeset_revision column in the database. - repository.changeset_revision = updated_changeset_revision - repository.ctx_rev = updated_ctx_rev - # Update the repository.tool_shed_status column in the database. - tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( app, repository ) - if tool_shed_status_dict: - repository.tool_shed_status = tool_shed_status_dict - else: - repository.tool_shed_status = None - app.install_model.context.add( repository ) - app.install_model.context.flush() - app.install_model.context.refresh( repository ) - return repository diff -r 95937a7e51f20da0ebf0853f9385286d2c70c4eb -r 6294a65d863a4875a9e096ca5e81c3f30e8b0bb6 lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -111,11 +111,11 @@ created_tool_shed_repositories = self.create_or_update_tool_shed_repository_records( name, changeset_revision, repository_dependencies_dict ) - # Order the repositories for proper installation. This process is similar to the process used when installing tool - # shed repositories (i.e., the order_components_for_installation() method in ~/lib/tool_shed/galaxy_install/ - # repository_util), but does not handle managing tool panel sections and other components since repository dependency - # definitions contained in tool shed repositories with migrated tools must never define a relationship to a repository - # dependency that contains a tool. + # Order the repositories for proper installation. This process is similar to the + # process used when installing tool shed repositories, but does not handle managing + # tool panel sections and other components since repository dependency definitions + # contained in tool shed repositories with migrated tools must never define a relationship + # to a repository dependency that contains a tool. ordered_tool_shed_repositories = self.order_repositories_for_installation( created_tool_shed_repositories, repository_dependencies_dict ) @@ -585,11 +585,10 @@ in the list of repositories about to be installed, then they are not considered. Repository dependency definitions that contain circular dependencies should not result in an infinite loop, but obviously prior installation will not be handled for one or more of the repositories that require prior installation. This - process is similar to the process used when installing tool shed repositories (i.e., the - order_components_for_installation() method in ~/lib/tool_shed/galaxy_install/repository_util), but does not - handle managing tool panel sections and other components since repository dependency definitions contained - in tool shed repositories with migrated tools must never define a relationship to a repository dependency - that contains a tool. + process is similar to the process used when installing tool shed repositories, but does not handle managing + tool panel sections and other components since repository dependency definitions contained in tool shed + repositories with migrated tools must never define a relationship to a repository dependency that contains + a tool. """ ordered_tool_shed_repositories = [] ordered_tsr_ids = [] diff -r 95937a7e51f20da0ebf0853f9385286d2c70c4eb -r 6294a65d863a4875a9e096ca5e81c3f30e8b0bb6 lib/tool_shed/galaxy_install/update_repository_manager.py --- a/lib/tool_shed/galaxy_install/update_repository_manager.py +++ b/lib/tool_shed/galaxy_install/update_repository_manager.py @@ -4,6 +4,8 @@ import logging import threading import tool_shed.util.shed_util_common as suc +from tool_shed.util import common_util +from tool_shed.util import encoding_util log = logging.getLogger( __name__ ) @@ -20,6 +22,52 @@ self.restarter.start() self.seconds_to_sleep = int( app.config.hours_between_check * 3600 ) + def get_update_to_changeset_revision_and_ctx_rev( self, repository ): + """Return the changeset revision hash to which the repository can be updated.""" + changeset_revision_dict = {} + tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, str( repository.tool_shed ) ) + params = '?name=%s&owner=%s&changeset_revision=%s' % ( str( repository.name ), + str( repository.owner ), + str( repository.installed_changeset_revision ) ) + url = common_util.url_join( tool_shed_url, 'repository/get_changeset_revision_and_ctx_rev%s' % params ) + try: + encoded_update_dict = common_util.tool_shed_get( self.app, tool_shed_url, url ) + if encoded_update_dict: + update_dict = encoding_util.tool_shed_decode( encoded_update_dict ) + includes_data_managers = update_dict.get( 'includes_data_managers', False ) + includes_datatypes = update_dict.get( 'includes_datatypes', False ) + includes_tools = update_dict.get( 'includes_tools', False ) + includes_tools_for_display_in_tool_panel = update_dict.get( 'includes_tools_for_display_in_tool_panel', False ) + includes_tool_dependencies = update_dict.get( 'includes_tool_dependencies', False ) + includes_workflows = update_dict.get( 'includes_workflows', False ) + has_repository_dependencies = update_dict.get( 'has_repository_dependencies', False ) + has_repository_dependencies_only_if_compiling_contained_td = update_dict.get( 'has_repository_dependencies_only_if_compiling_contained_td', False ) + changeset_revision = update_dict.get( 'changeset_revision', None ) + ctx_rev = update_dict.get( 'ctx_rev', None ) + changeset_revision_dict[ 'includes_data_managers' ] = includes_data_managers + changeset_revision_dict[ 'includes_datatypes' ] = includes_datatypes + changeset_revision_dict[ 'includes_tools' ] = includes_tools + changeset_revision_dict[ 'includes_tools_for_display_in_tool_panel' ] = includes_tools_for_display_in_tool_panel + changeset_revision_dict[ 'includes_tool_dependencies' ] = includes_tool_dependencies + changeset_revision_dict[ 'includes_workflows' ] = includes_workflows + changeset_revision_dict[ 'has_repository_dependencies' ] = has_repository_dependencies + changeset_revision_dict[ 'has_repository_dependencies_only_if_compiling_contained_td' ] = has_repository_dependencies_only_if_compiling_contained_td + changeset_revision_dict[ 'changeset_revision' ] = changeset_revision + changeset_revision_dict[ 'ctx_rev' ] = ctx_rev + except Exception, e: + log.debug( "Error getting change set revision for update from the tool shed for repository '%s': %s" % ( repository.name, str( e ) ) ) + changeset_revision_dict[ 'includes_data_managers' ] = False + changeset_revision_dict[ 'includes_datatypes' ] = False + changeset_revision_dict[ 'includes_tools' ] = False + changeset_revision_dict[ 'includes_tools_for_display_in_tool_panel' ] = False + changeset_revision_dict[ 'includes_tool_dependencies' ] = False + changeset_revision_dict[ 'includes_workflows' ] = False + changeset_revision_dict[ 'has_repository_dependencies' ] = False + changeset_revision_dict[ 'has_repository_dependencies_only_if_compiling_contained_td' ] = False + changeset_revision_dict[ 'changeset_revision' ] = None + changeset_revision_dict[ 'ctx_rev' ] = None + return changeset_revision_dict + def __restarter( self ): log.info( 'Update repository manager restarter starting up...' ) while self.running: @@ -48,6 +96,26 @@ self.running = False self.sleeper.wake() + def update_repository_record( self, repository, updated_metadata_dict, updated_changeset_revision, updated_ctx_rev ): + """ + Update a tool_shed_repository database record with new information retrieved from the + Tool Shed. This happens when updating an installed repository to a new changeset revision. + """ + repository.metadata = updated_metadata_dict + # Update the repository.changeset_revision column in the database. + repository.changeset_revision = updated_changeset_revision + repository.ctx_rev = updated_ctx_rev + # Update the repository.tool_shed_status column in the database. + tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( self.app, repository ) + if tool_shed_status_dict: + repository.tool_shed_status = tool_shed_status_dict + else: + repository.tool_shed_status = None + self.app.install_model.context.add( repository ) + self.app.install_model.context.flush() + self.app.install_model.context.refresh( repository ) + return repository + class Sleeper( object ): """ diff -r 95937a7e51f20da0ebf0853f9385286d2c70c4eb -r 6294a65d863a4875a9e096ca5e81c3f30e8b0bb6 lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -1,12 +1,11 @@ import copy +import json import logging import os import urllib import urllib2 from galaxy import util from galaxy import web -from galaxy.util import json -from galaxy.tools.deps.resolvers import INDETERMINATE_DEPENDENCY import tool_shed.util.shed_util_common as suc from tool_shed.util import common_util from tool_shed.util import container_util @@ -15,7 +14,6 @@ from tool_shed.util import datatype_util from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util -from tool_shed.util import xml_util log = logging.getLogger( __name__ ) @@ -452,7 +450,7 @@ response = urllib2.urlopen( request ).read() if response: try: - required_repo_info_dict = json.from_json_string( response ) + required_repo_info_dict = json.loads( response ) except Exception, e: log.exception( e ) return all_repo_info_dicts diff -r 95937a7e51f20da0ebf0853f9385286d2c70c4eb -r 6294a65d863a4875a9e096ca5e81c3f30e8b0bb6 lib/tool_shed/util/common_util.py --- a/lib/tool_shed/util/common_util.py +++ b/lib/tool_shed/util/common_util.py @@ -1,7 +1,8 @@ +import json import logging import os import urllib2 -from galaxy.util import json +from galaxy import util from galaxy.util.odict import odict from galaxy.web import url_for from tool_shed.util import encoding_util @@ -109,12 +110,12 @@ tool_shed_url = get_tool_shed_url_from_tool_shed_registry( app, str( repository.tool_shed ) ) return url_join( tool_shed_url, 'repos', str( repository.owner ), str( repository.name ) ) -def generate_clone_url_for_repository_in_tool_shed( trans, repository ): +def generate_clone_url_for_repository_in_tool_shed( user, repository ): """Generate the URL for cloning a repository that is in the tool shed.""" base_url = url_for( '/', qualified=True ).rstrip( '/' ) - if trans and trans.user: + if user: protocol, base = base_url.split( '://' ) - username = '%s@' % trans.user.username + username = '%s@' % user.username return '%s://%s%s/repos/%s/%s' % ( protocol, username, base, repository.user.username, repository.name ) else: return '%s/repos/%s/%s' % ( base_url, repository.user.username, repository.name ) @@ -156,7 +157,7 @@ print "The URL\n%s\nraised the exception:\n%s\n" % ( url, str( e ) ) if tool_shed_accessible: if len( raw_text ) > 2: - encoded_text = json.from_json_string( raw_text ) + encoded_text = json.loads( raw_text ) repository_dependencies_dict = encoding_util.tool_shed_decode( encoded_text ) return tool_shed_accessible, repository_dependencies_dict @@ -194,6 +195,27 @@ tool_dependencies.append( ( tool_dependency_name, tool_dependency_version, tool_dependency_type ) ) return tool_shed_accessible, tool_dependencies +def get_tool_shed_repository_ids( as_string=False, **kwd ): + tsrid = kwd.get( 'tool_shed_repository_id', None ) + tsridslist = util.listify( kwd.get( 'tool_shed_repository_ids', None ) ) + if not tsridslist: + tsridslist = util.listify( kwd.get( 'id', None ) ) + if tsridslist is not None: + if tsrid is not None and tsrid not in tsridslist: + tsridslist.append( tsrid ) + if as_string: + return ','.join( tsridslist ) + return tsridslist + else: + tsridslist = util.listify( kwd.get( 'ordered_tsr_ids', None ) ) + if tsridslist is not None: + if as_string: + return ','.join( tsridslist ) + return tsridslist + if as_string: + '' + return [] + def get_tool_shed_url_from_tool_shed_registry( app, tool_shed ): """ The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is diff -r 95937a7e51f20da0ebf0853f9385286d2c70c4eb -r 6294a65d863a4875a9e096ca5e81c3f30e8b0bb6 lib/tool_shed/util/export_util.py --- a/lib/tool_shed/util/export_util.py +++ b/lib/tool_shed/util/export_util.py @@ -256,7 +256,7 @@ repo_info_dict = {} # Cast unicode to string. repo_info_dict[ str( repository.name ) ] = ( str( repository.description ), - common_util.generate_clone_url_for_repository_in_tool_shed( trans, repository ), + common_util.generate_clone_url_for_repository_in_tool_shed( trans.user, repository ), str( changeset_revision ), str( ctx.rev() ), str( repository.user.username ), diff -r 95937a7e51f20da0ebf0853f9385286d2c70c4eb -r 6294a65d863a4875a9e096ca5e81c3f30e8b0bb6 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -1776,7 +1776,7 @@ log.debug( "Resetting all metadata on repository: %s" % repository.name ) repo_dir = repository.repo_path( trans.app ) repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) - repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans, repository ) + repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans.user, repository ) # The list of changeset_revisions refers to repository_metadata records that have been created or updated. When the following loop # completes, we'll delete all repository_metadata records for this repository that do not have a changeset_revision value in this list. changeset_revisions = [] @@ -1955,7 +1955,7 @@ message = '' status = 'done' encoded_id = trans.security.encode_id( repository.id ) - repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans, repository ) + repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans.user, repository ) repo_dir = repository.repo_path( trans.app ) repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) metadata_dict, invalid_file_tups = generate_metadata_for_changeset_revision( app=trans.app, diff -r 95937a7e51f20da0ebf0853f9385286d2c70c4eb -r 6294a65d863a4875a9e096ca5e81c3f30e8b0bb6 lib/tool_shed/util/repository_maintenance_util.py --- a/lib/tool_shed/util/repository_maintenance_util.py +++ b/lib/tool_shed/util/repository_maintenance_util.py @@ -3,9 +3,12 @@ import os import re import tool_shed.util.shed_util_common as suc +from tool_shed.util import common_util from tool_shed.util import hg_util from tool_shed.util import import_util +from tool_shed.util import repository_dependency_util from galaxy import util +from galaxy import web from galaxy.web.form_builder import build_select_field from galaxy.webapps.tool_shed.model import directory_hash_id @@ -53,6 +56,79 @@ fp.write( 'hgext.purge=' ) fp.close() +def create_repo_info_dict( app, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_name=None, + repository=None, repository_metadata=None, tool_dependencies=None, repository_dependencies=None ): + """ + Return a dictionary that includes all of the information needed to install a repository into a local + Galaxy instance. The dictionary will also contain the recursive list of repository dependencies defined + for the repository, as well as the defined tool dependencies. + + This method is called from Galaxy under four scenarios: + 1. During the tool shed repository installation process via the tool shed's get_repository_information() + method. In this case both the received repository and repository_metadata will be objects, but + tool_dependencies and repository_dependencies will be None. + 2. When getting updates for an install repository where the updates include newly defined repository + dependency definitions. This scenario is similar to 1. above. The tool shed's get_repository_information() + method is the caller, and both the received repository and repository_metadata will be objects, but + tool_dependencies and repository_dependencies will be None. + 3. When a tool shed repository that was uninstalled from a Galaxy instance is being reinstalled with no + updates available. In this case, both repository and repository_metadata will be None, but tool_dependencies + and repository_dependencies will be objects previously retrieved from the tool shed if the repository includes + definitions for them. + 4. When a tool shed repository that was uninstalled from a Galaxy instance is being reinstalled with updates + available. In this case, this method is reached via the tool shed's get_updated_repository_information() + method, and both repository and repository_metadata will be objects but tool_dependencies and + repository_dependencies will be None. + """ + repo_info_dict = {} + repository = suc.get_repository_by_name_and_owner( app, repository_name, repository_owner ) + if app.name == 'tool_shed': + # We're in the tool shed. + repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, + app.security.encode_id( repository.id ), + changeset_revision ) + if repository_metadata: + metadata = repository_metadata.metadata + if metadata: + tool_shed_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) + # Get a dictionary of all repositories upon which the contents of the received repository depends. + repository_dependencies = \ + repository_dependency_util.get_repository_dependencies_for_changeset_revision( app=app, + repository=repository, + repository_metadata=repository_metadata, + toolshed_base_url=tool_shed_url, + key_rd_dicts_to_be_processed=None, + all_repository_dependencies=None, + handled_key_rd_dicts=None, + circular_repository_dependencies=None ) + tool_dependencies = metadata.get( 'tool_dependencies', {} ) + if tool_dependencies: + new_tool_dependencies = {} + for dependency_key, requirements_dict in tool_dependencies.items(): + if dependency_key in [ 'set_environment' ]: + new_set_environment_dict_list = [] + for set_environment_dict in requirements_dict: + set_environment_dict[ 'repository_name' ] = repository_name + set_environment_dict[ 'repository_owner' ] = repository_owner + set_environment_dict[ 'changeset_revision' ] = changeset_revision + new_set_environment_dict_list.append( set_environment_dict ) + new_tool_dependencies[ dependency_key ] = new_set_environment_dict_list + else: + requirements_dict[ 'repository_name' ] = repository_name + requirements_dict[ 'repository_owner' ] = repository_owner + requirements_dict[ 'changeset_revision' ] = changeset_revision + new_tool_dependencies[ dependency_key ] = requirements_dict + tool_dependencies = new_tool_dependencies + # Cast unicode to string, with the exception of description, since it is free text and can contain special characters. + repo_info_dict[ str( repository.name ) ] = ( repository.description, + str( repository_clone_url ), + str( changeset_revision ), + str( ctx_rev ), + str( repository_owner ), + repository_dependencies, + tool_dependencies ) + return repo_info_dict + def create_repository( trans, name, type, description, long_description, user_id, category_ids=[] ): # Add the repository record to the database. repository = trans.app.model.Repository( name=name, @@ -182,6 +258,62 @@ import_results_tups.append( ( ok, ( str( name ), str( username ) ), results_message ) ) return import_results_tups +def get_repo_info_dict( app, user, repository_id, changeset_revision ): + repository = suc.get_repository_in_tool_shed( app, repository_id ) + repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) + repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( user, repository ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, + repository_id, + changeset_revision ) + if not repository_metadata: + # The received changeset_revision is no longer installable, so get the next changeset_revision + # in the repository's changelog. This generally occurs only with repositories of type + # repository_suite_definition or tool_dependency_definition. + next_downloadable_changeset_revision = \ + suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) + if next_downloadable_changeset_revision: + repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, + repository_id, + next_downloadable_changeset_revision ) + if repository_metadata: + # For now, we'll always assume that we'll get repository_metadata, but if we discover our assumption + # is not valid we'll have to enhance the callers to handle repository_metadata values of None in the + # returned repo_info_dict. + metadata = repository_metadata.metadata + if 'tools' in metadata: + includes_tools = True + else: + includes_tools = False + includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel + repository_dependencies_dict = metadata.get( 'repository_dependencies', {} ) + repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] ) + has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \ + suc.get_repository_dependency_types( repository_dependencies ) + if 'tool_dependencies' in metadata: + includes_tool_dependencies = True + else: + includes_tool_dependencies = False + else: + # Here's where we may have to handle enhancements to the callers. See above comment. + includes_tools = False + has_repository_dependencies = False + has_repository_dependencies_only_if_compiling_contained_td = False + includes_tool_dependencies = False + includes_tools_for_display_in_tool_panel = False + ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) + repo_info_dict = create_repo_info_dict( app=app, + repository_clone_url=repository_clone_url, + changeset_revision=changeset_revision, + ctx_rev=str( ctx.rev() ), + repository_owner=repository.user.username, + repository_name=repository.name, + repository=repository, + repository_metadata=repository_metadata, + tool_dependencies=None, + repository_dependencies=None ) + return repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, \ + has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td + def get_repository_admin_role_name( repository_name, repository_owner ): return '%s_%s_admin' % ( str( repository_name ), str( repository_owner ) ) diff -r 95937a7e51f20da0ebf0853f9385286d2c70c4eb -r 6294a65d863a4875a9e096ca5e81c3f30e8b0bb6 templates/webapps/tool_shed/repository/common.mako --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -216,7 +216,7 @@ <%def name="render_clone_str( repository )"><% from tool_shed.util.common_util import generate_clone_url_for_repository_in_tool_shed - clone_str = generate_clone_url_for_repository_in_tool_shed( trans, repository ) + clone_str = generate_clone_url_for_repository_in_tool_shed( trans.user, repository ) %> hg clone <a href="${clone_str}">${clone_str}</a></%def> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.