[galaxy-commits] commit/galaxy-central: greg: Add support for repairing an installed tool shed repository and all of it's repository dependency hierarchy. Repairing a repository hierarchy will attempt to ensure all repositories in the hierarchy are correctly installed and all tool dependencies of each repository in the hierarchy are correctly installed.

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3fa9df444b4b/ Changeset: 3fa9df444b4b User: greg Date: 2013-06-27 03:17:00 Summary: Add support for repairing an installed tool shed repository and all of it's repository dependency hierarchy. Repairing a repository hierarchy will attempt to ensure all repositories in the hierarchy are correctly installed and all tool dependencies of each repository in the hierarchy are correctly installed. There is a new Repository Actions pop-up menu item for each repository labeled "Repair repository" that enables this in the browser. This is also supported in the enhanced Galaxy API using a command like the following from the ~/scripts/api directory: python ./repair_tool_shed_repository.py --api <Galaxy admin API key> -l http://localhost:8763 --url http://localhost:9009/ -o test -r 1018e3cee313 --name chemicaltoolbox Affected #: 10 files diff -r eaa5112fefcd41ce6509bc22791303a155007680 -r 3fa9df444b4b81f94b1c42a033c685a6e23827be lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -9,6 +9,7 @@ from tool_shed.galaxy_install import repository_util from tool_shed.util import common_util +from tool_shed.util import encoding_util import tool_shed.util.shed_util_common as suc log = logging.getLogger( __name__ ) @@ -244,7 +245,7 @@ # order the list of tsr_ids to ensure all repositories install in the required order. tsr_ids = [ trans.security.encode_id( tool_shed_repository.id ) for tool_shed_repository in tool_shed_repositories ] ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ - repository_util.order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys ) + repository_util.order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) # Install the repositories, keeping track of each one for later display. for index, tsr_id in enumerate( ordered_tsr_ids ): tool_shed_repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) @@ -358,3 +359,56 @@ elif isinstance( installed_tool_shed_repositories, list ): all_installed_tool_shed_repositories.extend( installed_tool_shed_repositories ) return all_installed_tool_shed_repositories + + @web.expose_api + def repair_repository_revision( self, trans, payload, **kwd ): + """ + POST /api/tool_shed_repositories/repair_repository_revision + Repair a specified repository revision previously installed into Galaxy. + + :param key: the current Galaxy admin user's API key + + The following parameters are included in the payload. + :param tool_shed_url (required): the base URL of the Tool Shed from which the Repository was installed + :param name (required): the name of the Repository + :param owner (required): the owner of the Repository + :param changset_revision (required): the changset_revision of the RepositoryMetadata object associated with the Repository + """ + api_key = kwd.get( 'key', None ) + # Get the information about the repository to be installed from the payload. + tool_shed_url = payload.get( 'tool_shed_url', '' ) + if not tool_shed_url: + raise HTTPBadRequest( detail="Missing required parameter 'tool_shed_url'." ) + name = payload.get( 'name', '' ) + if not name: + raise HTTPBadRequest( detail="Missing required parameter 'name'." ) + owner = payload.get( 'owner', '' ) + if not owner: + raise HTTPBadRequest( detail="Missing required parameter 'owner'." ) + changeset_revision = payload.get( 'changeset_revision', '' ) + if not changeset_revision: + raise HTTPBadRequest( detail="Missing required parameter 'changeset_revision'." ) + tool_shed_repositories = [] + tool_shed_repository = suc.get_tool_shed_repository_by_shed_name_owner_changeset_revision( trans.app, tool_shed_url, name, owner, changeset_revision ) + repair_dict = repository_util.get_repair_dict( trans, tool_shed_repository ) + ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] ) + ordered_repo_info_dicts = repair_dict.get( 'ordered_repo_info_dicts', [] ) + if ordered_tsr_ids and ordered_repo_info_dicts: + repositories_for_repair = [] + for index, tsr_id in enumerate( ordered_tsr_ids ): + repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) + repo_info_dict = ordered_repo_info_dicts[ index ] + # TODO: handle errors in repair_dict. + repair_dict = repository_util.repair_tool_shed_repository( trans, + repository, + encoding_util.tool_shed_encode( repo_info_dict ) ) + repository_dict = repository.get_api_value( value_mapper=default_tool_shed_repository_value_mapper( trans, repository ) ) + repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', + action='show', + id=trans.security.encode_id( repository.id ) ) + if repair_dict: + errors = repair_dict.get( repository.name, [] ) + repository_dict[ 'errors_attempting_repair' ] = ' '.join( errors ) + tool_shed_repositories.append( repository_dict ) + # Display the list of repaired repositories. + return tool_shed_repositories diff -r eaa5112fefcd41ce6509bc22791303a155007680 -r 3fa9df444b4b81f94b1c42a033c685a6e23827be lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -163,12 +163,13 @@ webapp.mapper.connect("workflow_dict", '/api/workflows/download/{workflow_id}', controller='workflows', action='workflow_dict', conditions=dict(method=['GET'])) # Galaxy API for tool shed features. webapp.mapper.resource( 'tool_shed_repository', - 'tool_shed_repositories', - controller='tool_shed_repositories', - name_prefix='tool_shed_repository_', - path_prefix='/api', - new={ 'install_repository_revision' : 'POST' }, - parent_resources=dict( member_name='tool_shed_repository', collection_name='tool_shed_repositories' ) ) + 'tool_shed_repositories', + member={ 'repair_repository_revision' : 'POST' }, + controller='tool_shed_repositories', + name_prefix='tool_shed_repository_', + path_prefix='/api', + new={ 'install_repository_revision' : 'POST' }, + parent_resources=dict( member_name='tool_shed_repository', collection_name='tool_shed_repositories' ) ) # Connect logger from app if app.trace_logger: webapp.trace_logger = app.trace_logger diff -r eaa5112fefcd41ce6509bc22791303a155007680 -r 3fa9df444b4b81f94b1c42a033c685a6e23827be lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -319,27 +319,6 @@ @web.expose @web.require_admin - def get_repository_dependencies( self, trans, repository_id, repository_name, repository_owner, changeset_revision ): - """ - Send a request to the appropriate tool shed to retrieve the dictionary of repository dependencies defined for the received repository - name, owner and changeset revision. The received repository_id is the encoded id of the installed tool shed repository in Galaxy. We - need it so that we can derive the tool shed from which it was installed. - """ - repository = suc.get_installed_tool_shed_repository( trans, repository_id ) - tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed ) - url = suc.url_join( tool_shed_url, - 'repository/get_repository_dependencies?name=%s&owner=%s&changeset_revision=%s' % \ - ( repository_name, repository_owner, changeset_revision ) ) - raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) - if len( raw_text ) > 2: - encoded_text = json.from_json_string( raw_text ) - text = encoding_util.tool_shed_decode( encoded_text ) - else: - text = '' - return text - - @web.expose - @web.require_admin def get_tool_dependencies( self, trans, repository_id, repository_name, repository_owner, changeset_revision ): """ Send a request to the appropriate tool shed to retrieve the dictionary of tool dependencies defined for the received repository name, @@ -548,12 +527,7 @@ def manage_repositories( self, trans, **kwd ): message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) - tsrid = kwd.get( 'tool_shed_repository_id', None ) - tsridslist = util.listify( kwd.get( 'tool_shed_repository_ids', None ) ) - if not tsridslist: - tsridslist = util.listify( kwd.get( 'id', None ) ) - if tsrid and tsrid not in tsridslist: - tsridslist.append( tsrid ) + tsridslist = repository_util.get_tool_shed_repository_ids( **kwd ) if 'operation' in kwd: operation = kwd[ 'operation' ].lower() if not tsridslist: @@ -596,7 +570,7 @@ # Some repositories may have repository dependencies that are required to be installed before the dependent repository, so we'll # order the list of tsr_ids to ensure all repositories install in the required order. ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ - repository_util.order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys ) + repository_util.order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) for tsr_id in ordered_tsr_ids: repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) if repository.status in [ trans.model.ToolShedRepository.installation_status.NEW, @@ -615,6 +589,14 @@ else: kwd[ 'message' ] = 'All selected tool shed repositories are already installed.' kwd[ 'status' ] = 'error' + elif operation == "repair": + # In this case, tsridslist is ordered. + repositories_for_repair = [] + for tsr_id in tsridslist: + repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) + repositories_for_repair.append( repository ) + ordered_repo_info_dicts = kwd.get( 'ordered_repo_info_dicts', [] ) + self.repair_tool_shed_repositories( trans, repositories_for_repair, ordered_repo_info_dicts ) return self.repository_installation_grid( trans, **kwd ) @web.expose @@ -799,14 +781,9 @@ @web.expose @web.require_admin def monitor_repository_installation( self, trans, **kwd ): - tsrid = kwd.get( 'tool_shed_repository_id', None ) - tsridslist = util.listify( kwd.get( 'tool_shed_repository_ids', None ) ) + tsridslist = repository_util.get_tool_shed_repository_ids( **kwd ) if not tsridslist: - tsridslist = util.listify( kwd.get( 'id', None ) ) - if tsrid and tsrid not in tsridslist: - tsridslist.append( tsrid ) - if not tsridslist: - tsridslist = get_ids_of_tool_shed_repositories_being_installed( trans, as_string=False ) + tsridslist = suc.get_ids_of_tool_shed_repositories_being_installed( trans, as_string=False ) kwd[ 'tool_shed_repository_ids' ] = tsridslist return self.repository_installation_grid( trans, **kwd ) @@ -1097,11 +1074,7 @@ else: # Entering this else block occurs only if the tool_shed_repository does not include any valid tools. if install_repository_dependencies: - repository_dependencies = self.get_repository_dependencies( trans=trans, - repository_id=repository_id, - repository_name=tool_shed_repository.name, - repository_owner=tool_shed_repository.owner, - changeset_revision=tool_shed_repository.changeset_revision ) + repository_dependencies = repository_dependency_util.get_repository_dependencies_for_installed_tool_shed_repository( trans, tool_shed_repository ) else: repository_dependencies = None if metadata: @@ -1174,10 +1147,28 @@ Inspect the repository dependency hierarchy for a specified repository and attempt to make sure they are all properly installed as well as each repository's tool dependencies. """ + # TODO: figure out how to handle installing repositories and tool dependencies consecutively since each redirects to an ajaxian grid. This + # is not a problem in that Galaxy API. message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) - repository_id = kwd[ 'id' ] + repository_id = kwd.get( 'id', None ) + if not repository_id: + message = 'Invalid installed tool shed repository id %s received.' % str( repository_id ) + status = 'error' + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='browse_repositories', + message=message, + status=status ) ) tool_shed_repository = suc.get_installed_tool_shed_repository( trans, repository_id ) + repair_dict = repository_util.get_repair_dict( trans, tool_shed_repository ) + ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] ) + ordered_repo_info_dicts = repair_dict.get( 'ordered_repo_info_dicts', [] ) + if ordered_tsr_ids and ordered_repo_info_dicts: + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='manage_repositories', + operation='repair', + ordered_tsr_ids=ordered_tsr_ids, + ordered_repo_info_dicts=ordered_repo_info_dicts ) ) @web.json def repository_installation_status_updates( self, trans, ids=None, status_list=None ): @@ -1202,6 +1193,21 @@ @web.expose @web.require_admin + def repair_tool_shed_repositories( self, trans, tool_shed_repositories, repo_info_dicts, **kwd ): + """Repair specified tool shed repositories.""" + # The received lists of tool_shed_repositories and repo_info_dicts are ordered. + for index, tool_shed_repository in enumerate( tool_shed_repositories ): + repo_info_dict = repo_info_dicts[ index ] + repair_dict = repository_util.repair_tool_shed_repository( trans, + tool_shed_repository, + encoding_util.tool_shed_encode( repo_info_dict ) ) + tsr_ids_for_monitoring = [ trans.security.encode_id( tsr.id ) for tsr in tool_shed_repositories ] + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='monitor_repository_installation', + tool_shed_repository_ids=tsr_ids_for_monitoring ) ) + + @web.expose + @web.require_admin def reselect_tool_panel_section( self, trans, **kwd ): """ Select or change the tool panel section to contain the tools included in the tool shed repository being reinstalled. If there are updates @@ -1268,11 +1274,7 @@ raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) readme_files_dict = json.from_json_string( raw_text ) tool_dependencies = metadata.get( 'tool_dependencies', None ) - repository_dependencies = self.get_repository_dependencies( trans=trans, - repository_id=repository_id, - repository_name=tool_shed_repository.name, - repository_owner=tool_shed_repository.owner, - changeset_revision=tool_shed_repository.changeset_revision ) + repository_dependencies = repository_dependency_util.get_repository_dependencies_for_installed_tool_shed_repository( trans, tool_shed_repository ) repo_info_dict = repository_util.create_repo_info_dict( trans=trans, repository_clone_url=repository_clone_url, changeset_revision=tool_shed_repository.changeset_revision, @@ -1439,13 +1441,13 @@ """An error occurred while cloning the repository, so reset everything necessary to enable another attempt.""" repository = suc.get_installed_tool_shed_repository( trans, kwd[ 'id' ] ) if kwd.get( 'reset_repository', False ): - suc.set_repository_attributes( trans, - repository, - status=trans.model.ToolShedRepository.installation_status.NEW, - error_message=None, - deleted=False, - uninstalled=False, - remove_from_disk=True ) + repository_util.set_repository_attributes( trans, + repository, + status=trans.model.ToolShedRepository.installation_status.NEW, + error_message=None, + deleted=False, + uninstalled=False, + remove_from_disk=True ) new_kwd = {} new_kwd[ 'message' ] = "You can now attempt to install the repository named <b>%s</b> again." % repository.name new_kwd[ 'status' ] = "done" diff -r eaa5112fefcd41ce6509bc22791303a155007680 -r 3fa9df444b4b81f94b1c42a033c685a6e23827be lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -6,6 +6,7 @@ from galaxy import tools from galaxy.util import asbool from galaxy.util import json +from galaxy import util from galaxy import web from galaxy.model.orm import or_ from galaxy.webapps.tool_shed.util import container_util @@ -93,6 +94,30 @@ tool_dependencies ) return repo_info_dict +def get_installed_repositories_from_repository_dependencies( trans, repository_dependencies_dict ): + installed_repositories = [] + for rd_key, rd_vals in repository_dependencies_dict.items(): + if rd_key in [ 'root_key', 'description' ]: + continue + # rd_key is something like: 'http://localhost:9009__ESEP__package_rdkit_2012_12__ESEP__test__ESEP__d635ff...' + # rd_val is something like: [['http://localhost:9009', 'package_numpy_1_7', 'test', 'cddd64ecd985', 'True']] + try: + tool_shed, name, owner, changeset_revision, prior_installation_required = container_util.get_components_from_key( rd_key ) + except: + tool_shed, name, owner, changeset_revision = container_util.get_components_from_key( rd_val ) + installed_repository = suc.get_tool_shed_repository_by_shed_name_owner_changeset_revision( trans.app, tool_shed, name, owner, changeset_revision ) + if installed_repository not in installed_repositories: + installed_repositories.append( installed_repository ) + for rd_val in rd_vals: + try: + tool_shed, name, owner, changeset_revision, prior_installation_required = container_util.get_components_from_key( rd_key ) + except: + tool_shed, name, owner, changeset_revision = container_util.get_components_from_key( rd_val ) + installed_repository = suc.get_tool_shed_repository_by_shed_name_owner_changeset_revision( trans.app, tool_shed, name, owner, changeset_revision ) + if installed_repository not in installed_repositories: + installed_repositories.append( installed_repository ) + return installed_repositories + def get_next_prior_install_required_dict_entry( prior_install_required_dict, processed_tsr_ids ): """ The order in which the prior_install_required_dict is processed is critical in order to ensure that the ultimate repository installation order is correctly @@ -142,6 +167,51 @@ prior_install_required_dict[ encoded_repository_id ] = prior_install_ids return prior_install_required_dict +def get_repair_dict( trans, repository ): + """ + Inspect the installed repository dependency hierarchy for a specified repository and attempt to make sure they are all properly installed as well as + each repository's tool dependencies. This method is called only from Galaxy when attempting to correct issues with an installed repository that has + installation problems somewhere in it's dependency hierarchy. + """ + repair_dict = {} + # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. + repository_dependencies_dict = repository_dependency_util.get_repository_dependencies_for_installed_tool_shed_repository( trans, repository ) + # Generate the list of installed repositories from the information contained in the repository_dependencies dictionary. + installed_repositories = get_installed_repositories_from_repository_dependencies( trans, repository_dependencies_dict ) + # Some repositories may have repository dependencies that are required to be installed before the dependent repository, so we'll order the list of + # tsr_ids to ensure all repositories are repaired in the required order. + tsr_ids = [] + repo_info_dicts = [] + for installed_repository in installed_repositories: + tsr_ids.append( trans.security.encode_id( installed_repository.id ) ) + repository_clone_url = suc.generate_clone_url_for_installed_repository( trans.app, installed_repository ) + repository_dependencies = repository_dependency_util.get_repository_dependencies_for_installed_tool_shed_repository( trans, installed_repository ) + metadata = installed_repository.metadata + if metadata: + tool_dependencies = metadata.get( 'tool_dependencies', None ) + else: + tool_dependencies = None + repo_info_dict = create_repo_info_dict( trans=trans, + repository_clone_url=repository_clone_url, + changeset_revision=installed_repository.changeset_revision, + ctx_rev=installed_repository.ctx_rev, + repository_owner=installed_repository.owner, + repository_name=installed_repository.name, + repository=None, + repository_metadata=None, + tool_dependencies=tool_dependencies, + repository_dependencies=repository_dependencies ) + repo_info_dicts.append( repo_info_dict ) + # We don't consider tool_panel_section_keys since the repository database records already exist. + ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = order_components_for_installation( trans, + tsr_ids, + repo_info_dicts, + tool_panel_section_keys=None ) + repair_dict[ 'ordered_tsr_ids' ] = ordered_tsr_ids + repair_dict[ 'ordered_repo_info_dicts' ] = ordered_repo_info_dicts + repair_dict[ 'ordered_tool_panel_section_keys' ] = ordered_tool_panel_section_keys + return repair_dict + def get_repo_info_dict( trans, repository_id, changeset_revision ): repository = suc.get_repository_in_tool_shed( trans, repository_id ) repository_clone_url = suc.generate_clone_url_for_repository_in_tool_shed( trans, repository ) @@ -175,7 +245,7 @@ repository_dependencies=None ) return repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, has_repository_dependencies -def get_repository_components_for_installation( encoded_tsr_id, encoded_tsr_ids, repo_info_dicts, tool_panel_section_keys ): +def get_repository_components_for_installation( encoded_tsr_id, encoded_tsr_ids, repo_info_dicts, tool_panel_section_keys=None ): """ The received encoded_tsr_ids, repo_info_dicts, and tool_panel_section_keys are 3 lists that contain associated elements at each location in the list. This method will return the elements from repo_info_dicts and tool_panel_section_keys associated with the received encoded_tsr_id @@ -184,7 +254,10 @@ for index, tsr_id in enumerate( encoded_tsr_ids ): if tsr_id == encoded_tsr_id: repo_info_dict = repo_info_dicts[ index ] - tool_panel_section_key = tool_panel_section_keys[ index ] + if tool_panel_section_keys: + tool_panel_section_key = tool_panel_section_keys[ index ] + else: + tool_panel_section_key = None return repo_info_dict, tool_panel_section_key return None, None @@ -220,6 +293,26 @@ prior_install_ids.append( encoded_repository_id ) return prior_install_ids +def get_tool_shed_repository_ids( as_string=False, **kwd ): + tsrid = kwd.get( 'tool_shed_repository_id', None ) + tsridslist = util.listify( kwd.get( 'tool_shed_repository_ids', None ) ) + if not tsridslist: + tsridslist = util.listify( kwd.get( 'id', None ) ) + if tsridslist: + if tsrid and tsrid not in tsridslist: + tsridslist.append( tsrid ) + if as_string: + return ','.join( tsridslist ) + return tsridslist + else: + tsridslist = util.listify( kwd.get( 'ordered_tsr_ids', None ) ) + if as_string: + return ','.join( tsridslist ) + return tsridslist + if as_string: + '' + return [] + def get_update_to_changeset_revision_and_ctx_rev( trans, repository ): """Return the changeset revision hash to which the repository can be updated.""" changeset_revision_dict = {} @@ -447,7 +540,7 @@ cloned_ok, error_message = suc.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev ) if cloned_ok: if reinstalling: - # Since we're reinstalling the repository we need to find the latest changeset revision to which is can be updated. + # Since we're reinstalling the repository we need to find the latest changeset revision to which it can be updated. changeset_revision_dict = get_update_to_changeset_revision_and_ctx_rev( trans, tool_shed_repository ) current_changeset_revision = changeset_revision_dict.get( 'changeset_revision', None ) current_ctx_rev = changeset_revision_dict.get( 'ctx_rev', None ) @@ -503,13 +596,13 @@ suc.update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.INSTALLED ) else: # An error occurred while cloning the repository, so reset everything necessary to enable another attempt. - suc.set_repository_attributes( trans, - tool_shed_repository, - status=trans.model.ToolShedRepository.installation_status.ERROR, - error_message=error_message, - deleted=False, - uninstalled=False, - remove_from_disk=True ) + set_repository_attributes( trans, + tool_shed_repository, + status=trans.model.ToolShedRepository.installation_status.ERROR, + error_message=error_message, + deleted=False, + uninstalled=False, + remove_from_disk=True ) def merge_containers_dicts_for_new_install( containers_dicts ): """ @@ -591,7 +684,7 @@ lock.release() return new_containers_dict -def order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys ): +def order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys=None ): """ Some repositories may have repository dependencies that are required to be installed before the dependent repository. This method will inspect the list of repositories about to be installed and make sure to order them appropriately. For each repository about to be installed, if required repositories are not @@ -617,14 +710,19 @@ prior_repo_info_dict, prior_tool_panel_section_key = get_repository_components_for_installation( prior_install_required_id, tsr_ids, repo_info_dicts, - tool_panel_section_keys ) + tool_panel_section_keys=tool_panel_section_keys ) ordered_tsr_ids.append( prior_install_required_id ) ordered_repo_info_dicts.append( prior_repo_info_dict ) - ordered_tool_panel_section_keys.append( prior_tool_panel_section_key ) - repo_info_dict, tool_panel_section_key = get_repository_components_for_installation( tsr_id, tsr_ids, repo_info_dicts, tool_panel_section_keys ) + if tool_panel_section_keys: + ordered_tool_panel_section_keys.append( prior_tool_panel_section_key ) + repo_info_dict, tool_panel_section_key = get_repository_components_for_installation( tsr_id, + tsr_ids, + repo_info_dicts, + tool_panel_section_keys=tool_panel_section_keys ) ordered_tsr_ids.append( tsr_id ) ordered_repo_info_dicts.append( repo_info_dict ) - ordered_tool_panel_section_keys.append( tool_panel_section_key ) + if tool_panel_section_keys: + ordered_tool_panel_section_keys.append( tool_panel_section_key ) return ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys def populate_containers_dict_for_new_install( trans, tool_shed_url, tool_path, readme_files_dict, installed_repository_dependencies, missing_repository_dependencies, @@ -663,3 +761,101 @@ def pull_repository( repo, repository_clone_url, ctx_rev ): """Pull changes from a remote repository to a local one.""" commands.pull( suc.get_configured_ui(), repo, source=repository_clone_url, rev=[ ctx_rev ] ) + +def repair_tool_shed_repository( trans, repository, repo_info_dict ): + + def add_repair_dict_entry( repository_name, error_message ): + if repository_name in repair_dict: + repair_dict[ repository_name ].append( error_message ) + else: + repair_dict[ repository_name ] = [ error_message ] + return repair_dict + + metadata = repository.metadata + repair_dict = {} + if repository.status in [ trans.model.ToolShedRepository.installation_status.DEACTIVATED ]: + try: + common_install_util.activate_repository( trans, repository ) + except Exception, e: + error_message = "Error activating repository %s: %s" % ( repository.name, str( e ) ) + log.debug( error_message ) + repair_dict [ repository.name ] = error_message + elif repository.status not in [ trans.model.ToolShedRepository.installation_status.INSTALLED ]: + # TODO: this may cause problems if the repository is currently being installed. + shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) + # Reset the repository attributes to the New state for installation. + if metadata: + tool_panel_section_key = tool_util.handle_tool_panel_selection( trans, + metadata, + no_changes_checked=True, + tool_panel_section=None, + new_tool_panel_section=None ) + else: + # The tools will be loaded outside of any sections in the tool panel. + tool_panel_section_key = None + set_repository_attributes( trans, + repository, + status=trans.model.ToolShedRepository.installation_status.NEW, + error_message=None, + deleted=False, + uninstalled=False, + remove_from_disk=True ) + install_tool_shed_repository( trans, + repository, + repo_info_dict, + tool_panel_section_key, + shed_tool_conf, + tool_path, + install_tool_dependencies=True, + reinstalling=True ) + if repository.status in [ trans.model.ToolShedRepository.installation_status.ERROR ]: + repair_dict = add_repair_dict_entry( repository.name, repository.error_message ) + else: + # We have an installed tool shed repository, so handle tool dependencies if necessary. + if repository.missing_tool_dependencies and metadata and 'tool_dependencies' in metadata: + work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-itdep" ) + # Reset missing tool dependencies. + for tool_dependency in repository.missing_tool_dependencies: + if tool_dependency.status in [ trans.model.ToolDependency.installation_status.ERROR ]: + tool_dependency_util.set_tool_dependency_attributes( trans, + tool_dependency, + trans.model.ToolDependency.installation_status.UNINSTALLED, + None, + remove_from_disk=True ) + trans.sa_session.refresh( tool_dependency ) + # Install tool dependencies. + suc.update_tool_shed_repository_status( trans.app, + repository, + trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) + # Get the tool_dependencies.xml file from the repository. + tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', repository.repo_path( trans.app ) ) + installed_tool_dependencies = common_install_util.handle_tool_dependencies( app=trans.app, + tool_shed_repository=repository, + tool_dependencies_config=tool_dependencies_config, + tool_dependencies=repository.tool_dependencies ) + for installed_tool_dependency in installed_tool_dependencies: + if installed_tool_dependency.status in [ trans.model.ToolDependency.installation_status.ERROR ]: + repair_dict = add_repair_dict_entry( repository.name, installed_tool_dependency.error_message ) + try: + shutil.rmtree( work_dir ) + except: + pass + suc.update_tool_shed_repository_status( trans.app, repository, trans.model.ToolShedRepository.installation_status.INSTALLED ) + return repair_dict + +def set_repository_attributes( trans, repository, status, error_message, deleted, uninstalled, remove_from_disk=False ): + if remove_from_disk: + relative_install_dir = repository.repo_path( trans.app ) + if relative_install_dir: + clone_dir = os.path.abspath( relative_install_dir ) + try: + shutil.rmtree( clone_dir ) + log.debug( "Removed repository installation directory: %s" % str( clone_dir ) ) + except Exception, e: + log.debug( "Error removing repository installation directory %s: %s" % ( str( clone_dir ), str( e ) ) ) + repository.error_message = error_message + repository.status = status + repository.deleted = deleted + repository.uninstalled = uninstalled + trans.sa_session.add( repository ) + trans.sa_session.flush() diff -r eaa5112fefcd41ce6509bc22791303a155007680 -r 3fa9df444b4b81f94b1c42a033c685a6e23827be lib/tool_shed/util/repository_dependency_util.py --- a/lib/tool_shed/util/repository_dependency_util.py +++ b/lib/tool_shed/util/repository_dependency_util.py @@ -4,6 +4,7 @@ from galaxy.util import json from galaxy.webapps.tool_shed.util import container_util import tool_shed.util.shed_util_common as suc +from tool_shed.util import common_util from tool_shed.util import common_install_util from tool_shed.util import encoding_util from tool_shed.util import metadata_util @@ -285,6 +286,23 @@ # Default prior_installation_required to False. return False +def get_repository_dependencies_for_installed_tool_shed_repository( trans, repository ): + """ + Send a request to the appropriate tool shed to retrieve the dictionary of repository dependencies defined for the received repository which is + installed into Galaxy. This method is called only from Galaxy. + """ + tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed ) + url = suc.url_join( tool_shed_url, + 'repository/get_repository_dependencies?name=%s&owner=%s&changeset_revision=%s' % \ + ( str( repository.name ), str( repository.owner ), str( repository.changeset_revision ) ) ) + raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) + if len( raw_text ) > 2: + encoded_text = json.from_json_string( raw_text ) + text = encoding_util.tool_shed_decode( encoded_text ) + else: + text = '' + return text + def get_repository_dependencies_for_changeset_revision( trans, repository, repository_metadata, toolshed_base_url, key_rd_dicts_to_be_processed=None, all_repository_dependencies=None, handled_key_rd_dicts=None, circular_repository_dependencies=None ): diff -r eaa5112fefcd41ce6509bc22791303a155007680 -r 3fa9df444b4b81f94b1c42a033c685a6e23827be lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -547,6 +547,24 @@ return 'DELETED' return None +def get_ids_of_tool_shed_repositories_being_installed( trans, as_string=False ): + installing_repository_ids = [] + new_status = trans.model.ToolShedRepository.installation_status.NEW + cloning_status = trans.model.ToolShedRepository.installation_status.CLONING + setting_tool_versions_status = trans.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS + installing_dependencies_status = trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES + loading_datatypes_status = trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES + for tool_shed_repository in trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( or_( trans.model.ToolShedRepository.status == new_status, + trans.model.ToolShedRepository.status == cloning_status, + trans.model.ToolShedRepository.status == setting_tool_versions_status, + trans.model.ToolShedRepository.status == installing_dependencies_status, + trans.model.ToolShedRepository.status == loading_datatypes_status ) ): + installing_repository_ids.append( trans.security.encode_id( tool_shed_repository.id ) ) + if as_string: + return ','.join( installing_repository_ids ) + return installing_repository_ids + def get_installed_tool_shed_repository( trans, id ): """Get a tool shed repository record from the Galaxy database defined by the id.""" return trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( id ) ) @@ -1223,20 +1241,6 @@ """Return a reversed list of changesets in the repository changelog up to and including the included_upper_bounds_changeset_revision.""" return reversed_lower_upper_bounded_changelog( repo, INITIAL_CHANGELOG_HASH, included_upper_bounds_changeset_revision ) -def set_repository_attributes( trans, repository, status, error_message, deleted, uninstalled, remove_from_disk=False ): - if remove_from_disk: - relative_install_dir = repository.repo_path( trans.app ) - if relative_install_dir: - clone_dir = os.path.abspath( relative_install_dir ) - shutil.rmtree( clone_dir ) - log.debug( "Removed repository installation directory: %s" % str( clone_dir ) ) - repository.error_message = error_message - repository.status = status - repository.deleted = deleted - repository.uninstalled = uninstalled - trans.sa_session.add( repository ) - trans.sa_session.flush() - def set_prior_installation_required( repository, required_repository ): """Return True if the received required_repository must be installed before the received repository.""" # This method is called only from Galaxy when rendering repository dependencies for an installed tool shed repository. diff -r eaa5112fefcd41ce6509bc22791303a155007680 -r 3fa9df444b4b81f94b1c42a033c685a6e23827be lib/tool_shed/util/tool_dependency_util.py --- a/lib/tool_shed/util/tool_dependency_util.py +++ b/lib/tool_shed/util/tool_dependency_util.py @@ -344,3 +344,12 @@ removed = True error_message = '' return removed, error_message + +def set_tool_dependency_attributes( trans, tool_dependency, status, error_message, remove_from_disk=False ): + if remove_from_disk: + installation_directory = tool_dependency.installation_directory( trans.app ) + removed, err_msg = remove_tool_dependency_installation_directory( installation_directory ) + tool_dependency.error_message = error_message + tool_dependency.status = status + trans.sa_session.add( tool_dependency ) + trans.sa_session.flush() diff -r eaa5112fefcd41ce6509bc22791303a155007680 -r 3fa9df444b4b81f94b1c42a033c685a6e23827be scripts/api/install_tool_shed_repositories.py --- a/scripts/api/install_tool_shed_repositories.py +++ b/scripts/api/install_tool_shed_repositories.py @@ -10,7 +10,7 @@ </section> Here is a working example of how to use this script to install a repository from the test tool shed. -./install_repository_tools.py --api <api key> --local <galaxy base url> --url http://testtoolshed.g2.bx.psu.edu --name gregs_filter --owner greg --revision f28d5018f9cb --tool-deps +./install_tool_shed_repositories.py --api <api key> --local <galaxy base url> --url http://testtoolshed.g2.bx.psu.edu --name gregs_filter --owner greg --revision f28d5018f9cb --tool-deps """ import os @@ -34,7 +34,7 @@ data[ 'install_repository_dependencies' ] = options.install_repository_dependencies if options.install_tool_dependencies: data[ 'install_tool_dependencies' ] = options.install_tool_dependencies - submit( options.api, '%s%s' % ( options.local_url.strip('/'), '/api/tool_shed_repositories/new/install_repository_revision' ), data ) + submit( options.api, '%s%s' % ( options.local_url.strip( '/' ), '/api/tool_shed_repositories/new/install_repository_revision' ), data ) if __name__ == '__main__': parser = argparse.ArgumentParser( description='Installation of tool shed repositories via the Galaxy API.' ) diff -r eaa5112fefcd41ce6509bc22791303a155007680 -r 3fa9df444b4b81f94b1c42a033c685a6e23827be scripts/api/repair_tool_shed_repository.py --- /dev/null +++ b/scripts/api/repair_tool_shed_repository.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +""" +Repair a specified repository revision previously installed into Galaxy. +</section> + +Here is a working example of how to use this script to repair a repository installed into Galaxy. +./repair_tool_shed_repository.py --api <api key> --local <galaxy base url> --url http://testtoolshed.g2.bx.psu.edu --name gregs_filter --owner greg --revision f28d5018f9cb +""" + +import os +import sys +import argparse +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import display +from common import submit + +def clean_url( url ): + if url.find( '//' ) > 0: + # We have an url that includes a protocol, something like: http://localhost:9009 + items = url.split( '//' ) + return items[ 1 ].rstrip( '/' ) + return url.rstrip( '/' ) + +def main( options ): + """Collect all user data and install the tools via the Galaxy API.""" + api_key = options.api + base_galaxy_url = options.local_url.rstrip( '/' ) + base_tool_shed_url = options.tool_shed_url.rstrip( '/' ) + cleaned_tool_shed_url = clean_url( base_tool_shed_url ) + installed_tool_shed_repositories_url = '%s/api/%s' % ( base_galaxy_url, 'tool_shed_repositories' ) + data = {} + data[ 'tool_shed_url' ] = cleaned_tool_shed_url + data[ 'name' ] = options.name + data[ 'owner' ] = options.owner + data[ 'changeset_revision' ] = options.changeset_revision + tool_shed_repository_id = None + installed_tool_shed_repositories = display( api_key, installed_tool_shed_repositories_url, return_formatted=False ) + for installed_tool_shed_repository in installed_tool_shed_repositories: + tool_shed = str( installed_tool_shed_repository[ 'tool_shed' ] ) + name = str( installed_tool_shed_repository[ 'name' ] ) + owner = str( installed_tool_shed_repository[ 'owner' ] ) + changeset_revision = str( installed_tool_shed_repository[ 'changeset_revision' ] ) + if tool_shed == cleaned_tool_shed_url and name == options.name and owner == options.owner and changeset_revision == options.changeset_revision: + tool_shed_repository_id = installed_tool_shed_repository[ 'id' ] + break + if tool_shed_repository_id: + url = '%s%s' % ( base_galaxy_url, '/api/tool_shed_repositories/%s/repair_repository_revision' % str( tool_shed_repository_id ) ) + submit( options.api, url, data ) + else: + print "Invalid tool_shed / name / owner / changeset_revision." + +if __name__ == '__main__': + parser = argparse.ArgumentParser( description='Installation of tool shed repositories via the Galaxy API.' ) + parser.add_argument( "-u", "--url", dest="tool_shed_url", required=True, help="Tool Shed URL" ) + parser.add_argument( "-a", "--api", dest="api", required=True, help="API Key" ) + parser.add_argument( "-l", "--local", dest="local_url", required=True, help="URL of the galaxy instance." ) + parser.add_argument( "-n", "--name", required=True, help="Repository name." ) + parser.add_argument( "-o", "--owner", required=True, help="Repository owner." ) + parser.add_argument( "-r", "--revision", dest="changeset_revision", required=True, help="Repository owner." ) + options = parser.parse_args() + main( options ) diff -r eaa5112fefcd41ce6509bc22791303a155007680 -r 3fa9df444b4b81f94b1c42a033c685a6e23827be templates/admin/tool_shed_repository/repository_actions_menu.mako --- a/templates/admin/tool_shed_repository/repository_actions_menu.mako +++ b/templates/admin/tool_shed_repository/repository_actions_menu.mako @@ -24,6 +24,7 @@ <a class="action-button" target="galaxy_main" href="${h.url_for( controller='admin_toolshed', action='manage_repository', id=trans.security.encode_id( repository.id ) )}">Manage repository</a><a class="action-button" target="galaxy_main" href="${h.url_for( controller='admin_toolshed', action='browse_repository', id=trans.security.encode_id( repository.id ) )}">Browse repository files</a><a class="action-button" target="galaxy_main" href="${h.url_for( controller='admin_toolshed', action='check_for_updates', id=trans.security.encode_id( repository.id ) )}">Get repository updates</a> + <a class="action-button" target="galaxy_main" href="${h.url_for( controller='admin_toolshed', action='repair_repository', id=trans.security.encode_id( repository.id ) )}">Repair repository</a> %if repository.can_reset_metadata: <a class="action-button" target="galaxy_main" href="${h.url_for( controller='admin_toolshed', action='reset_repository_metadata', id=trans.security.encode_id( repository.id ) )}">Reset repository metadata</a> %endif Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.