[galaxy-commits] commit/galaxy-central: greg: Add the ability to purge a "white ghost" tool shed repository from Galaxy.

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7d6f8afcac4d/ Changeset: 7d6f8afcac4d User: greg Date: 2014-06-12 20:16:00 Summary: Add the ability to purge a "white ghost" tool shed repository from Galaxy. Affected #: 6 files diff -r 08fe9352836ee4d18ac3d446c45d84e5a7db9611 -r 7d6f8afcac4d6c208e864c9b8eb35d14d98725cc lib/galaxy/model/tool_shed_install/__init__.py --- a/lib/galaxy/model/tool_shed_install/__init__.py +++ b/lib/galaxy/model/tool_shed_install/__init__.py @@ -238,7 +238,7 @@ def is_deactivated_or_installed( self ): return self.status in [ self.installation_status.DEACTIVATED, self.installation_status.INSTALLED ] - + @property def is_installed( self ): return self.status == self.installation_status.INSTALLED @@ -250,6 +250,10 @@ return False @property + def is_new( self ): + return self.status == self.installation_status.NEW + + @property def missing_repository_dependencies( self ): """Return the repository's repository dependencies that are not currently installed, and may not ever have been installed.""" missing_required_repositories = [] @@ -629,11 +633,11 @@ def to_dict( self, view='element' ): rval = super( ToolVersion, self ).to_dict( view=view ) - rval['tool_name'] = self.tool_id + rval[ 'tool_name' ] = self.tool_id for a in self.parent_tool_association: - rval['parent_tool_id'] = a.parent_id + rval[ 'parent_tool_id' ] = a.parent_id for a in self.child_tool_association: - rval['child_tool_id'] = a.tool_id + rval[ 'child_tool_id' ] = a.tool_id return rval diff -r 08fe9352836ee4d18ac3d446c45d84e5a7db9611 -r 7d6f8afcac4d6c208e864c9b8eb35d14d98725cc lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -45,7 +45,7 @@ def activate_repository( self, trans, **kwd ): """Activate a repository that was deactivated but not uninstalled.""" repository_id = kwd[ 'id' ] - repository = suc.get_installed_tool_shed_repository( trans, repository_id ) + repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) try: common_install_util.activate_repository( trans.app, repository ) except Exception, e: @@ -71,7 +71,7 @@ def browse_repository( self, trans, **kwd ): message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) - repository = suc.get_installed_tool_shed_repository( trans, kwd[ 'id' ] ) + repository = suc.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) return trans.fill_template( '/admin/tool_shed_repository/browse_repository.mako', repository=repository, message=message, @@ -99,8 +99,13 @@ return trans.response.send_redirect( web.url_for( controller='admin_toolshed', action='reset_to_install', **kwd ) ) + if operation == "purge": + kwd[ 'purge_repository' ] = True + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='purge_repository', + **kwd ) ) if operation == "activate or reinstall": - repository = suc.get_installed_tool_shed_repository( trans, kwd[ 'id' ] ) + repository = suc.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) if repository.uninstalled: # Since we're reinstalling the repository we need to find the latest changeset revision to which it can # be updated so that we can reset the metadata if necessary. This will ensure that information about @@ -134,7 +139,7 @@ **kwd ) ) else: message = "Unable to get latest revision for repository <b>%s</b> from " % str( repository.name ) - message += "the Tool Shed, so repository reinstallation is not possible at this time." + message += "the Tool Shed, so repository re-installation is not possible at this time." status = "error" return trans.response.send_redirect( web.url_for( controller='admin_toolshed', action='browse_repositories', @@ -152,6 +157,10 @@ return trans.response.send_redirect( web.url_for( controller='admin_toolshed', action='install_latest_repository_revision', **kwd ) ) + if operation == 'install': + # The user is attempting to install a white ghost. + kwd[ 'status' ] = 'error' + kwd[ 'message' ] = 'It seems you are attempting to install a "white ghost", which should instead be purged.' return self.installed_repository_grid( trans, **kwd ) @web.expose @@ -196,7 +205,7 @@ def check_for_updates( self, trans, **kwd ): """Send a request to the relevant tool shed to see if there are any updates.""" repository_id = kwd.get( 'id', None ) - repository = suc.get_installed_tool_shed_repository( trans, repository_id ) + repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) params = '?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s' % \ ( web.url_for( '/', qualified=True ), @@ -220,7 +229,7 @@ status = kwd.get( 'status', 'done' ) remove_from_disk = kwd.get( 'remove_from_disk', '' ) remove_from_disk_checked = CheckboxField.is_checked( remove_from_disk ) - tool_shed_repository = suc.get_installed_tool_shed_repository( trans, kwd[ 'id' ] ) + tool_shed_repository = suc.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository ) if relative_install_dir: if tool_path: @@ -375,7 +384,7 @@ of the installed tool shed repository in Galaxy. We need it so that we can derive the tool shed from which it was installed. """ - repository = suc.get_installed_tool_shed_repository( trans, repository_id ) + repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) if tool_shed_url is None or repository_name is None or repository_owner is None or changeset_revision is None: message = "Unable to retrieve tool dependencies from the Tool Shed because one or more of the following required " @@ -400,7 +409,7 @@ Send a request to the appropriate tool shed to retrieve the dictionary of information required to reinstall an updated revision of an uninstalled tool shed repository. """ - repository = suc.get_installed_tool_shed_repository( trans, repository_id ) + repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) if tool_shed_url is None or repository_name is None or repository_owner is None or changeset_revision is None: message = "Unable to retrieve updated repository information from the Tool Shed because one or more of the following " @@ -495,7 +504,7 @@ status = kwd.get( 'status', 'done' ) repository_id = kwd.get( 'id', None ) if repository_id is not None: - repository = suc.get_installed_tool_shed_repository( trans, repository_id ) + repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) if repository is not None: tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) name = str( repository.name ) @@ -553,7 +562,7 @@ dependencies are included in the updated revision. """ updating_repository_id = kwd.get( 'updating_repository_id', None ) - repository = suc.get_installed_tool_shed_repository( trans, updating_repository_id ) + repository = suc.get_installed_tool_shed_repository( trans.app, updating_repository_id ) # All received dependencies need to be installed - confirmed by the caller. encoded_tool_dependencies_dict = kwd.get( 'encoded_tool_dependencies_dict', None ) if encoded_tool_dependencies_dict is not None: @@ -730,7 +739,7 @@ if repository_id is None: return trans.show_error_message( 'Missing required encoded repository id.' ) operation = kwd.get( 'operation', None ) - repository = suc.get_installed_tool_shed_repository( trans, repository_id ) + repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) if repository is None: return trans.show_error_message( 'Invalid repository specified.' ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) @@ -1254,6 +1263,32 @@ @web.expose @web.require_admin + def purge_repository( self, trans, **kwd ): + """Purge a "white ghost" repository from the database.""" + repository_id = kwd.get( 'id', None ) + new_kwd = {} + if repository_id is not None: + repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + if repository: + if kwd.get( 'purge_repository', False ): + purge_status, purge_message = repository_util.purge_repository( trans.app, repository ) + if purge_status == 'ok': + new_kwd[ 'status' ] = "done" + else: + new_kwd[ 'status' ] = 'error' + new_kwd[ 'message' ] = purge_message + else: + new_kwd[ 'status' ] = 'error' + new_kwd[ 'message' ] = 'Cannot locate the database record for the repository with encoded id %s.' % str( repository_id ) + else: + new_kwd[ 'status' ] = 'error' + new_kwd[ 'message' ] = 'Invalid repository id value "None" received for repository to be purged.' + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='browse_repositories', + **new_kwd ) ) + + @web.expose + @web.require_admin def reinstall_repository( self, trans, **kwd ): """ Reinstall a tool shed repository that has been previously uninstalled, making sure to handle all repository @@ -1262,7 +1297,7 @@ message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) repository_id = kwd[ 'id' ] - tool_shed_repository = suc.get_installed_tool_shed_repository( trans, repository_id ) + tool_shed_repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) no_changes = kwd.get( 'no_changes', '' ) no_changes_checked = CheckboxField.is_checked( no_changes ) install_repository_dependencies = CheckboxField.is_checked( kwd.get( 'install_repository_dependencies', '' ) ) @@ -1407,7 +1442,7 @@ action='browse_repositories', message=message, status=status ) ) - tool_shed_repository = suc.get_installed_tool_shed_repository( trans, repository_id ) + tool_shed_repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) if kwd.get( 'repair_repository_button', False ): encoded_repair_dict = kwd.get( 'repair_dict', None ) if encoded_repair_dict: @@ -1424,7 +1459,7 @@ repository = trans.install_model.context.query( trans.install_model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) repositories_for_repair.append( repository ) return self.repair_tool_shed_repositories( trans, repositories_for_repair, ordered_repo_info_dicts ) - tool_shed_repository = suc.get_installed_tool_shed_repository( trans, repository_id ) + tool_shed_repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) repair_dict = repository_util.get_repair_dict( trans, tool_shed_repository ) encoded_repair_dict = encoding_util.tool_shed_encode( repair_dict ) ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] ) @@ -1486,7 +1521,7 @@ repository_id = kwd.get( 'id', None ) latest_changeset_revision = kwd.get( 'latest_changeset_revision', None ) latest_ctx_rev = kwd.get( 'latest_ctx_rev', None ) - tool_shed_repository = suc.get_installed_tool_shed_repository( trans, repository_id ) + tool_shed_repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) repository_clone_url = common_util.generate_clone_url_for_installed_repository( trans.app, tool_shed_repository ) metadata = tool_shed_repository.metadata tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( tool_shed_repository.tool_shed ) ) @@ -1672,7 +1707,7 @@ @web.require_admin def reset_repository_metadata( self, trans, id ): """Reset all metadata on a single installed tool shed repository.""" - repository = suc.get_installed_tool_shed_repository( trans, id ) + repository = suc.get_installed_tool_shed_repository( trans.app, id ) repository_clone_url = common_util.generate_clone_url_for_installed_repository( trans.app, repository ) tool_path, relative_install_dir = repository.get_tool_relative_path( trans.app ) if relative_install_dir: @@ -1711,7 +1746,7 @@ @web.require_admin def reset_to_install( self, trans, **kwd ): """An error occurred while cloning the repository, so reset everything necessary to enable another attempt.""" - repository = suc.get_installed_tool_shed_repository( trans, kwd[ 'id' ] ) + repository = suc.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) if kwd.get( 'reset_repository', False ): repository_util.set_repository_attributes( trans.app, repository, @@ -1721,7 +1756,7 @@ uninstalled=False, remove_from_disk=True ) new_kwd = {} - new_kwd[ 'message' ] = "You can now attempt to install the repository named <b>%s</b> again." % repository.name + new_kwd[ 'message' ] = "You can now attempt to install the repository named <b>%s</b> again." % str( repository.name ) new_kwd[ 'status' ] = "done" return trans.response.send_redirect( web.url_for( controller='admin_toolshed', action='browse_repositories', @@ -1737,7 +1772,7 @@ Get the tool_versions from the tool shed for each tool in the installed revision of a selected tool shed repository and update the metadata for the repository's revision in the Galaxy database. """ - repository = suc.get_installed_tool_shed_repository( trans, kwd[ 'id' ] ) + repository = suc.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) params = '?name=%s&owner=%s&changeset_revision=%s' % ( str( repository.name ), str( repository.owner ), @@ -2052,7 +2087,7 @@ def view_tool_metadata( self, trans, repository_id, tool_id, **kwd ): message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) - repository = suc.get_installed_tool_shed_repository( trans, repository_id ) + repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) repository_metadata = repository.metadata shed_config_dict = repository.get_shed_config_dict( trans.app ) tool_metadata = {} diff -r 08fe9352836ee4d18ac3d446c45d84e5a7db9611 -r 7d6f8afcac4d6c208e864c9b8eb35d14d98725cc lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py --- a/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py +++ b/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py @@ -214,7 +214,14 @@ target=None, url_args=dict( controller='admin_toolshed', action='browse_repositories', - operation='activate or reinstall' ) ) ] + operation='activate or reinstall' ) ), + grids.GridOperation( label="Purge", + condition=( lambda item: item.is_new ), + allow_multiple=False, + target=None, + url_args=dict( controller='admin_toolshed', + action='browse_repositories', + operation='purge' ) ) ] standard_filters = [] default_filter = dict( deleted="False" ) num_rows_per_page = 50 diff -r 08fe9352836ee4d18ac3d446c45d84e5a7db9611 -r 7d6f8afcac4d6c208e864c9b8eb35d14d98725cc lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -976,6 +976,117 @@ """Pull changes from a remote repository to a local one.""" commands.pull( hg_util.get_configured_ui(), repo, source=repository_clone_url, rev=[ ctx_rev ] ) +def purge_repository( app, repository ): + """Purge a repository with status New (a white ghost) from the database.""" + sa_session = app.model.context.current + status = 'ok' + message = '' + purged_tool_versions = 0 + purged_tool_dependencies = 0 + purged_required_repositories = 0 + purged_orphan_repository_repository_dependency_association_records = 0 + purged_orphan_repository_dependency_records = 0 + if repository.is_new: + # Purge this repository's associated tool versions. + if repository.tool_versions: + for tool_version in repository.tool_versions: + try: + tool_version_association = tool_version.parent_tool_association + sa_session.delete( tool_version_association ) + sa_session.flush() + except Exception, e: + status = 'error' + message = 'Error attempting to purge tool_versions for the repository named %s with status %s: %s.' % \ + ( str( repository.name ), str( repository.status ), str( e ) ) + return status, message + try: + tool_version_association = tool_version.child_tool_association + sa_session.delete( tool_version_association ) + sa_session.flush() + except Exception, e: + status = 'error' + message = 'Error attempting to purge tool_versions for the repository named %s with status %s: %s.' % \ + ( str( repository.name ), str( repository.status ), str( e ) ) + return status, message + try: + sa_session.delete( tool_version ) + sa_session.flush() + purged_tool_versions += 1 + except Exception, e: + status = 'error' + message = 'Error attempting to purge tool_versions for the repository named %s with status %s: %s.' % \ + ( str( repository.name ), str( repository.status ), str( e ) ) + return status, message + # Purge this repository's associated tool dependencies. + if repository.tool_dependencies: + for tool_dependency in repository.tool_dependencies: + try: + sa_session.delete( tool_dependency ) + sa_session.flush() + purged_tool_dependencies += 1 + except Exception, e: + status = 'error' + message = 'Error attempting to purge tool_dependencies for the repository named %s with status %s: %s.' % \ + ( str( repository.name ), str( repository.status ), str( e ) ) + return status, message + # Purge this repository's associated required repositories. + if repository.required_repositories: + for rrda in repository.required_repositories: + try: + sa_session.delete( rrda ) + sa_session.flush() + purged_required_repositories += 1 + except Exception, e: + status = 'error' + message = 'Error attempting to purge required_repositories for the repository named %s with status %s: %s.' % \ + ( str( repository.name ), str( repository.status ), str( e ) ) + return status, message + # Purge any "orphan" repository_dependency records associated with the repository, but not with any + # repository_repository_dependency_association records. + for orphan_repository_dependency in \ + sa_session.query( app.install_model.RepositoryDependency ) \ + .filter( app.install_model.RepositoryDependency.table.c.tool_shed_repository_id == repository.id ): + # Purge any repository_repository_dependency_association records whose repository_dependency_id is + # the id of the orphan repository_dependency record. + for orphan_rrda in \ + sa_session.query( app.install_model.RepositoryRepositoryDependencyAssociation ) \ + .filter( app.install_model.RepositoryRepositoryDependencyAssociation.table.c.repository_dependency_id == orphan_repository_dependency.id ): + try: + sa_session.delete( orphan_rrda ) + sa_session.flush() + purged_orphan_repository_repository_dependency_association_records += 1 + except Exception, e: + status = 'error' + message = 'Error attempting to purge repository_repository_dependency_association records associated with ' + message += 'an orphan repository_dependency record for the repository named %s with status %s: %s.' % \ + ( str( repository.name ), str( repository.status ), str( e ) ) + return status, message + try: + sa_session.delete( orphan_repository_dependency ) + sa_session.flush() + purged_orphan_repository_dependency_records += 1 + except Exception, e: + status = 'error' + message = 'Error attempting to purge orphan repository_dependency records for the repository named %s with status %s: %s.' % \ + ( str( repository.name ), str( repository.status ), str( e ) ) + return status, message + # Purge the repository. + sa_session.delete( repository ) + sa_session.flush() + message = 'The repository named <b>%s</b> with status <b>%s</b> has been purged.<br/>' % \ + ( str( repository.name ), str( repository.status ) ) + message += 'Total associated tool_version records purged: %d<br/>' % purged_tool_versions + message += 'Total associated tool_dependency records purged: %d<br/>' % purged_tool_dependencies + message += 'Total associated repository_repository_dependency_association records purged: %d<br/>' % purged_required_repositories + message += 'Total associated orphan repository_repository_dependency_association records purged: %d<br/>' % \ + purged_orphan_repository_repository_dependency_association_records + message += 'Total associated orphan repository_dependency records purged: %d<br/>' % purged_orphan_repository_dependency_records + else: + status = 'error' + message = 'A repository must have the status <b>New</b> in order to be purged. This repository has ' + message += ' the status %s.' % str( repository.status ) + return status, message + def repair_tool_shed_repository( trans, repository, repo_info_dict ): def add_repair_dict_entry( repository_name, error_message ): diff -r 08fe9352836ee4d18ac3d446c45d84e5a7db9611 -r 7d6f8afcac4d6c208e864c9b8eb35d14d98725cc lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -1696,7 +1696,7 @@ """Reset all metadata on a single tool shed repository installed into a Galaxy instance.""" invalid_file_tups = [] metadata_dict = {} - repository = suc.get_installed_tool_shed_repository( trans, id ) + repository = suc.get_installed_tool_shed_repository( trans.app, id ) repository_clone_url = common_util.generate_clone_url_for_installed_repository( trans.app, repository ) tool_path, relative_install_dir = repository.get_tool_relative_path( trans.app ) if relative_install_dir: @@ -1886,7 +1886,7 @@ invalid_file_tups, metadata_dict = reset_all_metadata_on_repository_in_tool_shed( trans, repository_id ) else: # We're in Galaxy. - repository = suc.get_installed_tool_shed_repository( trans, repository_id ) + repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) owner = str( repository.owner ) invalid_file_tups, metadata_dict = reset_all_metadata_on_installed_repository( trans, repository_id ) if invalid_file_tups: diff -r 08fe9352836ee4d18ac3d446c45d84e5a7db9611 -r 7d6f8afcac4d6c208e864c9b8eb35d14d98725cc lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -491,9 +491,9 @@ return ','.join( installing_repository_ids ) return installing_repository_ids -def get_installed_tool_shed_repository( trans, id ): +def get_installed_tool_shed_repository( app, id ): """Get a tool shed repository record from the Galaxy database defined by the id.""" - return trans.install_model.context.query( trans.install_model.ToolShedRepository ).get( trans.security.decode_id( id ) ) + return app.install_model.context.query( app.install_model.ToolShedRepository ).get( app.security.decode_id( id ) ) def get_latest_changeset_revision( app, repository, repo ): repository_tip = repository.tip( app ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.