1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/64f283100f96/ Changeset: 64f283100f96 User: greg Date: 2013-10-16 20:19:26 Summary: Enhance the Tool Shed and Galaxy APIs to support resetting metadata on repositories in the Tool Shed or installed into Galaxy respectively. When resetting repository metadata in the Tool Shed, if the API key is associated with an admin user, then setting the my_writable param value to True will restrict resetting metadata to only repositories that are writable by the user in addition to those repositories of type tool_dependency_definition. This param is ignored if the current user is not an admin user, in which case this same restriction is automatic. Also, when resetting metadata in the Tool Shed, repositories of type tool_dependency_definition are reset before repositories of type unrestricted. When setting metadata for installed repositories in Galaxy, neither repository types nor order are considered since these concepts are irrelevant in that environment. Affected #: 10 files diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -1,6 +1,7 @@ import logging from paste.httpexceptions import HTTPBadRequest, HTTPForbidden +from time import strftime from galaxy import util from galaxy import web @@ -10,6 +11,7 @@ from tool_shed.galaxy_install import repository_util from tool_shed.util import common_util from tool_shed.util import encoding_util +from tool_shed.util import metadata_util from tool_shed.util import workflow_util import tool_shed.util.shed_util_common as suc @@ -488,6 +490,49 @@ return tool_shed_repositories @web.expose_api + def reset_metadata_on_installed_repositories( self, trans, payload, **kwd ): + """ + PUT /api/tool_shed_repositories/reset_metadata_on_installed_repositories + + Resets all metadata on all repositories installed into Galaxy in an "orderly fashion". + + :param key: the API key of the Galaxy admin user. + """ + try: + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results = dict( start_time=start_time, + successful_count=0, + unsuccessful_count=0, + repository_status=[] ) + # Make sure the current user's API key proves he is an admin user in this Galaxy instance. + if not trans.user_is_admin(): + raise HTTPForbidden( detail='You are not authorized to reset metadata on repositories installed into this Galaxy instance.' ) + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=False ) + # Now reset metadata on all remaining repositories. + for repository in query: + repository_id = trans.security.encode_id( repository.id ) + try: + invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_installed_repository( trans, repository_id ) + if invalid_file_tups: + message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) + results[ 'unsuccessful_count' ] += 1 + else: + message = "Successfully reset metadata on repository %s" % str( repository.name ) + results[ 'successful_count' ] += 1 + except Exception, e: + message = "Error resetting metadata on repository %s: %s" % ( str( repository.name ), str( e ) ) + results[ 'unsuccessful_count' ] += 1 + results[ 'repository_status' ].append( message ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) + except Exception, e: + message = "Error in the Galaxy tool_shed_repositories API in reset_metadata_on_installed_repositories: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + + @web.expose_api def show( self, trans, id, **kwd ): """ GET /api/tool_shed_repositories/{encoded_tool_shed_repsository_id} diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -187,6 +187,7 @@ 'exported_workflows' : 'GET', 'import_workflow' : 'POST', 'import_workflows' : 'POST' }, + collection={ 'reset_metadata_on_installed_repositories' : 'POST' }, controller='tool_shed_repositories', name_prefix='tool_shed_repository_', path_prefix='/api', diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -1,11 +1,16 @@ import logging +from time import strftime from galaxy.web.framework.helpers import time_ago from galaxy import eggs from galaxy import web from galaxy import util +from galaxy.util import json from galaxy.web.base.controller import BaseAPIController +import tool_shed.repository_types.util as rt_util import tool_shed.util.shed_util_common as suc from tool_shed.galaxy_install import repository_util +from tool_shed.util import metadata_util +from tool_shed.util import tool_util eggs.require( 'mercurial' ) @@ -171,6 +176,69 @@ trans.response.status = 500 return message + @web.expose_api + def reset_metadata_on_repositories( self, trans, payload, **kwd ): + """ + PUT /api/repositories/reset_metadata_on_repositories + + Resets all metadata on all repositories in the Tool Shed in an "orderly fashion". Since there are currently only two + repository types (tool_dependecy_definition and unrestricted), the order in which metadata is reset is repositories of + type tool_dependecy_definition first followed by repositories of type unrestricted, and only one pass is necessary. If + a new repository type is introduced, the process will undoubtedly need to be revisited. To facilitate this order, an + in-memory list of repository ids that have been processed is maintained. + + :param key: the API key of the Tool Shed user. + + The following parameters can optionally be included in the payload. + :param my_writable (optional): if the API key is associated with an admin user in the Tool Shed, setting this param value + to True will restrict resetting metadata to only repositories that are writable by the user + in addition to those repositories of type tool_dependency_definition. This param is ignored + if the current user is not an admin user, in which case this same restriction is automatic. + """ + def handle_repository( trans, repository, results ): + repository_id = trans.security.encode_id( repository.id ) + try: + invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans, repository_id ) + if invalid_file_tups: + message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) + results[ 'unsuccessful_count' ] += 1 + else: + message = "Successfully reset metadata on repository %s" % str( repository.name ) + results[ 'successful_count' ] += 1 + except Exception, e: + message = "Error resetting metadata on repository %s: %s" % ( str( repository.name ), str( e ) ) + results[ 'unsuccessful_count' ] += 1 + results[ 'repository_status' ].append( message ) + return results + try: + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results = dict( start_time=start_time, + repository_status=[], + successful_count=0, + unsuccessful_count=0 ) + handled_repository_ids = [] + if trans.user_is_admin(): + my_writable = util.asbool( payload.get( 'my_writable', False ) ) + else: + my_writable = True + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) + # First reset metadata on all repositories of type repository_dependency_definition. + for repository in query: + if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + results = handle_repository( trans, repository, results ) + # Now reset metadata on all remaining repositories. + for repository in query: + if repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + results = handle_repository( trans, repository, results ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) + except Exception, e: + message = "Error in the Tool Shed repositories API in reset_metadata_on_repositories: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + @web.expose_api_anonymous def show( self, trans, id, **kwd ): """ diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/galaxy/webapps/tool_shed/buildapp.py --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -84,7 +84,8 @@ 'repositories', controller='repositories', collection={ 'get_repository_revision_install_info' : 'GET', - 'get_ordered_installable_revisions' : 'GET' }, + 'get_ordered_installable_revisions' : 'GET', + 'reset_metadata_on_repositories' : 'POST' }, name_prefix='repository_', path_prefix='/api', parent_resources=dict( member_name='repository', collection_name='repositories' ) ) diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/tool_shed/scripts/api/reset_metadata_on_repositories.py --- /dev/null +++ b/lib/tool_shed/scripts/api/reset_metadata_on_repositories.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +""" +Script to reset metadata on certain repositories in the Tool Shed. If the received API key is associated +with an admin user in the Tool Shed, setting the my_writable param value to True will restrict resetting +metadata to only repositories that are writable by the user in addition to those repositories of type +tool_dependency_definition. The my_writable param is ignored if the current user is not an admin user, +in which case this same restriction is automatic. + +usage: reset_metadata_on_repositories.py key <my_writable> + +Here is a working example of how to use this script to reset metadata on certain repositories in a specified Tool Shed. +python ./reset_metadata_on_repositories.py -a 22be3b -m True -u http://localhost:9009/ +""" + +import argparse +import os +import sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +def main( options ): + api_key = options.api + my_writable = options.my_writable + base_tool_shed_url = options.tool_shed_url.rstrip( '/' ) + data = dict( my_writable=my_writable ) + url = '%s/api/repositories/reset_metadata_on_repositories' % base_tool_shed_url + submit( url, data, options.api ) + +if __name__ == '__main__': + parser = argparse.ArgumentParser( description='Reset metadata on certain repositories in the Tool Shed via the Tool Shed API.' ) + parser.add_argument( "-a", "--api", dest="api", required=True, help="API Key" ) + parser.add_argument( "-m", "--my_writable", dest="my_writable", required=False, default='False', help="Restrict to my writable repositories" ) + parser.add_argument( "-u", "--url", dest="tool_shed_url", required=True, help="Tool Shed URL" ) + options = parser.parse_args() + main( options ) diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -1782,10 +1782,10 @@ log.debug( message ) unsuccessful_count += 1 else: - log.debug( "Successfully reset metadata on repository %s" % repository.name ) + log.debug( "Successfully reset metadata on repository %s" % str( repository.name ) ) successful_count += 1 except: - log.exception( "Error attempting to reset metadata on repository %s", repository.name ) + log.exception( "Error attempting to reset metadata on repository %s", str( repository.name ) ) unsuccessful_count += 1 message = "Successfully reset metadata on %d %s. " % ( successful_count, inflector.cond_plural( successful_count, "repository" ) ) if unsuccessful_count: diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -20,6 +20,7 @@ from tool_shed.util import common_util from tool_shed.util import encoding_util from tool_shed.util import xml_util +import tool_shed.repository_types.util as rt_util from xml.etree import ElementTree as XmlET from urllib2 import HTTPError @@ -104,45 +105,15 @@ def build_repository_ids_select_field( trans, name='repository_ids', multiple=True, display='checkboxes', my_writable=False ): """Method called from both Galaxy and the Tool Shed to generate the current list of repositories for resetting metadata.""" repositories_select_field = SelectField( name=name, multiple=multiple, display=display ) - if trans.webapp.name == 'tool_shed': - # We're in the tool shed. - if my_writable: - username = trans.user.username - clause_list = [] - for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.deleted == False ): - allow_push = repository.allow_push( trans.app ) - if allow_push: - allow_push_usernames = allow_push.split( ',' ) - if username in allow_push_usernames: - clause_list.append( trans.model.Repository.table.c.id == repository.id ) - if clause_list: - for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( or_( *clause_list ) ) \ - .order_by( trans.model.Repository.table.c.name, - trans.model.Repository.table.c.user_id ): - owner = repository.user.username - option_label = '%s (%s)' % ( repository.name, owner ) - option_value = '%s' % trans.security.encode_id( repository.id ) - repositories_select_field.add_option( option_label, option_value ) + query = get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=True ) + for repository in query: + if trans.webapp.name == 'tool_shed': + owner = str( repository.user.username ) else: - for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.deleted == False ) \ - .order_by( trans.model.Repository.table.c.name, - trans.model.Repository.table.c.user_id ): - owner = repository.user.username - option_label = '%s (%s)' % ( repository.name, owner ) - option_value = '%s' % trans.security.encode_id( repository.id ) - repositories_select_field.add_option( option_label, option_value ) - else: - # We're in Galaxy. - for repository in trans.sa_session.query( trans.model.ToolShedRepository ) \ - .filter( trans.model.ToolShedRepository.table.c.uninstalled == False ) \ - .order_by( trans.model.ToolShedRepository.table.c.name, - trans.model.ToolShedRepository.table.c.owner ): - option_label = '%s (%s)' % ( repository.name, repository.owner ) - option_value = trans.security.encode_id( repository.id ) - repositories_select_field.add_option( option_label, option_value ) + owner = str( repository.owner ) + option_label = '%s (%s)' % ( str( repository.name ), owner ) + option_value = '%s' % trans.security.encode_id( repository.id ) + repositories_select_field.add_option( option_label, option_value ) return repositories_select_field def build_tool_dependencies_select_field( trans, tool_shed_repository, name, multiple=True, display='checkboxes', uninstalled_only=False ): @@ -836,6 +807,63 @@ prior_import_or_install_required_dict[ encoded_repository_id ] = prior_import_or_install_ids return prior_import_or_install_required_dict +def get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=True ): + """ + Return a query containing repositories for resetting metadata. This method is called from both the Tool Shed and Galaxy. The + my_writable parameter is ignored unless called from the Tool Shed, and the order parameter is used for displaying the list of + repositories ordered alphabetically for display on a page. When called from wither the Tool Shed or Galaxy API, order is False. + """ + if trans.webapp.name == 'tool_shed': + # When called from the Tool Shed API, the metadata is reset on all repositories of type tool_dependency_definition in addition + # to other selected repositories. + if my_writable: + username = trans.user.username + clause_list = [] + for repository in trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.deleted == False ): + allow_push = repository.allow_push( trans.app ) + if not order: + # We've been called from the Tool Shed API, so reset metadata on all repositories of type tool_dependency_definition. + if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION: + clause_list.append( trans.model.Repository.table.c.id == repository.id ) + elif allow_push: + # Include all repositories that are writable by the current user. + allow_push_usernames = allow_push.split( ',' ) + if username in allow_push_usernames: + clause_list.append( trans.model.Repository.table.c.id == repository.id ) + if clause_list: + if order: + return trans.sa_session.query( trans.model.Repository ) \ + .filter( or_( *clause_list ) ) \ + .order_by( trans.model.Repository.table.c.name, + trans.model.Repository.table.c.user_id ) + else: + return trans.sa_session.query( trans.model.Repository ) \ + .filter( or_( *clause_list ) ) + else: + # Return an empty query. + return trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.id == -1 ) + else: + if order: + return trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.deleted == False ) \ + .order_by( trans.model.Repository.table.c.name, + trans.model.Repository.table.c.user_id ) + else: + return trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.deleted == False ) + else: + # We're in Galaxy. + if order: + return trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( trans.model.ToolShedRepository.table.c.uninstalled == False ) \ + .order_by( trans.model.ToolShedRepository.table.c.name, + trans.model.ToolShedRepository.table.c.owner ) + else: + return trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( trans.model.ToolShedRepository.table.c.uninstalled == False ) + def get_repo_info_tuple_contents( repo_info_tuple ): """Take care in handling the repo_info_tuple as it evolves over time as new tool shed features are introduced.""" if len( repo_info_tuple ) == 6: diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 scripts/api/import_workflows_from_installed_tool_shed_repository.py --- a/scripts/api/import_workflows_from_installed_tool_shed_repository.py +++ b/scripts/api/import_workflows_from_installed_tool_shed_repository.py @@ -45,7 +45,7 @@ data = {} # NOTE: to import a single workflow, add an index to data (e.g., # data[ 'index' ] = 0 - # and change the url to be ~/import_workflow (simgular). For example, + # and change the url to be ~/import_workflow (singular). For example, # url = '%s%s' % ( base_galaxy_url, '/api/tool_shed_repositories/%s/import_workflow' % str( tool_shed_repository_id ) ) url = '%s%s' % ( base_galaxy_url, '/api/tool_shed_repositories/%s/import_workflows' % str( tool_shed_repository_id ) ) submit( options.api, url, data ) diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 scripts/api/reset_metadata_on_installed_repositories.py --- /dev/null +++ b/scripts/api/reset_metadata_on_installed_repositories.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +""" +Script to reset metadata on all Tool Shed repositories installed into a Galaxy instance. The received +API key must be associated with a Galaxy admin user. + +usage: reset_metadata_on_installed_repositories.py key + +Here is a working example of how to use this script. +python ./reset_metadata_on_installed_repositories.py -a 22be3b -u http://localhost:8763/ +""" + +import argparse +import os +import sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +def main( options ): + api_key = options.api + base_galaxy_url = options.galaxy_url.rstrip( '/' ) + url = '%s/api/tool_shed_repositories/reset_metadata_on_installed_repositories' % base_galaxy_url + submit( options.api, url, {} ) + +if __name__ == '__main__': + parser = argparse.ArgumentParser( description='Reset metadata on all Tool Shed repositories installed into Galaxy via the Galaxy API.' ) + parser.add_argument( "-a", "--api", dest="api", required=True, help="API Key" ) + parser.add_argument( "-u", "--url", dest="galaxy_url", required=True, help="Galaxy URL" ) + options = parser.parse_args() + main( options ) diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -969,7 +969,7 @@ self.check_for_strings( [ 'Metadata has been reset' ] ) def reset_metadata_on_selected_repositories( self, repository_ids ): - self.visit_url( '/admin/reset_metadata_on_selected_repositories_in_tool_shed' ) + self.visit_url( '/admin/reset_metadata_on_selected_repositories' ) kwd = dict( repository_ids=repository_ids ) self.submit_form( form_no=1, button="reset_metadata_on_selected_repositories_button", **kwd ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.