1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/af60bdf0d4ad/ Changeset: af60bdf0d4ad User: greg Date: 2014-05-23 19:18:06 Summary: Enhance the Tool Shed API as well as the Tool Shed's install and test framework in order to integrate the latter into the Tool Shed's new repository registry. Affected #: 9 files diff -r 1a38b9cf737f9b54a992e973b124daf73cf32acc -r af60bdf0d4add74f225d163eca4438ab7ccc0e52 lib/galaxy/security/validate_user_input.py --- a/lib/galaxy/security/validate_user_input.py +++ b/lib/galaxy/security/validate_user_input.py @@ -1,5 +1,8 @@ +import logging import re +log = logging.getLogger( __name__ ) + VALID_PUBLICNAME_RE = re.compile( "^[a-z0-9\-]+$" ) VALID_PUBLICNAME_SUB = re.compile( "[^a-z0-9\-]" ) # Basic regular expression to check email validity. diff -r 1a38b9cf737f9b54a992e973b124daf73cf32acc -r af60bdf0d4add74f225d163eca4438ab7ccc0e52 lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -8,6 +8,7 @@ from galaxy import web from galaxy.util import json from galaxy.web.base.controller import BaseAPIController +from galaxy.web.base.controller import HTTPBadRequest from galaxy.web.framework.helpers import time_ago import tool_shed.repository_types.util as rt_util import tool_shed.util.shed_util_common as suc @@ -25,6 +26,48 @@ class RepositoriesController( BaseAPIController ): """RESTful controller for interactions with repositories in the Tool Shed.""" + @web.expose_api + def add_repository_registry_entry( self, trans, payload, **kwd ): + """ + POST /api/repository_revisions/add_repository_registry_entry + Adds appropriate entries to the repository registry for the repository defined by the received name and owner. + + :param key: the user's API key + + The following parameters are included in the payload. + :param tool_shed_url (required): the base URL of the Tool Shed containing the Repository + :param name (required): the name of the Repository + :param owner (required): the owner of the Repository + """ + response_dict = {} + if not trans.user_is_admin(): + response_dict[ 'status' ] = 'error' + response_dict[ 'message' ] = "You are not authorized to add entries to this Tool Shed's repository registry." + return response_dict + tool_shed_url = payload.get( 'tool_shed_url', '' ) + if not tool_shed_url: + raise HTTPBadRequest( detail="Missing required parameter 'tool_shed_url'." ) + tool_shed_url = tool_shed_url.rstrip( '/' ) + name = payload.get( 'name', '' ) + if not name: + raise HTTPBadRequest( detail="Missing required parameter 'name'." ) + owner = payload.get( 'owner', '' ) + if not owner: + raise HTTPBadRequest( detail="Missing required parameter 'owner'." ) + repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + if repository is None: + error_message = 'Cannot locate repository with name %s and owner %s,' % ( str( name ), str( owner ) ) + log.debug( error_message ) + response_dict[ 'status' ] = 'error' + response_dict[ 'message' ] = error_message + return response_dict + # Update the repository registry. + trans.app.repository_registry.add_entry( repository ) + response_dict[ 'status' ] = 'ok' + response_dict[ 'message' ] = 'Entries for repository %s owned by %s have been added to the Tool Shed repository registry..' \ + % ( name, owner ) + return response_dict + @web.expose_api_anonymous def get_ordered_installable_revisions( self, trans, name, owner, **kwd ): """ @@ -269,6 +312,48 @@ return repository_dicts @web.expose_api + def remove_repository_registry_entry( self, trans, payload, **kwd ): + """ + POST /api/repository_revisions/add_repository_registry_entry + Removes appropriate entries from the repository registry for the repository defined by the received name and owner. + + :param key: the user's API key + + The following parameters are included in the payload. + :param tool_shed_url (required): the base URL of the Tool Shed containing the Repository + :param name (required): the name of the Repository + :param owner (required): the owner of the Repository + """ + response_dict = {} + if not trans.user_is_admin(): + response_dict[ 'status' ] = 'error' + response_dict[ 'message' ] = "You are not authorized to remove entries from this Tool Shed's repository registry." + return response_dict + tool_shed_url = payload.get( 'tool_shed_url', '' ) + if not tool_shed_url: + raise HTTPBadRequest( detail="Missing required parameter 'tool_shed_url'." ) + tool_shed_url = tool_shed_url.rstrip( '/' ) + name = payload.get( 'name', '' ) + if not name: + raise HTTPBadRequest( detail="Missing required parameter 'name'." ) + owner = payload.get( 'owner', '' ) + if not owner: + raise HTTPBadRequest( detail="Missing required parameter 'owner'." ) + repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + if repository is None: + error_message = 'Cannot locate repository with name %s and owner %s,' % ( str( name ), str( owner ) ) + log.debug( error_message ) + response_dict[ 'status' ] = 'error' + response_dict[ 'message' ] = error_message + return response_dict + # Update the repository registry. + trans.app.repository_registry.remove_entry( repository ) + response_dict[ 'status' ] = 'ok' + response_dict[ 'message' ] = 'Entries for repository %s owned by %s have been removed from the Tool Shed repository registry.' \ + % ( name, owner ) + return response_dict + + @web.expose_api def repository_ids_for_setting_metadata( self, trans, my_writable=False, **kwd ): """ GET /api/repository_ids_for_setting_metadata diff -r 1a38b9cf737f9b54a992e973b124daf73cf32acc -r af60bdf0d4add74f225d163eca4438ab7ccc0e52 lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -232,6 +232,8 @@ """ PUT /api/repository_revisions/{encoded_repository_metadata_id}/{payload} Updates the value of specified columns of the repository_metadata table based on the key / value pairs in payload. + + :param id: the encoded id of the `RepositoryMetadata` object """ repository_metadata_id = kwd.get( 'id', None ) if repository_metadata_id is None: diff -r 1a38b9cf737f9b54a992e973b124daf73cf32acc -r af60bdf0d4add74f225d163eca4438ab7ccc0e52 lib/galaxy/webapps/tool_shed/buildapp.py --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -94,8 +94,10 @@ webapp.mapper.resource( 'repository', 'repositories', controller='repositories', - collection={ 'get_repository_revision_install_info' : 'GET', + collection={ 'add_repository_registry_entry' : 'POST', + 'get_repository_revision_install_info' : 'GET', 'get_ordered_installable_revisions' : 'GET', + 'remove_repository_registry_entry' : 'POST', 'repository_ids_for_setting_metadata' : 'GET', 'reset_metadata_on_repositories' : 'POST', 'reset_metadata_on_repository' : 'POST' }, diff -r 1a38b9cf737f9b54a992e973b124daf73cf32acc -r af60bdf0d4add74f225d163eca4438ab7ccc0e52 lib/tool_shed/repository_registry.py --- a/lib/tool_shed/repository_registry.py +++ b/lib/tool_shed/repository_registry.py @@ -41,28 +41,34 @@ self.load_viewable_repositories_and_suites_by_category() self.load_repository_and_suite_tuples() - def add_entry( self, repository ): - if repository: - is_valid = self.is_valid( repository ) - for rca in repository.categories: - category = rca.category - category_name = str( category.name ) - self.viewable_repositories_and_suites_by_category[ category_name ] += 1 - if is_valid: - self.viewable_valid_repositories_and_suites_by_category[ category_name ] += 1 - if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: - self.viewable_suites_by_category[ category_name ] += 1 + def add_entry( self, repository ): + try: + if repository: + is_valid = self.is_valid( repository ) + certified_level_one_tuple = self.get_certified_level_one_tuple( repository ) + latest_installable_changeset_revision, is_level_one_certified = certified_level_one_tuple + for rca in repository.categories: + category = rca.category + category_name = str( category.name ) + self.viewable_repositories_and_suites_by_category[ category_name ] += 1 if is_valid: - self.viewable_valid_suites_by_category[ category_name ] += 1 - certified_level_one_tuple = self.get_certified_level_one_tuple( repository ) - latest_installable_changeset_revision, is_level_one_certified = certified_level_one_tuple - if is_level_one_certified: - self.certified_level_one_viewable_repositories_and_suites_by_category[ category_name ] += 1 - if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: - self.certified_level_one_viewable_suites_by_category[ category_name ] += 1 - self.load_repository_and_suite_tuple( repository ) - if is_level_one_certified: - self.load_certified_level_one_repository_and_suite_tuple( repository ) + self.viewable_valid_repositories_and_suites_by_category[ category_name ] += 1 + if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: + self.viewable_suites_by_category[ category_name ] += 1 + if is_valid: + self.viewable_valid_suites_by_category[ category_name ] += 1 + if is_level_one_certified: + self.certified_level_one_viewable_repositories_and_suites_by_category[ category_name ] += 1 + if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: + self.certified_level_one_viewable_suites_by_category[ category_name ] += 1 + self.load_repository_and_suite_tuple( repository ) + if is_level_one_certified: + self.load_certified_level_one_repository_and_suite_tuple( repository ) + except Exception, e: + # The viewable repository numbers and the categorized (filtered) lists of repository tuples + # may be slightly skewed, but that is no reason to result in a potential server error. All + # will be corrected at next server start. + log.exception( "Handled error adding entry to repository registry: %s." % str( e ) ) def get_certified_level_one_clause_list( self ): certified_level_one_tuples = [] @@ -242,28 +248,34 @@ if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION ]: self.certified_level_one_viewable_suites_by_category[ category_name ] += 1 - def remove_entry( self, repository ): - if repository: - is_valid = self.is_valid( repository ) - for rca in repository.categories: - category = rca.category - category_name = str( category.name ) - self.viewable_repositories_and_suites_by_category[ category_name ] -= 1 - if is_valid: - self.viewable_valid_repositories_and_suites_by_category[ category_name ] -= 1 - if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: - self.viewable_suites_by_category[ category_name ] -= 1 + def remove_entry( self, repository ): + try: + if repository: + is_valid = self.is_valid( repository ) + certified_level_one_tuple = self.get_certified_level_one_tuple( repository ) + latest_installable_changeset_revision, is_level_one_certified = certified_level_one_tuple + for rca in repository.categories: + category = rca.category + category_name = str( category.name ) + self.viewable_repositories_and_suites_by_category[ category_name ] -= 1 if is_valid: - self.viewable_valid_suites_by_category[ category_name ] -= 1 - certified_level_one_tuple = self.get_certified_level_one_tuple( repository ) - latest_installable_changeset_revision, is_level_one_certified = certified_level_one_tuple - if is_level_one_certified: - self.certified_level_one_viewable_repositories_and_suites_by_category[ category_name ] -= 1 - if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: - self.certified_level_one_viewable_suites_by_category[ category_name ] -= 1 - self.unload_repository_and_suite_tuple( repository ) - if is_level_one_certified: - self.unload_certified_level_one_repository_and_suite_tuple( repository ) + self.viewable_valid_repositories_and_suites_by_category[ category_name ] -= 1 + if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: + self.viewable_suites_by_category[ category_name ] -= 1 + if is_valid: + self.viewable_valid_suites_by_category[ category_name ] -= 1 + if is_level_one_certified: + self.certified_level_one_viewable_repositories_and_suites_by_category[ category_name ] -= 1 + if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: + self.certified_level_one_viewable_suites_by_category[ category_name ] -= 1 + self.unload_repository_and_suite_tuple( repository ) + if is_level_one_certified: + self.unload_certified_level_one_repository_and_suite_tuple( repository ) + except Exception, e: + # The viewable repository numbers and the categorized (filtered) lists of repository tuples + # may be slightly skewed, but that is no reason to result in a potential server error. All + # will be corrected at next server start. + log.exception( "Handled error removing entry from repository registry: %s." % str( e ) ) @property def sa_session( self ): diff -r 1a38b9cf737f9b54a992e973b124daf73cf32acc -r af60bdf0d4add74f225d163eca4438ab7ccc0e52 lib/tool_shed/scripts/api/add_repository_registry_entry.py --- /dev/null +++ b/lib/tool_shed/scripts/api/add_repository_registry_entry.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +""" +Add appropriate entries to the Tool Shed's repository registry for a specified repository. + +Here is a working example of how to use this script. +python ./add_repository_registry_entry.py -a <api key> -u <tool shed url> -n <repository name> -o <repository owner> +""" + +import os +import sys +import argparse +import urllib2 +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +def main( options ): + api_key = options.api_key + if api_key: + if options.tool_shed_url and options.name and options.owner: + base_tool_shed_url = options.tool_shed_url.rstrip( '/' ) + data = {} + data[ 'tool_shed_url' ] = base_tool_shed_url + data[ 'name' ] = options.name + data[ 'owner' ] = options.owner + url = '%s%s' % ( base_tool_shed_url, '/api/repositories/add_repository_registry_entry' ) + response_dict = submit( url, data, api_key=api_key, return_formatted=False ) + print response_dict + else: + print "Invalid tool_shed: ", base_tool_shed_url, " name: ", name, " or owner: ", owner, "." + else: + print "An API key for an admin user in the Tool Shed is required to add entries into the Tool Shed's repository registry." + +if __name__ == '__main__': + parser = argparse.ArgumentParser( description='Add entries into the Tool Shed repository registry for a specified repository.' ) + parser.add_argument( "-a", "--api_key", dest="api_key", required=True, help="API Key for user adding entries into the Tool Shed's repository registry." ) + parser.add_argument( "-u", "--url", dest="tool_shed_url", required=True, help="Tool Shed URL" ) + parser.add_argument( "-n", "--name", dest='name', required=True, help="Repository name." ) + parser.add_argument( "-o", "--owner", dest='owner', required=True, help="Repository owner." ) + options = parser.parse_args() + main( options ) diff -r 1a38b9cf737f9b54a992e973b124daf73cf32acc -r af60bdf0d4add74f225d163eca4438ab7ccc0e52 lib/tool_shed/scripts/api/remove_repository_registry_entry.py --- /dev/null +++ b/lib/tool_shed/scripts/api/remove_repository_registry_entry.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +""" +Remove appropriate entries from the Tool Shed's repository registry for a specified repository. + +Here is a working example of how to use this script. +python ./remove_repository_registry_entry.py -a <api key> -u <tool shed url> -n <repository name> -o <repository owner> +""" + +import os +import sys +import argparse +import urllib2 +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +def main( options ): + api_key = options.api_key + if api_key: + if options.tool_shed_url and options.name and options.owner: + base_tool_shed_url = options.tool_shed_url.rstrip( '/' ) + data = {} + data[ 'tool_shed_url' ] = base_tool_shed_url + data[ 'name' ] = options.name + data[ 'owner' ] = options.owner + url = '%s%s' % ( base_tool_shed_url, '/api/repositories/remove_repository_registry_entry' ) + response_dict = submit( url, data, api_key=api_key, return_formatted=False ) + print response_dict + else: + print "Invalid tool_shed: ", base_tool_shed_url, " name: ", name, " or owner: ", owner, "." + else: + print "An API key for an admin user in the Tool Shed is required to remove entries from the Tool Shed's repository registry." + +if __name__ == '__main__': + parser = argparse.ArgumentParser( description='Remove entries from the Tool Shed repository registry for a specified repository.' ) + parser.add_argument( "-a", "--api_key", dest="api_key", required=True, help="API Key for user removing entries from the Tool Shed's repository registry." ) + parser.add_argument( "-u", "--url", dest="tool_shed_url", required=True, help="Tool Shed URL" ) + parser.add_argument( "-n", "--name", dest='name', required=True, help="Repository name." ) + parser.add_argument( "-o", "--owner", dest='owner', required=True, help="Repository owner." ) + options = parser.parse_args() + main( options ) diff -r 1a38b9cf737f9b54a992e973b124daf73cf32acc -r af60bdf0d4add74f225d163eca4438ab7ccc0e52 test/install_and_test_tool_shed_repositories/base/util.py --- a/test/install_and_test_tool_shed_repositories/base/util.py +++ b/test/install_and_test_tool_shed_repositories/base/util.py @@ -13,6 +13,7 @@ eggs.require( 'mercurial' ) eggs.require( "nose" ) +import json import logging import install_and_test_tool_shed_repositories.base.test_db_util as test_db_util import install_and_test_tool_shed_repositories.functional.test_install_repositories as test_install_repositories @@ -31,12 +32,12 @@ from common import get_latest_downloadable_changeset_revision_via_api from common import get_repository_dict from common import json_from_url +from common import submit from common import update from galaxy.util import asbool from galaxy.util import listify from galaxy.util import unicodify -from galaxy.util.json import to_json_string import galaxy.webapps.tool_shed.model.mapping from nose.plugins import Plugin @@ -691,8 +692,8 @@ else: xml_element = tool_shed for reason_section in xml_element: - reason_text = reason_section.find( 'text' ) - if reason_text: + reason_text = reason_section.find( 'text', None ) + if reason_text is not None: reason = str( reason_text.text ) else: reason = 'No reason provided.' @@ -985,7 +986,7 @@ """ if shed_tools_dict is None: shed_tools_dict = {} - file( galaxy_shed_tools_dict_file, 'w' ).write( to_json_string( shed_tools_dict ) ) + file( galaxy_shed_tools_dict_file, 'w' ).write( json.dumps( shed_tools_dict ) ) def print_install_and_test_results( install_stage_type, install_and_test_statistics_dict, error_message ): "Print statistics for the current test run." @@ -1081,11 +1082,34 @@ owner = repository_dict.get( 'owner', None ) changeset_revision = repository_dict.get( 'changeset_revision', None ) if name is None or owner is None or changeset_revision is None: - print 'Entries for name, owner or changeset_revision missing from repository_dict %s' % repository_dict + print 'Entries for name: ', name, ' owner: ', owner, ' or changeset_revision: ', changeset_revision, \ + ' missing from repository_dict:' % repository_dict else: name = str( name ) owner = str( owner ) changeset_revision = str( changeset_revision ) + # With regard to certification level one, the status of this repository may or may not have changed between + # this install and test run and the previous install and test run. Rather than attempting to determine if + # anything has changed here, we'll let the Tool Shed's repository registry handle the process of proper + # categorization. To enable this, we'll just remove entries from the Tool Shed's repository registry and + # then add them back. This will ensure proper categorization for this repository. + registry_params = dict( tool_shed_url=galaxy_tool_shed_url, name=name, owner=owner ) + print "Removing entries for repository ", name, " owned by ", owner, "from the Tool Shed's repository registry." + url = '%s' % ( common_util.url_join( galaxy_tool_shed_url, 'api', 'repositories', 'remove_repository_registry_entry' ) ) + response_dict = submit( url, registry_params, api_key=tool_shed_api_key, return_formatted=False ) + status = response_dict.get( 'status', 'ok' ) + if status == 'error': + default_message = 'An unknown error occurred attempting to remove entries from the repository registry.' + error_message = response_dict.get( 'message', default_message ) + print error_message + print "Adding entries for repository ", name, " owned by ", owner, "into the Tool Shed's repository registry." + url = '%s' % ( common_util.url_join( galaxy_tool_shed_url, 'api', 'repositories', 'add_repository_registry_entry' ) ) + response_dict = submit( url, registry_params, api_key=tool_shed_api_key, return_formatted=False ) + status = response_dict.get( 'status', 'ok' ) + if status == 'error': + default_message = 'An unknown error occurred attempting to add entries into the repository registry.' + error_message = response_dict.get( 'message', default_message ) + print error_message print '\n=============================================================\n' print 'Inserting the following into tool_test_results for revision %s of repository %s owned by %s:\n%s' % \ ( changeset_revision, name, owner, str( tool_test_results_dict ) ) @@ -1094,7 +1118,7 @@ params[ 'tool_test_results' ] = tool_test_results_dicts # Set the time_last_tested entry so that the repository_metadata.time_last_tested will be set in the tool shed. params[ 'time_last_tested' ] = 'This entry will result in this value being set via the Tool Shed API.' - url = '%s' % ( common_util.url_join( galaxy_tool_shed_url,'api', 'repository_revisions', str( metadata_revision_id ) ) ) + url = '%s' % ( common_util.url_join( galaxy_tool_shed_url, 'api', 'repository_revisions', str( metadata_revision_id ) ) ) print 'url: ', url print 'params: ', params try: diff -r 1a38b9cf737f9b54a992e973b124daf73cf32acc -r af60bdf0d4add74f225d163eca4438ab7ccc0e52 test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py --- a/test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py +++ b/test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py @@ -11,8 +11,10 @@ def do_install( self, repository_dict ): self.logout() - self.login( email='test@bx.psu.edu', username='test' ) - admin_user = test_db_util.get_user( 'test@bx.psu.edu' ) + admin_email = 'test@bx.psu.edu' + admin_username = 'test' + self.login( email=admin_email, username=admin_username ) + admin_user = test_db_util.get_user( admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email admin_user_private_role = test_db_util.get_private_role( admin_user ) # Install the repository through the web interface using twill. The install_repository() method may Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.