1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c07417582dd3/ changeset: c07417582dd3 user: inithello date: 2013-03-11 18:49:12 summary: Improve structure of invalid tools dict. Clarify summary and detailed log output. Add check for required environment variables. affected #: 3 files diff -r a2f58a2b87876f8db9a8703782633b84f4dc13ef -r c07417582dd369a02625d3b3d170718fd69bdd12 install_and_test_tool_shed_repositories.sh --- a/install_and_test_tool_shed_repositories.sh +++ b/install_and_test_tool_shed_repositories.sh @@ -2,5 +2,42 @@ # A good place to look for nose info: http://somethingaboutorange.com/mrl/projects/nose/ -python test/install_and_test_tool_shed_repositories/functional_tests.py $* -v --with-nosehtml --html-report-file ./test/install_and_test_tool_shed_repositories/run_functional_tests.html test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py test/functional/test_toolbox.py +# The test/install_and_test_tool_shed_repositories/functional_tests.py can not be executed directly, because it must have certain functional test definitions +# in sys.argv. Running it through this shell script is the best way to ensure that it has the required definitions. +# This script requires the following environment variables: +# GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY - must be set to the API key for the tool shed that is being checked. +# GALAXY_INSTALL_TEST_TOOL_SHED_URL - must be set to a URL that the tool shed is listening on. +# If the tool shed url is not specified in tool_sheds_conf.xml, GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF must be set to a tool sheds configuration file +# that does specify that url, otherwise repository installation will fail. + +if [ -z $GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY ] ; then + echo "This script requires the GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY environment variable to be set and non-empty." + exit 1 +fi + +if [ -z $GALAXY_INSTALL_TEST_TOOL_SHED_URL ] ; then + echo "This script requires the GALAXY_INSTALL_TEST_TOOL_SHED_URL environment variable to be set and non-empty." + exit 1 +fi + +if [ -z "$GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF" ] ; then + if grep --quiet $GALAXY_INSTALL_TEST_TOOL_SHED_URL tool_sheds_conf.xml; then + echo "Tool sheds configuration tool_sheds_conf.xml ok, proceeding." + else + echo "ERROR: Tool sheds configuration tool_sheds_conf.xml does not have an entry for $GALAXY_INSTALL_TEST_TOOL_SHED_URL." + exit 1 + fi +else + if grep --quiet $GALAXY_INSTALL_TEST_TOOL_SHED_URL $GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF; then + echo "Tool sheds configuration $GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF ok, proceeding." + else + echo "ERROR: Tool sheds configuration $GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF does not have an entry for $GALAXY_INSTALL_TEST_TOOL_SHED_URL" + exit 1 + fi +fi + +python test/install_and_test_tool_shed_repositories/functional_tests.py $* -v --with-nosehtml --html-report-file \ + test/install_and_test_tool_shed_repositories/run_functional_tests.html \ + test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py \ + test/functional/test_toolbox.py diff -r a2f58a2b87876f8db9a8703782633b84f4dc13ef -r c07417582dd369a02625d3b3d170718fd69bdd12 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -55,7 +55,11 @@ type="int", help="Control the amount of detail in the log output. --verbosity=1 is " "the same as -v") ( options, args ) = parser.parse_args() - ini_file = args[0] + try: + ini_file = args[0] + except IndexError: + print "Usage: python %s <tool shed .ini file> [options]" % sys.argv[ 0 ] + exit( 127 ) config_parser = ConfigParser.ConfigParser( {'here':os.getcwd()} ) config_parser.read( ini_file ) config_dict = {} @@ -161,6 +165,8 @@ has_tests = 0 no_tests = 0 no_tools = 0 + valid_revisions = 0 + invalid_revisions = 0 # Get the list of metadata records to check for functional tests and test data. Limit this to records that have not been flagged do_not_test # or tools_functionally_correct. Also filter out changeset revisions that are not downloadable, because it's redundant to test a revision that # a user can't install. @@ -173,6 +179,7 @@ repository_status[ 'invalid_tests' ] = [] metadata_records_to_check = app.sa_session.query( app.model.RepositoryMetadata ) \ .filter( and_( app.model.RepositoryMetadata.table.c.downloadable == True, + app.model.RepositoryMetadata.table.c.includes_tools == True, app.model.RepositoryMetadata.table.c.do_not_test == False, app.model.RepositoryMetadata.table.c.tools_functionally_correct == False ) ) \ .all() @@ -185,87 +192,14 @@ repository_status[ 'invalid_tests' ] = [] if metadata_record.repository.id not in checked_repository_ids: checked_repository_ids.append( metadata_record.repository.id ) - if verbosity >= 1: - print '# Checking for functional tests in changeset revision %s of %s, owned by %s.' % \ - ( changeset_revision, name, owner ) + if verbosity >= 1: + print '# -------------------------------------------------------------------------------------------' + print '# Now checking revision %s of %s, owned by %s.' % ( changeset_revision, name, owner ) # If this changeset revision has no tools, we don't need to do anything here, the install and test script has a filter for returning # only repositories that contain tools. if 'tools' not in metadata_record.metadata: - no_tools += 1 continue else: - # Loop through all the tools in this metadata record, checking each one for defined functional tests. - for tool_metadata in metadata_record.metadata[ 'tools' ]: - tool_count += 1 - tool_id = tool_metadata[ 'id' ] - tool_version = tool_metadata[ 'version' ] - tool_guid = tool_metadata[ 'guid' ] - if verbosity >= 2: - print "# Checking tool ID '%s' in changeset revision %s of %s." % \ - ( tool_id, changeset_revision, name ) - # If there are no tests, this tool should not be tested, since the tool functional tests only report failure if the test itself fails, - # not if it's missing or undefined. Filtering out those repositories at this step will reduce the number of "false negatives" the - # automated functional test framework produces. - if 'tests' not in tool_metadata or not tool_metadata[ 'tests' ]: - if verbosity >= 2: - print '# No functional tests defined for %s.' % tool_id - # The repository_metadata.tool_test_errors attribute should always have the following structure: - # { - # "test_environment": - # { - # "galaxy_revision": "9001:abcd1234", - # "galaxy_database_version": "114", - # "tool_shed_revision": "9001:abcd1234", - # "tool_shed_mercurial_version": "2.3.1", - # "tool_shed_database_version": "17", - # "python_version": "2.7.2", - # "architecture": "x86_64", - # "system": "Darwin 12.2.0" - # }, - # "test_errors": - # [ - # { - # "test_id": "The test ID, generated by twill", - # "tool_id": "The tool ID that was tested", - # "tool_version": "The tool version that was tested", - # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." - # "traceback": "The captured traceback." - # }, - # ] - # "passed_tests": - # [ - # { - # "test_id": "The test ID, generated by twill", - # "tool_id": "The tool ID that was tested", - # "tool_version": "The tool version that was tested", - # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." - # }, - # ] - # "invalid_tests": - # [ - # { - # "tool_id": "The tool ID that does not have functional tests defined.", - # "tool_version": "The version of the tool." - # "tool_guid": "The guid of the tool." - # "reason_test_is_invalid": "A short explanation of what is invalid. - # }, - # ] - # } - # Optionally, "traceback" may be included in a test_errors dict, if it is relevant. No script should overwrite anything other - # than the list relevant to what it is testing. - test_errors = dict( tool_id=tool_id, tool_version=tool_version, tool_guid=tool_guid ) - repository_status[ 'invalid_tests' ].append( test_errors ) - no_tests += 1 - else: - if verbosity >= 2: - print "# Tool ID '%s' in changeset revision %s of %s has one or more valid functional tests defined." % \ - ( tool_id, changeset_revision, name ) - has_tests += 1 - if verbosity >= 1: - if not repository_status[ 'invalid_tests' ]: - print '# All tools have functional tests in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) - else: - print '# Some tools missing functional tests in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) has_test_data = False # Clone the repository up to the changeset revision we're checking. repo_dir = metadata_record.repository.repo_path( app ) @@ -285,9 +219,47 @@ # Remove the cloned repository path. if os.path.exists( work_dir ): shutil.rmtree( work_dir ) - if not has_test_data: - if verbosity >= 1: + if verbosity >= 1: + if not has_test_data: print '# Test data missing in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) + else: + print '# Test data found in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) + print '# Checking for functional tests in changeset revision %s of %s, owned by %s.' % \ + ( changeset_revision, name, owner ) + # Loop through all the tools in this metadata record, checking each one for defined functional tests. + for tool_metadata in metadata_record.metadata[ 'tools' ]: + tool_count += 1 + tool_id = tool_metadata[ 'id' ] + tool_version = tool_metadata[ 'version' ] + tool_guid = tool_metadata[ 'guid' ] + if verbosity >= 2: + print "# Checking tool ID '%s' in changeset revision %s of %s." % \ + ( tool_id, changeset_revision, name ) + # If there are no tests, this tool should not be tested, since the tool functional tests only report failure if the test itself fails, + # not if it's missing or undefined. Filtering out those repositories at this step will reduce the number of "false negatives" the + # automated functional test framework produces. + tool_has_tests = True + if 'tests' not in tool_metadata or not tool_metadata[ 'tests' ]: + tool_has_tests = False + if verbosity >= 2: + print '# No functional tests defined for %s.' % tool_id + no_tests += 1 + else: + tool_has_tests = True + if verbosity >= 2: + print "# Tool ID '%s' in changeset revision %s of %s has one or more valid functional tests defined." % \ + ( tool_id, changeset_revision, name ) + has_tests += 1 + failure_reason = '' + problem_found = False + if not has_test_data: + failure_reason += 'Repository does not have a test-data directory. ' + problem_found = True + if not tool_has_tests: + failure_reason += 'Functional test definitions missing for %s. ' % tool_id + problem_found = True + test_errors = dict( tool_id=tool_id, tool_version=tool_version, tool_guid=tool_guid, + reason_test_is_invalid=failure_reason ) # The repository_metadata.tool_test_errors attribute should always have the following structure: # { # "test_environment": @@ -323,21 +295,28 @@ # "invalid_tests": # [ # { - # "tool_id": "The tool ID that does not have functional tests defined.", + # "tool_id": "The ID of the tool that does not have valid tests.", # "tool_version": "The version of the tool." # "tool_guid": "The guid of the tool." - # "reason_test_is_invalid": "A short explanation of what is invalid. + # "reason_test_is_invalid": "A short explanation of what is invalid." # }, # ] # } + # # Optionally, "traceback" may be included in a test_errors dict, if it is relevant. No script should overwrite anything other # than the list relevant to what it is testing. - test_errors = dict( tool_id=None, tool_version=None, tool_guid=None, - reason_test_is_invalid="Repository %s is missing a test-data directory." % name ) - repository_status[ 'invalid_tests' ].append( test_errors ) + # Only append this error dict if it hasn't already been added. + if problem_found: + if test_errors not in repository_status[ 'invalid_tests' ]: + repository_status[ 'invalid_tests' ].append( test_errors ) + if not repository_status[ 'invalid_tests' ]: + valid_revisions += 1 + if verbosity >= 1: + print '# All tools have functional tests in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) else: + invalid_revisions += 1 if verbosity >= 1: - print '# Test data found in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) + print '# Some tools missing functional tests in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) if not info_only: # If repository_status[ 'test_errors' ] is empty, no issues were found, and we can just update time_last_tested with the platform # on which this script was run. @@ -356,8 +335,10 @@ app.sa_session.add( metadata_record ) app.sa_session.flush() stop = time.time() + print '# -------------------------------------------------------------------------------------------' print '# Checked %d repositories with %d tools in %d changeset revisions.' % ( len( checked_repository_ids ), tool_count, len( metadata_records_to_check ) ) - print '# Skipped %d changeset revisions without tools.' % no_tools + print '# %d revisions found with functional tests and test data for all tools.' % valid_revisions + print '# %d revisions found with one or more tools missing functional tests and/or test data.' % invalid_revisions print '# Found %d tools without functional tests.' % no_tests print '# Found %d tools with functional tests.' % has_tests if info_only: diff -r a2f58a2b87876f8db9a8703782633b84f4dc13ef -r c07417582dd369a02625d3b3d170718fd69bdd12 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -104,11 +104,20 @@ </tables> ''' -# Define a default location to find the list of repositories to check. -galaxy_repository_list = os.environ.get( 'GALAXY_INSTALL_TEST_REPOSITORY_LIST_LOCATIOM', 'repository_list.json' ) -galaxy_tool_shed_url = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_URL', 'http://localhost:9009' ) +# The tool shed url and api key must be set for this script to work correctly. Additionally, if the tool shed url does not +# point to one of the defaults, the GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF needs to point to a tool sheds configuration file +# that contains a definition for that tool shed. + +galaxy_tool_shed_url = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_URL', None ) tool_shed_api_key = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY', None ) -assert tool_shed_api_key is not None, 'Unable to proceed without API key.' + +if tool_shed_api_key is None: + print "This script requires the GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY environment variable to be set and non-empty." + exit( 1 ) + +if galaxy_tool_shed_url is None: + print "This script requires the GALAXY_INSTALL_TEST_TOOL_SHED_URL environment variable to be set and non-empty." + exit( 1 ) if 'GALAXY_INSTALL_TEST_SECRET' not in os.environ: galaxy_encode_secret = 'changethisinproductiontoo' @@ -462,8 +471,7 @@ # Get a list of repositories to test from the tool shed specified in the GALAXY_INSTALL_TEST_TOOL_SHED_URL environment variable. log.info( "Retrieving repositories to install from the URL:\n%s\n" % str( galaxy_tool_shed_url ) ) repositories_to_install = get_repositories_to_install( galaxy_tool_shed_url, source='url' ) - log.info( "Retrieved %d repositories to install..." % len( repositories_to_install ) ) - repositories_tested = len( repositories_to_install ) + log.info( "Retrieved %d repositories from the API." % len( repositories_to_install ) ) for repository_to_install_dict in repositories_to_install: # We need to get some details from the tool shed API, such as repository name and owner, to pass on to the # module that will generate the install methods. @@ -473,13 +481,15 @@ # and therefore do not need to be checked. If they are undeleted, this script will then test them the next time it runs. if repository_info_dict[ 'deleted' ]: log.info( "Skipping revision %s of repository id %s (%s/%s) since the repository is deleted..." % \ - ( repository_info_dict[ 'changeset_revision' ], - repository_info_dict[ 'repository_id' ], + ( repository_to_install_dict[ 'changeset_revision' ], + repository_to_install_dict[ 'repository_id' ], repository_info_dict[ 'owner' ], repository_info_dict[ 'name' ] ) ) continue # Now merge the dict returned from /api/repository_revisions with the detailed dict we just retrieved. detailed_repository_list.append( dict( repository_info_dict.items() + repository_to_install_dict.items() ) ) + repositories_tested = len( detailed_repository_list ) + log.info( 'After removing deleted repositories from the list, %d remain to be tested.' % repositories_tested ) if '-list_repositories' in sys.argv: log.info( "The API returned the following repositories, not counting deleted:" ) for repository_info_dict in detailed_repository_list: @@ -741,15 +751,15 @@ print "# Repository revisions tested: %d" % repositories_tested if repositories_tested > 0: if repositories_passed: - print "# " + print '# ----------------------------------------------------------------------------------' print "# Repositories passed:" show_summary_output( repositories_passed ) if repositories_failed: - print "# " + print '# ----------------------------------------------------------------------------------' print "# Repositories failed:" show_summary_output( repositories_failed ) if repositories_failed_install: - print "# " + print '# ----------------------------------------------------------------------------------' print "# Repositories not installed correctly:" show_summary_output( repositories_failed_install ) print "####################################################################################" Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.