1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7677b8c3ef43/ changeset: 7677b8c3ef43 user: inithello date: 2013-03-06 22:07:32 summary: Added more information to the tool test environment dict. Restrict when do_not_test flag is set. Made test logging format more structured. Moved some utility code to the util module. affected #: 3 files diff -r 48dad8860ee1b1196a1f242c9dd18d218e6db573 -r 7677b8c3ef43d06eda783f1c08ec521d2bd625e5 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -12,9 +12,9 @@ from galaxy import eggs import pkg_resources -pkg_resources.require( "SQLAlchemy >= 0.4" ) -pkg_resources.require( 'mercurial' ) -from mercurial import hg, ui, commands +eggs.require( "SQLAlchemy >= 0.4" ) +eggs.require( 'mercurial' ) +from mercurial import hg, ui, commands, __version__ import time, ConfigParser, shutil from datetime import datetime, timedelta @@ -24,12 +24,13 @@ import galaxy.webapps.tool_shed.config as tool_shed_config import galaxy.webapps.tool_shed.model.mapping import sqlalchemy as sa -from galaxy.model.orm import and_, not_, distinct +from galaxy.model.orm import and_, not_ from galaxy.util.json import from_json_string, to_json_string from galaxy.web import url_for +from galaxy.tools import parameters from tool_shed.util.shed_util_common import clone_repository, get_configured_ui -from base.util import get_test_environment +from base.util import get_test_environment, get_database_version, get_repository_current_revision assert sys.version_info[:2] >= ( 2, 4 ) @@ -50,6 +51,7 @@ now = strftime( "%Y-%m-%d %H:%M:%S" ) print "#############################################################################" print "# %s - Checking repositories for tools with functional tests." % now + print "# This tool shed is configured to listen on %s:%s." % ( config_parser.get( 'server:main', 'host' ), config_parser.get( 'server:main', 'port' ) ) app = FlagRepositoriesApplication( config ) if options.info_only: @@ -113,7 +115,7 @@ } ''' start = time.time() - repository_ids_to_check = [] + checked_repository_ids = [] tool_count = 0 has_tests = 0 no_tests = 0 @@ -121,25 +123,31 @@ # Get the list of metadata records to check for functional tests and test data. Limit this to records that have not been flagged do_not_test # or tools_functionally_correct. Also filter out changeset revisions that are not downloadable, because it's redundant to test a revision that # a user can't install. + # Initialize the repository_status dict with the test environment, but leave the test_errors empty. + repository_status = {} + repository_status[ 'test_environment' ] = get_test_environment() + repository_status[ 'test_environment' ][ 'tool_shed_database_version' ] = get_database_version( app ) + repository_status[ 'test_environment' ][ 'tool_shed_mercurial_version' ] = __version__.version + repository_status[ 'test_environment' ][ 'tool_shed_revision' ] = get_repository_current_revision( os.getcwd() ) + repository_status[ 'test_errors' ] = [] metadata_records_to_check = app.sa_session.query( app.model.RepositoryMetadata ) \ .filter( and_( app.model.RepositoryMetadata.table.c.downloadable == True, app.model.RepositoryMetadata.table.c.do_not_test == False, app.model.RepositoryMetadata.table.c.tools_functionally_correct == False ) ) \ .all() for metadata_record in metadata_records_to_check: + repository_status[ 'test_errors' ] = [] name = metadata_record.repository.name owner = metadata_record.repository.user.username changeset_revision = str( metadata_record.changeset_revision ) - repository_status = {} + if metadata_record.repository.id not in checked_repository_ids: + checked_repository_ids.append( metadata_record.repository.id ) # If this changeset revision has no tools, we don't need to do anything here, the install and test script has a filter for returning # only repositories that contain tools. if 'tools' not in metadata_record.metadata: no_tools += 1 continue else: - # Initialize the repository_status dict with the test environment, but leave the test_errors empty. - repository_status[ 'test_environment' ] = get_test_environment() - repository_status[ 'test_errors' ] = [] # Loop through all the tools in this metadata record, checking each one for defined functional tests. for tool_metadata in metadata_record.metadata[ 'tools' ]: tool_count += 1 @@ -153,12 +161,9 @@ if 'tests' not in tool_metadata or not tool_metadata[ 'tests' ]: if verbose: print '# No functional tests defined for %s.' % tool_id - if 'test_errors' not in repository_status: - repository_status[ 'test_errors' ] = [] - test_id = 'Functional tests for %s' % tool_id # The repository_metadata.tool_test_errors attribute should always have the following structure: # { - # "environment": + # "test_environment": # { # "python_version": "2.7.2", # "architecture": "x86_64", @@ -168,15 +173,16 @@ # [ # { # "test_id": "Something that will easily identify what the problem is", - # "stdout": "The output of the test, or a more detailed description of what was tested and why it failed." + # "stderr": "The output of the test, or a more detailed description of what was tested and why it failed." # }, # ] # } - # Optionally, "stderr" and "traceback" may be included in a test_errors dict, if they are relevant. - test_errors = dict( test_id=test_id, - stdout='No functional tests defined in changeset revision %s of repository %s owned by %s.' % \ - ( changeset_revision, name, owner ) ) + # Optionally, "stdout" and "traceback" may be included in a test_errors dict, if they are relevant. + test_id = 'Functional tests for %s' % tool_id + test_errors = dict( stderr='No functional tests defined for tool %s in changeset revision %s of repository %s owned by %s.' % \ + ( tool_id, changeset_revision, name, owner ) ) repository_status[ 'test_errors' ].append( test_errors ) + repository_status[ 'status' ] = 'failed' no_tests += 1 else: has_tests += 1 @@ -201,14 +207,12 @@ if 'test-data' in dirs: has_test_data = True break - # Remove the cloned path. + # Remove the cloned repository path. if os.path.exists( work_dir ): shutil.rmtree( work_dir ) if not has_test_data: if verbose: print '# Test data missing in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) - repository_status[ 'test_environment' ] = get_test_environment() - test_id = 'Find functional test data for %s' % metadata_record.repository.name # The repository_metadata.tool_test_errors attribute should always have the following structure: # { # "test_environment": @@ -221,28 +225,40 @@ # [ # { # "test_id": "Something that will easily identify what the problem is", - # "stdout": "The output of the test, or a more detailed description of what was tested and why it failed." + # "stderr": "The output of the test, or a more detailed description of what was tested and why it failed." # }, # ] # } - # Optionally, "stderr" and "traceback" may be included in a test_errors dict, if they are relevant. - test_errors = dict( test_id=test_id, - stdout='No test data found for changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) ) + # Optionally, "stdout" and "traceback" may be included in a test_errors dict, if they are relevant. + test_id = 'Find functional test data for %s' % metadata_record.repository.name + test_errors = dict( stderr='No test data found for changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) ) repository_status[ 'test_errors' ].append( test_errors ) + repository_status[ 'status' ] = 'failed' else: if verbose: print '# Test data found in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) if not info_only: + # If repository_status[ 'test_errors' ] is empty, no issues were found, and we can just update time_last_tested with the platform + # on which this script was run. if repository_status[ 'test_errors' ]: - metadata_record.do_not_test = True + # If functional test definitions or test data are missing, set do_not_test = True if and only if: + # a) There are multiple downloadable revisions, and the revision being tested is not the most recent downloadable revision. In this case, + # the revision will never be updated with correct data, and re-testing it would be redundant. + # b) There are one or more downloadable revisions, and the revision being tested is the most recent downloadable revision. In this case, if + # the repository is updated with test data or functional tests, the downloadable changeset revision that was tested will be replaced + # with the new changeset revision, which will be automatically tested. + if should_set_do_not_test_flag( app, metadata_record.repository, changeset_revision ): + metadata_record.do_not_test = True metadata_record.tools_functionally_correct = False - metadata_record.tool_test_errors = to_json_string( repository_status ) + else: + repository_status[ 'status' ] = 'passed' + metadata_record.tool_test_errors = repository_status metadata_record.time_last_tested = datetime.utcnow() app.sa_session.add( metadata_record ) app.sa_session.flush() stop = time.time() - print '# Checked %d tools in %d changeset revisions.' % ( tool_count, len( metadata_records_to_check ) ) - print '# Found %d changeset revisions without tools.' % no_tools + print '# Checked %d repositories with %d tools in %d changeset revisions.' % ( len( checked_repository_ids ), tool_count, len( metadata_records_to_check ) ) + print '# Skipped %d changeset revisions without tools.' % no_tools print '# Found %d tools without functional tests.' % no_tests print '# Found %d tools with functional tests.' % has_tests if info_only: @@ -250,6 +266,52 @@ print "# Elapsed time: ", stop - start print "#############################################################################" +def get_repo_changelog_tuples( repo_path ): + repo = hg.repository( ui.ui(), repo_path ) + changelog_tuples = [] + for changeset in repo.changelog: + ctx = repo.changectx( changeset ) + changelog_tuples.append( ( ctx.rev(), str( ctx ) ) ) + return changelog_tuples + +def is_most_recent_downloadable_revision( app, repository, changeset_revision, downloadable_revisions ): + # Get a list of ( numeric revision, changeset hash ) tuples from the changelog. + changelog = get_repo_changelog_tuples( repository.repo_path( app ) ) + latest_downloadable_revision = None + for ctx_rev, changeset_hash in changelog: + if changeset_hash in downloadable_revisions: + # The last changeset hash in the changelog that is present in the list of downloadable revisions will always be the most + # recent downloadable revision, since the changelog tuples are ordered from earliest to most recent. + latest_downloadable_revision = changeset_hash + if latest_downloadable_revision == changeset_revision: + return True + return False + +def should_set_do_not_test_flag( app, repository, changeset_revision ): + ''' + Returns True if: + a) There are multiple downloadable revisions, and the provided changeset revision is not the most recent downloadable revision. In this case, + the revision will never be updated with correct data, and re-testing it would be redundant. + b) There are one or more downloadable revisions, and the provided changeset revision is the most recent downloadable revision. In this case, if + the repository is updated with test data or functional tests, the downloadable changeset revision that was tested will be replaced + with the new changeset revision, which will be automatically tested. + ''' + metadata_records = app.sa_session.query( app.model.RepositoryMetadata ) \ + .filter( and_( app.model.RepositoryMetadata.table.c.downloadable == True, + app.model.RepositoryMetadata.table.c.repository_id == repository.id ) ) \ + .all() + downloadable_revisions = [ metadata_record.changeset_revision for metadata_record in metadata_records ] + is_latest_revision = is_most_recent_downloadable_revision( app, repository, changeset_revision, downloadable_revisions ) + if len( downloadable_revisions ) == 1: + return True + elif len( downloadable_revisions ) > 1 and is_latest_revision: + return True + elif len( downloadable_revisions ) > 1 and not is_latest_revision: + return True + else: + return False + + class FlagRepositoriesApplication( object ): """Encapsulates the state of a Universe application""" def __init__( self, config ): diff -r 48dad8860ee1b1196a1f242c9dd18d218e6db573 -r 7677b8c3ef43d06eda783f1c08ec521d2bd625e5 test/base/util.py --- a/test/base/util.py +++ b/test/base/util.py @@ -12,6 +12,44 @@ sys.path = new_path from galaxy.util import parse_xml +from galaxy import eggs + +eggs.require( 'mercurial' ) + +from mercurial import hg, ui, commands + +def get_repository_current_revision( repo_path ): + ''' + This method uses the python mercurial API to get the current working directory's mercurial changeset hash. Note that if the author of mercurial + changes the API, this method will have to be updated or replaced. + ''' + # Initialize a mercurial repo object from the provided path. + repo = hg.repository( ui.ui(), repo_path ) + # Get the working directory's change context. + ctx = repo[ None ] + # Extract the changeset hash of the first parent of that change context (the most recent changeset to which the working directory was updated). + changectx = ctx.parents()[ 0 ] + # Also get the numeric revision, so we can return the customary id:hash changeset identifiers. + ctx_rev = changectx.rev() + hg_id = '%d:%s' % ( ctx_rev, str( changectx ) ) + return hg_id + +def get_database_version( app ): + ''' + This method returns the value of the version column from the migrate_version table, using the provided app's SQLAlchemy session to determine + which table to get that from. This way, it's provided with an instance of a Galaxy UniverseApplication, it will return the Galaxy instance's + database migration version. If a tool shed UniverseApplication is provided, it returns the tool shed's database migration version. + ''' + sa_session = app.model.context.current + result = sa_session.execute( 'SELECT version FROM migrate_version LIMIT 1' ) + # This query will return the following structure: + # row = [ column 0, column 1, ..., column n ] + # rows = [ row 0, row 1, ..., row n ] + # The first column in the first row is the version number we want. + for row in result: + version = row[ 0 ] + break + return version def get_installed_repository_info( elem, last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision, tool_path ): """ diff -r 48dad8860ee1b1196a1f242c9dd18d218e6db573 -r 7677b8c3ef43d06eda783f1c08ec521d2bd625e5 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -61,7 +61,7 @@ import nose.loader import nose.plugins.manager -from base.util import parse_tool_panel_config +from base.util import parse_tool_panel_config, get_database_version, get_test_environment, get_repository_current_revision from common import update @@ -105,7 +105,7 @@ # Define a default location to find the list of repositories to check. galaxy_repository_list = os.environ.get( 'GALAXY_INSTALL_TEST_REPOSITORY_LIST_LOCATIOM', 'repository_list.json' ) -galaxy_tool_shed_url = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_URL', 'http://toolshed.local:10001' ) +galaxy_tool_shed_url = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_URL', 'http://localhost:9009' ) tool_shed_api_key = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY', None ) assert tool_shed_api_key is not None, 'Unable to proceed without API key.' @@ -191,25 +191,22 @@ else: raise AssertonError( 'Unknown format %s.' % format ) -def get_test_environment(): - rval = {} - rval[ 'python_version' ] = platform.python_version() - rval[ 'architecture' ] = platform.machine() - os, hostname, os_version, uname, arch, processor = platform.uname() - rval[ 'system' ] = '%s %s' % ( os, os_version ) - return rval - def json_from_url( url ): url_handle = urllib.urlopen( url ) url_contents = url_handle.read() return from_json_string( url_contents ) -def register_test_failure( url, metadata_id, test_errors ): - params = dict( tools_functionally_correct='false', do_not_test='true', tool_test_errors=test_errors ) - return update( tool_shed_api_key, '%s' % ( url_join( galaxy_tool_shed_url, 'api', 'repository_revisions', metadata_id ) ), params, return_formatted=False ) - -def register_test_success( url, metadata_id ): - params = dict( tools_functionally_correct='true', do_not_test='true' ) +def register_test_result( url, metadata_id, test_results_dict, tests_passed=False ): + params = {} + if tests_passed: + params[ 'tools_functionally_correct' ] = 'true' + params[ 'do_not_test' ] = 'true' + test_results_dict[ 'status' ] = 'passed' + else: + params[ 'tools_functionally_correct' ] = 'false' + params[ 'do_not_test' ] = 'true' + test_results_dict[ 'status' ] = 'failed' + params[ 'tool_test_errors' ] = test_results_dict return update( tool_shed_api_key, '%s' % ( url_join( galaxy_tool_shed_url, 'api', 'repository_revisions', metadata_id ) ), params, return_formatted=False ) def run_tests( test_config ): @@ -233,8 +230,8 @@ # Initialize some variables for the summary that will be printed to stdout. repositories_tested = 0 - repositories_passed = 0 - repositories_failed = 0 + repositories_passed = [] + repositories_failed = [] tool_path = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_PATH', 'tools' ) if 'HTTP_ACCEPT_LANGUAGE' not in os.environ: @@ -395,6 +392,9 @@ log.info( "Repositories will be installed from the tool shed at %s" % galaxy_tool_shed_url ) success = False repository_status = dict() + test_environment = get_test_environment() + test_environment[ 'galaxy_database_version' ] = get_database_version( app ) + test_environment[ 'galaxy_revision'] = get_repository_current_revision( os.getcwd() ) try: # Iterate through a list of repository info dicts. log.info( "Retrieving repositories to install from the URL:\n%s\n" % str( galaxy_tool_shed_url ) ) @@ -421,6 +421,7 @@ "url": "/api/repository_revisions/529fd61ab1c6cc36" } """ + repository_status = dict() repository_id = repository_to_install_dict.get( 'repository_id', None ) changeset_revision = repository_to_install_dict.get( 'changeset_revision', None ) metadata_revision_id = repository_to_install_dict.get( 'id', None ) @@ -460,6 +461,8 @@ "user_id": "529fd61ab1c6cc36" } """ + name = repository_dict[ 'name' ] + owner = repository_dict[ 'owner' ] # Use the repository information dict to generate an install method that will install the repository into the embedded # Galaxy application, with tool dependencies and repository dependencies, if any. test_install_repositories.generate_install_method( repository_dict ) @@ -489,13 +492,13 @@ ( repository_dict[ 'changeset_revision' ], repository_dict[ 'name' ], repository_dict[ 'owner' ] ) ) repository_status[ 'test_errors' ] = [ test_errors ] # Record the status of this repository in the tool shed. - register_test_failure( galaxy_tool_shed_url, metadata_revision_id, repository_status ) + register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=False ) # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the # repository using Twill. execute_uninstall_method( repository_dict ) # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. test_toolbox.toolbox = app.toolbox - repositories_failed += 1 + repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) else: # If the repository does have a test-data directory, we write the generated shed_tools_dict to a file, so the functional # test framework can find it. @@ -519,16 +522,17 @@ success = result.wasSuccessful() # Record some information about the environment in which this test was run, in case a failure is specific to a certain processor # architecture or operating system. - repository_dict[ 'test_environment' ] = get_test_environment() - repository_dict[ 'functional_tests_passed' ] = success + repository_dict[ 'test_environment' ] = test_environment test_errors = [] if success: # This repository's tools passed all functional tests. Update the repository_metadata table in the tool shed's database - # to reflect that. Call the register_test_success method, which executes a PUT request to the repository_revisions API + # to reflect that. Call the register_test_result method, which executes a PUT request to the repository_revisions API # controller with the status of the test. This also sets the do_not_test and tools_functionally correct flags, and # updates the time_last_tested field to today's date. - repositories_passed += 1 - register_test_success( galaxy_tool_shed_url, metadata_revision_id ) + repositories_passed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + repository_status[ 'test_environment' ] = test_environment + repository_status[ 'test_errors' ] = [] + register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=True ) log.debug( 'Revision %s of repository %s installed and passed functional tests.' % \ ( repository_dict[ 'changeset_revision' ], repository_dict[ 'name' ] ) ) else: @@ -569,13 +573,13 @@ test_errors.append( test_status ) if test_errors: # Only update test_errors for this repository if it's not empty. - repository_status[ 'test_environment' ] = get_test_environment() + repository_status[ 'test_environment' ] = test_environment repository_status[ 'test_errors' ] = test_errors - # Call the register_test_failure method, which executes a PUT request to the repository_revisions API controller with the failure - # status of the test, and updates tool_test_errors with the relevant log data. + # Call the register_test_result method, which executes a PUT request to the repository_revisions API controller with the outcome + # status of the tests, and updates tool_test_errors with the relevant log data. # This also sets the do_not_test and tools_functionally correct flags, and updates the time_last_tested field to today's date. - repositories_failed += 1 - register_test_failure( galaxy_tool_shed_url, metadata_revision_id, repository_status ) + repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=False ) log.debug( 'Revision %s of repository %s installed successfully, but did not pass functional tests.' % \ ( repository_dict[ 'changeset_revision' ], repository_dict[ 'name' ] ) ) # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the @@ -618,8 +622,14 @@ print "# %s - repository installation and testing script completed." % now print "# Repository revisions tested: %d" % repositories_tested if repositories_tested > 0: - print "# Repository revisions passed: %d" % repositories_passed - print "# Repository revisions failed: %d" % repositories_failed + if repositories_passed: + print "# Repositories passed:" + for repository in repositories_passed: + print "# %s owned by %s, changeset revision %s" % ( repository[ 'name' ], repository[ 'owner' ], repository[ 'changeset_revision' ] ) + if repositories_failed: + print "# Repositories failed:" + for repository in repositories_failed: + print "# %s owned by %s, changeset revision %s" % ( repository[ 'name' ], repository[ 'owner' ], repository[ 'changeset_revision' ] ) print "####################################################################################" if success: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.