1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a950019c26a9/ Changeset: a950019c26a9 User: greg Date: 2013-12-18 22:20:36 Summary: Corrections to statistics for the tool shed's install and test runs and successful installation results for repositories of type tool_dependency_definition now reported to the tool shed. Affected #: 5 files diff -r f1cb7f07af2c923d3f3be77a0ae0ccb25c6453a7 -r a950019c26a9189f145d4da87c589f64babaa8e2 install_and_test_tool_shed_repositories.sh --- a/install_and_test_tool_shed_repositories.sh +++ b/install_and_test_tool_shed_repositories.sh @@ -70,12 +70,16 @@ done case $which in - dependencies) + # Use "-w tool_dependency_definitions" when you want to test repositories of type tool_dependency_definition. + tool_dependency_definitions) test_tool_dependency_definitions ;; - tools) + # Use "-w repositories_with_tools" parameter when you want to test repositories that contain tools. + repositories_with_tools) test_repositories_with_tools ;; + # No received parameters or any received parameter not in [ tool_dependency_definitions, repositories_with_tools ] + # will execute both scripts. *) test_tool_dependency_definitions test_repositories_with_tools diff -r f1cb7f07af2c923d3f3be77a0ae0ccb25c6453a7 -r a950019c26a9189f145d4da87c589f64babaa8e2 lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py --- a/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py +++ b/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py @@ -89,8 +89,9 @@ now = strftime( "%Y-%m-%d %H:%M:%S" ) print "#############################################################################" - print "# %s - Validating repositories of type %s on %s..." % ( now, TOOL_DEPENDENCY_DEFINITION, config_parser.get( config_section, 'host' ) ) - print "# This tool shed is configured to listen on %s:%s" % ( config_parser.get( config_section, 'host' ), config_parser.get( config_section, 'port' ) ) + print "# %s - Validating repositories of type %s" % ( now, TOOL_DEPENDENCY_DEFINITION ) + print "# This tool shed is configured to listen on %s:%s" % ( config_parser.get( config_section, 'host' ), + config_parser.get( config_section, 'port' ) ) app = RepositoriesApplication( config ) diff -r f1cb7f07af2c923d3f3be77a0ae0ccb25c6453a7 -r a950019c26a9189f145d4da87c589f64babaa8e2 test/install_and_test_tool_shed_repositories/base/util.py --- a/test/install_and_test_tool_shed_repositories/base/util.py +++ b/test/install_and_test_tool_shed_repositories/base/util.py @@ -225,7 +225,8 @@ return suc.INITIAL_CHANGELOG_HASH, error_message def get_missing_tool_dependencies( repository ): - log.debug( 'Checking %s repository %s for missing tool dependencies.' % ( repository.status, repository.name ) ) + log.debug( 'Checking revision %s of repository %s owned by %s for missing tool dependencies.' % \ + ( str( repository.changeset_revision ), str( repository.name ), str( repository.changeset_revision ) ) ) missing_tool_dependencies = repository.missing_tool_dependencies for tool_dependency in repository.tool_dependencies: log.debug( 'Tool dependency %s version %s has status %s.' % ( tool_dependency.name, tool_dependency.version, tool_dependency.status ) ) @@ -423,12 +424,13 @@ return global_conf def handle_missing_dependencies( app, repository, missing_tool_dependencies, repository_dict, tool_test_results_dicts, - tool_test_results_dict, results_dict, can_update_tool_shed ): + tool_test_results_dict, can_update_tool_shed ): """Handle missing repository or tool dependencies for an installed repository.""" # If a tool dependency fails to install correctly, this should be considered an installation error, # and functional tests should be skipped, since the tool dependency needs to be correctly installed # for the test to be considered reliable. - log.debug( 'The following dependencies of this repository are missing, so skipping functional tests.' ) + log.debug( 'The following dependencies of revision %s of repository %s owned by %s are missing.' % \ + ( str( repository.changeset_revision ), str( repository.name ), str( repository.owner ) ) ) # In keeping with the standard display layout, add the error message to the dict for each tool individually. for dependency in repository.missing_tool_dependencies: name = str( dependency.name ) @@ -436,24 +438,25 @@ version = str( dependency.version ) error_message = unicodify( dependency.error_message ) log.debug( 'Missing tool dependency %s of type %s version %s: %s' % ( name, type, version, error_message ) ) - test_result = dict( type=dependency.type, - name=dependency.name, - version=dependency.version, - error_message=dependency.error_message ) - tool_test_results_dict[ 'installation_errors' ][ 'tool_dependencies' ].append( test_result ) + missing_tool_dependency_info_dict = dict( type=type, + name=name, + version=version, + error_message=error_message ) + tool_test_results_dict[ 'installation_errors' ][ 'tool_dependencies' ].append( missing_tool_dependency_info_dict ) for dependency in repository.missing_repository_dependencies: + tool_shed = str( dependency.tool_shed ) name = str( dependency.name ) owner = str( dependency.owner ) changeset_revision = str( dependency.changeset_revision ) error_message = unicodify( dependency.error_message ) log.debug( 'Missing repository dependency %s changeset revision %s owned by %s: %s' % \ ( name, changeset_revision, owner, error_message ) ) - test_result = dict( tool_shed=dependency.tool_shed, - name=dependency.name, - owner=dependency.owner, - changeset_revision=dependency.changeset_revision, - error_message=dependency.error_message ) - tool_test_results_dict[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) + missing_repository_dependency_info_dict = dict( tool_shed=tool_shed, + name=name, + owner=owner, + changeset_revision=changeset_revision, + error_message=error_message ) + tool_test_results_dict[ 'installation_errors' ][ 'repository_dependencies' ].append( missing_repository_dependency_info_dict ) # Record the status of this repository in the tool shed. params = dict( tools_functionally_correct=False, do_not_test=False, @@ -465,26 +468,25 @@ repository_dict, params, can_update_tool_shed ) - # Since this repository is missing components we do not want to test it, uninstall it. + # Uninstall this repository since it is missing dependencies. uninstall_repository_and_repository_dependencies( app, repository_dict ) - results_dict[ 'repositories_failed_install' ].append( dict( name=str( repository.name ), - owner=str( repository.owner ), - changeset_revision=str( repository.changeset_revision ) ) ) - return results_dict -def initialize_results_dict( test_framework ): +def initialize_install_and_test_statistics_dict( test_framework ): # Initialize a dictionary for the summary that will be printed to stdout. - results_dict = {} + install_and_test_statistics_dict = {} if test_framework == REPOSITORIES_WITH_TOOLS: - results_dict[ 'total_repositories_tested' ] = 0 - results_dict[ 'all_tests_passed' ] = [] - results_dict[ 'at_least_one_test_failed' ] = [] - results_dict[ 'repositories_failed_install' ] = [] + install_and_test_statistics_dict[ 'total_repositories_processed' ] = 0 + install_and_test_statistics_dict[ 'all_tests_passed' ] = [] + install_and_test_statistics_dict[ 'at_least_one_test_failed' ] = [] + install_and_test_statistics_dict[ 'successful_installations' ] = [] + install_and_test_statistics_dict[ 'repositories_with_installation_error' ] = [] + install_and_test_statistics_dict[ 'tool_dependencies_with_installation_error' ] = [] elif test_framework == TOOL_DEPENDENCY_DEFINITIONS: - results_dict[ 'total_repositories_installed' ] = 0 - results_dict[ 'repositories_with_installation_error' ] = 0 - results_dict[ 'tool_dependencies_with_installation_error' ] = 0 - return results_dict + install_and_test_statistics_dict[ 'total_repositories_processed' ] = 0 + install_and_test_statistics_dict[ 'successful_installations' ] = [] + install_and_test_statistics_dict[ 'repositories_with_installation_error' ] = [] + install_and_test_statistics_dict[ 'tool_dependencies_with_installation_error' ] = [] + return install_and_test_statistics_dict def initialize_tool_tests_results_dict( app, tool_test_results_dict ): test_environment_dict = tool_test_results_dict.get( 'test_environment', {} ) diff -r f1cb7f07af2c923d3f3be77a0ae0ccb25c6453a7 -r a950019c26a9189f145d4da87c589f64babaa8e2 test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py @@ -134,10 +134,10 @@ return tool_id, tool_version def install_and_test_repositories( app, galaxy_shed_tools_dict, galaxy_shed_tool_conf_file ): - results_dict = install_and_test_base_util.initialize_results_dict( test_framework ) + install_and_test_statistics_dict = install_and_test_base_util.initialize_install_and_test_statistics_dict( test_framework ) error_message = '' # Initialize a dictionary for the summary that will be printed to stdout. - total_repositories_tested = results_dict[ 'total_repositories_tested' ] + total_repositories_processed = install_and_test_statistics_dict[ 'total_repositories_processed' ] repositories_to_install, error_message = \ install_and_test_base_util.get_repositories_to_install( install_and_test_base_util.galaxy_tool_shed_url, test_framework ) if error_message: @@ -213,8 +213,7 @@ break if this_repository_is_in_the_exclude_list: tool_test_results_dict[ 'not_tested' ] = dict( reason=skip_reason ) - params = dict( tools_functionally_correct=False, - do_not_test=False ) + params = dict( do_not_test=False ) # TODO: do something useful with response_dict response_dict = install_and_test_base_util.register_test_result( install_and_test_base_util.galaxy_tool_shed_url, tool_test_results_dicts, @@ -223,30 +222,29 @@ params, can_update_tool_shed ) log.debug( "Not testing revision %s of repository %s owned by %s because it is in the exclude list for this test run." % \ - ( changeset_revision, name, owner ) ) + ( changeset_revision, name, owner ) ) else: tool_test_results_dict = install_and_test_base_util.initialize_tool_tests_results_dict( app, tool_test_results_dict ) # Explicitly clear tests from twill's test environment. remove_generated_tests( app ) # Proceed with installing repositories and testing contained tools. repository, error_message = install_and_test_base_util.install_repository( app, repository_dict ) + install_and_test_statistics_dict[ 'total_repositories_processed' ] += 1 if error_message: + # The repository installation failed. + log.debug( 'Installation failed for revision %s of repository %s owned by %s.' % \ + ( changeset_revision, name, owner ) ) + install_and_test_statistics_dict[ 'repositories_with_installation_error' ].append( repository_identifier_dict ) tool_test_results_dict[ 'installation_errors' ][ 'current_repository' ] = error_message # Even if the repository failed to install, execute the uninstall method, in case a dependency did succeed. - log.debug( 'Attempting to uninstall repository %s owned by %s.' % ( name, owner ) ) + log.debug( 'Attempting to uninstall revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) ) try: repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) except Exception, e: - error_message = 'Unable to find installed repository %s owned by %s: %s.' % ( name, owner, str( e ) ) + error_message = 'Unable to find revision %s of repository %s owned by %s: %s.' % \ + ( changeset_revision, name, owner, str( e ) ) log.exception( error_message ) - test_result = dict( tool_shed=install_and_test_base_util.galaxy_tool_shed_url, - name=name, - owner=owner, - changeset_revision=changeset_revision, - error_message=error_message ) - tool_test_results_dict[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) - params = dict( tools_functionally_correct=False, - test_install_error=True, + params = dict( test_install_error=True, do_not_test=False ) # TODO: do something useful with response_dict response_dict = install_and_test_base_util.register_test_result( install_and_test_base_util.galaxy_tool_shed_url, @@ -263,18 +261,23 @@ ( changeset_revision, name, owner, str( e ) ) ) # Clean out any generated tests. This is necessary for Twill. remove_generated_tests( app ) - results_dict[ 'repositories_failed_install' ].append( repository_identifier_dict ) - log.debug( 'Repository %s failed to install correctly.' % str( name ) ) + install_and_test_statistics_dict[ 'repositories_with_installation_error' ].append( repository_identifier_dict ) + log.debug( 'Installation succeeded for revision %s of repository %s owned by %s.' % \ + ( changeset_revision, name, owner ) ) else: - # Configure and run functional tests for this repository. This is equivalent to sh run_functional_tests.sh -installed - remove_install_tests() - log.debug( 'Installation of %s succeeded, running all defined functional tests.' % str( repository.name ) ) - # Generate the shed_tools_dict that specifies the location of test data contained within this repository. If the repository - # does not have a test-data directory, this will return has_test_data = False, and we will set the do_not_test flag to True, - # and the tools_functionally_correct flag to False, as well as updating tool_test_results. - file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( {} ) ) - has_test_data, shed_tools_dict = parse_tool_panel_config( galaxy_shed_tool_conf_file, - from_json_string( file( galaxy_shed_tools_dict, 'r' ).read() ) ) + log.debug( 'Installation succeeded for revision %s of repository %s owned by %s.' % \ + ( changeset_revision, name, owner ) ) + # Keep statistics for this repository's tool dependencies that resulted in installation errors. + for missing_tool_dependency in repository.missing_tool_dependencies: + name = str( missing_tool_dependency.name ) + type = str( missing_tool_dependency.type ) + version = str( missing_tool_dependency.version ) + error_message = unicodify( missing_tool_dependency.error_message ) + missing_tool_dependency_info_dict = dict( type=type, + name=name, + version=version, + error_message=error_message ) + install_and_test_statistics_dict[ 'tool_dependencies_with_installation_error' ].append( missing_tool_dependency_info_dict ) # Add an empty 'missing_test_results' entry if it is missing from the tool_test_results_dict. The # ~/tool_shed/scripts/check_repositories_for_functional_tests.py will have entered information in the # 'missing_test_components' entry of the tool_test_results_dict dictionary for repositories that are @@ -283,28 +286,40 @@ tool_test_results_dict[ 'missing_test_components' ] = [] missing_tool_dependencies = install_and_test_base_util.get_missing_tool_dependencies( repository ) if missing_tool_dependencies or repository.missing_repository_dependencies: - results_dict = install_and_test_base_util.handle_missing_dependencies( app, - repository, - missing_tool_dependencies, - repository_dict, - tool_test_results_dicts, - tool_test_results_dict, - results_dict, - can_update_tool_shed ) + install_and_test_statistics_dict = \ + install_and_test_base_util.handle_missing_dependencies( app, + repository, + missing_tool_dependencies, + repository_dict, + tool_test_results_dicts, + tool_test_results_dict, + install_and_test_statistics_dict, + can_update_tool_shed ) # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. test_toolbox.toolbox = app.toolbox else: + # This repository and all of its dependencies were successfully installed. + install_and_test_statistics_dict[ 'successful_installations' ].append( repository_identifier_dict ) + # Configure and run functional tests for this repository. This is equivalent to sh run_functional_tests.sh -installed + remove_install_tests() + log.debug( 'Installation of %s succeeded, running all defined functional tests.' % str( repository.name ) ) + # Generate the shed_tools_dict that specifies the location of test data contained within this repository. If the repository + # does not have a test-data directory, this will return has_test_data = False, and we will set the do_not_test flag to True, + # and the tools_functionally_correct flag to False, as well as updating tool_test_results. + file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( {} ) ) + has_test_data, shed_tools_dict = parse_tool_panel_config( galaxy_shed_tool_conf_file, + from_json_string( file( galaxy_shed_tools_dict, 'r' ).read() ) ) # If the repository has a test-data directory we write the generated shed_tools_dict to a file, so the functional # test framework can find it. file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( shed_tools_dict ) ) log.debug( 'Saved generated shed_tools_dict to %s\nContents: %s' % ( str( galaxy_shed_tools_dict ), str( shed_tools_dict ) ) ) try: - results_dict = test_repository_tools( app, - repository, - repository_dict, - tool_test_results_dicts, - tool_test_results_dict, - results_dict ) + install_and_test_statistics_dict = test_repository_tools( app, + repository, + repository_dict, + tool_test_results_dicts, + tool_test_results_dict, + install_and_test_statistics_dict ) except Exception, e: exception_message = 'Error executing tests for repository %s: %s' % ( name, str( e ) ) log.exception( exception_message ) @@ -321,10 +336,8 @@ repository_dict, params, can_update_tool_shed ) - results_dict[ 'at_least_one_test_failed' ].append( repository_identifier_dict ) - total_repositories_tested += 1 - results_dict[ 'total_repositories_tested' ] = total_repositories_tested - return results_dict, error_message + install_and_test_statistics_dict[ 'at_least_one_test_failed' ].append( repository_identifier_dict ) + return install_and_test_statistics_dict, error_message def main(): if install_and_test_base_util.tool_shed_api_key is None: @@ -535,35 +548,45 @@ persist=False ) now = time.strftime( "%Y-%m-%d %H:%M:%S" ) print "####################################################################################" - print "# %s - running repository installation and testing script." % now + print "# %s - installation script for repositories containing tools started." % now + if not can_update_tool_shed: + print "# This run will not update the Tool Shed database." print "####################################################################################" - results_dict, error_message = install_and_test_repositories( app, galaxy_shed_tools_dict, galaxy_shed_tool_conf_file ) + install_and_test_statistics_dict, error_message = \ + install_and_test_repositories( app, galaxy_shed_tools_dict, galaxy_shed_tool_conf_file ) if error_message: log.debug( error_message ) else: - total_repositories_tested = results_dict[ 'total_repositories_tested' ] - all_tests_passed = results_dict[ 'all_tests_passed' ] - at_least_one_test_failed = results_dict[ 'at_least_one_test_failed' ] - repositories_failed_install = results_dict[ 'repositories_failed_install' ] + total_repositories_processed = install_and_test_statistics_dict[ 'total_repositories_processed' ] + all_tests_passed = install_and_test_statistics_dict[ 'all_tests_passed' ] + at_least_one_test_failed = install_and_test_statistics_dict[ 'at_least_one_test_failed' ] + successful_installations = install_and_test_statistics_dict[ 'successful_installations' ] + repositories_with_installation_error = install_and_test_statistics_dict[ 'repositories_with_installation_error' ] + tool_dependencies_with_installation_error = install_and_test_statistics_dict[ 'tool_dependencies_with_installation_error' ] now = time.strftime( "%Y-%m-%d %H:%M:%S" ) print "####################################################################################" - print "# %s - repository installation and testing script completed." % now - print "# Repository revisions tested: %s" % str( total_repositories_tested ) - if not can_update_tool_shed: - print "# This run will not update the Tool Shed database." - if total_repositories_tested > 0: - if all_tests_passed: - print '# ----------------------------------------------------------------------------------' - print "# %d repositories successfully passed all functional tests:" % len( all_tests_passed ) - install_and_test_base_util.show_summary_output( all_tests_passed ) - if at_least_one_test_failed: - print '# ----------------------------------------------------------------------------------' - print "# %d repositories failed at least 1 functional test:" % len( at_least_one_test_failed ) - install_and_test_base_util.show_summary_output( at_least_one_test_failed ) - if repositories_failed_install: - print '# ----------------------------------------------------------------------------------' - print "# %d repositories have installation errors:" % len( repositories_failed_install ) - install_and_test_base_util.show_summary_output( repositories_failed_install ) + print "# %s - installation script for repositories containing tools completed." % now + print "# Repository revisions processed: %s" % str( total_repositories_processed ) + if successful_installations: + print "# ----------------------------------------------------------------------------------" + print "# The following %d revisions with all dependencies were successfully installed:" % len( successful_installations ) + install_and_test_base_util.show_summary_output( successful_installations ) + if all_tests_passed: + print '# ----------------------------------------------------------------------------------' + print "# %d repositories successfully passed all functional tests:" % len( all_tests_passed ) + install_and_test_base_util.show_summary_output( all_tests_passed ) + if at_least_one_test_failed: + print '# ----------------------------------------------------------------------------------' + print "# %d repositories failed at least 1 functional test:" % len( at_least_one_test_failed ) + install_and_test_base_util.show_summary_output( at_least_one_test_failed ) + if repositories_with_installation_error: + print '# ----------------------------------------------------------------------------------' + print "# %d repositories have installation errors:" % len( repositories_with_installation_error ) + install_and_test_base_util.show_summary_output( repositories_with_installation_error ) + if tool_dependencies_with_installation_error: + print "# ----------------------------------------------------------------------------------" + print "# The following %d tool dependencies have installation errors:" % len( tool_dependencies_with_installation_error ) + install_and_test_base_util.show_summary_output( tool_dependencies_with_installation_error ) print "####################################################################################" log.debug( "Shutting down..." ) # Gracefully shut down the embedded web server and UniverseApplication. @@ -632,7 +655,8 @@ # Now delete the tests found in the previous loop. del test_install_repositories.__dict__[ key ] -def test_repository_tools( app, repository, repository_dict, tool_test_results_dicts, tool_test_results_dict, results_dict ): +def test_repository_tools( app, repository, repository_dict, tool_test_results_dicts, tool_test_results_dict, + install_and_test_statistics_dict ): """Test tools contained in the received repository.""" name = str( repository.name ) owner = str( repository.owner ) @@ -685,7 +709,7 @@ failed_test_dicts = get_failed_test_dicts( result, from_tool_test=True ) tool_test_results_dict[ 'failed_tests' ] = failed_test_dicts failed_repository_dict = repository_identifier_dict - results_dict[ 'at_least_one_test_failed' ].append( failed_repository_dict ) + install_and_test_statistics_dict[ 'at_least_one_test_failed' ].append( failed_repository_dict ) set_do_not_test = not is_latest_downloadable_revision( install_and_test_base_util.galaxy_tool_shed_url, repository_dict ) params = dict( tools_functionally_correct=False, test_install_error=False, @@ -708,7 +732,7 @@ remove_generated_tests( app ) # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. test_toolbox.toolbox = app.toolbox - return results_dict + return install_and_test_statistics_dict if __name__ == "__main__": # The tool_test_results_dict should always have the following structure: diff -r f1cb7f07af2c923d3f3be77a0ae0ccb25c6453a7 -r a950019c26a9189f145d4da87c589f64babaa8e2 test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py @@ -73,10 +73,10 @@ test_framework = install_and_test_base_util.TOOL_DEPENDENCY_DEFINITIONS def install_and_test_repositories( app, galaxy_shed_tools_dict, galaxy_shed_tool_conf_file ): - results_dict = install_and_test_base_util.initialize_results_dict( test_framework ) + install_and_test_statistics_dict = install_and_test_base_util.initialize_install_and_test_statistics_dict( test_framework ) error_message = '' # Initialize a dictionary for the summary that will be printed to stdout. - total_repositories_installed = results_dict[ 'total_repositories_installed' ] + total_repositories_processed = install_and_test_statistics_dict[ 'total_repositories_processed' ] repositories_to_install, error_message = \ install_and_test_base_util.get_repositories_to_install( install_and_test_base_util.galaxy_tool_shed_url, test_framework ) if error_message: @@ -161,27 +161,26 @@ params, can_update_tool_shed ) log.debug( "Not testing revision %s of repository %s owned by %s because it is in the exclude list for this test run." % \ - ( changeset_revision, name, owner ) ) + ( changeset_revision, name, owner ) ) else: tool_test_results_dict = install_and_test_base_util.initialize_tool_tests_results_dict( app, tool_test_results_dict ) repository, error_message = install_and_test_base_util.install_repository( app, repository_dict ) + install_and_test_statistics_dict[ 'total_repositories_processed' ] += 1 if error_message: + # The repository installation failed. + log.debug( 'Installation failed for revision %s of repository %s owned by %s.' % \ + ( changeset_revision, name, owner ) ) + install_and_test_statistics_dict[ 'repositories_with_installation_error' ].append( repository_identifier_dict ) tool_test_results_dict[ 'installation_errors' ][ 'current_repository' ] = error_message # Even if the repository failed to install, execute the uninstall method, in case a dependency did succeed. - log.debug( 'Attempting to uninstall repository %s owned by %s.' % ( name, owner ) ) + log.debug( 'Attempting to uninstall revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) ) try: repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) except Exception, e: - error_message = 'Unable to find installed repository %s owned by %s: %s.' % ( name, owner, str( e ) ) + error_message = 'Unable to find revision %s of repository %s owned by %s: %s.' % \ + ( changeset_revision, name, owner, str( e ) ) log.exception( error_message ) - test_result = dict( tool_shed=install_and_test_base_util.galaxy_tool_shed_url, - name=name, - owner=owner, - changeset_revision=changeset_revision, - error_message=error_message ) - tool_test_results_dict[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) - params = dict( tools_functionally_correct=False, - test_install_error=True, + params = dict( test_install_error=True, do_not_test=False ) # TODO: do something useful with response_dict response_dict = install_and_test_base_util.register_test_result( install_and_test_base_util.galaxy_tool_shed_url, @@ -196,22 +195,50 @@ except Exception, e: log.exception( 'Error attempting to uninstall revision %s of repository %s owned by %s: %s' % \ ( changeset_revision, name, owner, str( e ) ) ) - results_dict[ 'repositories_with_installation_error' ].append( repository_identifier_dict ) - log.debug( 'Repository %s failed to install correctly.' % str( name ) ) + log.debug( 'Installation failed for revision %s of repository %s owned by %s.' % \ + ( changeset_revision, name, owner ) ) else: - log.debug( 'Installation of %s succeeded.' % str( repository.name ) ) + log.debug( 'Installation succeeded for revision %s of repository %s owned by %s.' % \ + ( changeset_revision, name, owner ) ) + # Keep statistics for this repository's tool dependencies that resulted in installation errors. + for missing_tool_dependency in repository.missing_tool_dependencies: + name = str( missing_tool_dependency.name ) + type = str( missing_tool_dependency.type ) + version = str( missing_tool_dependency.version ) + error_message = unicodify( missing_tool_dependency.error_message ) + missing_tool_dependency_info_dict = dict( type=type, + name=name, + version=version, + error_message=error_message ) + install_and_test_statistics_dict[ 'tool_dependencies_with_installation_error' ].append( missing_tool_dependency_info_dict ) + # Attempt to uninstall this repository and all of its dependencies if its repository dependencies or + # tool dependencies resulted in an installation error. missing_tool_dependencies = install_and_test_base_util.get_missing_tool_dependencies( repository ) if missing_tool_dependencies or repository.missing_repository_dependencies: - results_dict = install_and_test_base_util.handle_missing_dependencies( app, - repository, - missing_tool_dependencies, - repository_dict, - tool_test_results_dicts, - tool_test_results_dict, - results_dict, - can_update_tool_shed ) - results_dict[ 'total_repositories_installed' ] = total_repositories_installed - return results_dict, error_message + install_and_test_base_util.handle_missing_dependencies( app, + repository, + missing_tool_dependencies, + repository_dict, + tool_test_results_dicts, + tool_test_results_dict, + install_and_test_statistics_dict, + can_update_tool_shed ) + else: + # This repository and all of its dependencies were successfully installed. + install_and_test_statistics_dict[ 'successful_installations' ].append( repository_identifier_dict ) + tool_test_results_dict[ 'passed_tests' ].append( repository_identifier_dict ) + params = dict( test_install_error=False, + do_not_test=False ) + # TODO: do something useful with response_dict + response_dict = install_and_test_base_util.register_test_result( install_and_test_base_util.galaxy_tool_shed_url, + tool_test_results_dicts, + tool_test_results_dict, + repository_dict, + params, + can_update_tool_shed ) + + install_and_test_statistics_dict[ 'total_repositories_processed' ] = total_repositories_processed + return install_and_test_statistics_dict, error_message def main(): if install_and_test_base_util.tool_shed_api_key is None: @@ -419,28 +446,34 @@ persist=False ) now = time.strftime( "%Y-%m-%d %H:%M:%S" ) print "####################################################################################" - print "# %s - running repository installation and testing script." % now + print "# %s - installation script for repositories of type tool_dependency_definition started." % now + if not can_update_tool_shed: + print "# This run will not update the Tool Shed database." print "####################################################################################" - results_dict, error_message = install_and_test_repositories( app, galaxy_shed_tools_dict, galaxy_shed_tool_conf_file ) + install_and_test_statistics_dict, error_message = \ + install_and_test_repositories( app, galaxy_shed_tools_dict, galaxy_shed_tool_conf_file ) if error_message: log.debug( error_message ) else: - total_repositories_installed = results_dict[ 'total_repositories_installed' ] - repositories_with_installation_error = results_dict[ 'repositories_with_installation_error' ] - tool_dependencies_with_installation_error = results_dict[ 'tool_dependencies_with_installation_error' ] + total_repositories_processed = install_and_test_statistics_dict[ 'total_repositories_processed' ] + successful_installations = install_and_test_statistics_dict[ 'successful_installations' ] + repositories_with_installation_error = install_and_test_statistics_dict[ 'repositories_with_installation_error' ] + tool_dependencies_with_installation_error = install_and_test_statistics_dict[ 'tool_dependencies_with_installation_error' ] now = time.strftime( "%Y-%m-%d %H:%M:%S" ) print "####################################################################################" - print "# %s - repository installation and testing script completed." % now - print "# Repository revisions tested: %s" % str( total_repositories_installed ) - if not can_update_tool_shed: - print "# This run will not update the Tool Shed database." + print "# %s - installation script for repositories of type tool_dependency_definition completed." % now + print "# Repository revisions processed: %s" % str( total_repositories_processed ) + if successful_installations: + print "# ----------------------------------------------------------------------------------" + print "# The following %d revisions with all dependencies were successfully installed:" % len( successful_installations ) + install_and_test_base_util.show_summary_output( successful_installations ) if repositories_with_installation_error: - print '# ----------------------------------------------------------------------------------' - print "# %d repositories have installation errors:" % len( repositories_with_installation_error ) + print "# ----------------------------------------------------------------------------------" + print "# The following %d revisions have installation errors:" % len( repositories_with_installation_error ) install_and_test_base_util.show_summary_output( repositories_with_installation_error ) if tool_dependencies_with_installation_error: - print '# ----------------------------------------------------------------------------------' - print "# %d tool dependencies have installation errors:" % len( tool_dependencies_with_installation_error ) + print "# ----------------------------------------------------------------------------------" + print "# The following %d tool dependencies have installation errors:" % len( tool_dependencies_with_installation_error ) install_and_test_base_util.show_summary_output( tool_dependencies_with_installation_error ) print "####################################################################################" log.debug( "Shutting down..." ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.