- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: Dave Bouvier: Set the galaxy test host variable for test_toolbox.
by commits-noreply＠bitbucket.org 22 Nov '13

22 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/31138346d9ec/ Changeset: 31138346d9ec User: Dave Bouvier Date: 2013-11-22 20:41:04 Summary: Set the galaxy test host variable for test_toolbox. Affected #: 1 file diff -r 872a34a5760fd4d855c8d2ee8f6dffac6a3d64b9 -r 31138346d9ecb87887223e7940877b7949059a56 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -1020,6 +1020,9 @@ galaxy_test_host = os.environ.get( 'GALAXY_INSTALL_TEST_HOST', default_galaxy_test_host ) # Set the GALAXY_INSTALL_TEST_HOST variable so that Twill will have the Galaxy url to install repositories into. os.environ[ 'GALAXY_INSTALL_TEST_HOST' ] = galaxy_test_host + # Set the GALAXY_TEST_HOST environment variable so that the toolbox tests will have the Galaxy url to run tool functional + # tests on. + os.environ[ 'GALAXY_TEST_HOST' ] = galaxy_test_host galaxy_test_port = os.environ.get( 'GALAXY_INSTALL_TEST_PORT', str( default_galaxy_test_port_max ) ) os.environ[ 'GALAXY_TEST_PORT' ] = galaxy_test_port tool_path = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_PATH', 'tools' ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Fix undefined name.
by commits-noreply＠bitbucket.org 22 Nov '13

22 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/872a34a5760f/ Changeset: 872a34a5760f User: Dave Bouvier Date: 2013-11-22 20:21:10 Summary: Fix undefined name. Affected #: 1 file diff -r ee214245805dee0e811a93db339ef9ab134de9a3 -r 872a34a5760fd4d855c8d2ee8f6dffac6a3d64b9 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -411,6 +411,7 @@ error_message = '' latest_revision_only = '-check_all_revisions' not in sys.argv repository_dicts = [] + results_dict = {} # We are testing deprecated repositories because it is possible that a deprecated repository contains valid tools that # someone has previously installed. Deleted repositories have never been installed, so should not be tested. If they are # undeleted, this script will then test them the next time it runs. We don't need to check if a repository has been deleted Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Refactor the tool shed's install and test framework to enable restriction of bugs introduced into the Galaxy framework that result in server exceptions to specific repository tests rather than the entire framework.
by commits-noreply＠bitbucket.org 22 Nov '13

22 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ee214245805d/ Changeset: ee214245805d User: greg Date: 2013-11-22 19:49:30 Summary: Refactor the tool shed's install and test framework to enable restriction of bugs introduced into the Galaxy framework that result in server exceptions to specific repository tests rather than the entire framework. Affected #: 2 files diff -r d28d25d66095b84af1aaee1a487ee2b27c5a07ce -r ee214245805dee0e811a93db339ef9ab134de9a3 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -144,22 +144,6 @@ # If this changeset revision has no tools, we don't need to do anything here, the install and test script has a filter for returning # only repositories that contain tools. tool_dicts = repository_metadata.get( 'tools', None ) - """ - "{"tools": - [{"add_to_tool_panel": true, - "description": "data on any column using simple expressions", - "guid": "localhost:9009/repos/iuc/filtering_with_empty_test_tags/Filter1/1.1.0", - "id": "Filter1", - "name": "Filter", - "requirements": [], - "tests": - [{"inputs": [], "name": "Test-1", "outputs": [], "required_files": []}, - {"inputs": [], "name": "Test-2", "outputs": [], "required_files": []}], - "tool_config": "database/community_files/000/repo_6/filtering.xml", - "tool_type": "default", - "version": "1.1.0", - "version_string_cmd": null}]}" - """ if tool_dicts is not None: has_test_data = False testable_revision_found = False diff -r d28d25d66095b84af1aaee1a487ee2b27c5a07ce -r ee214245805dee0e811a93db339ef9ab134de9a3 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -91,7 +91,7 @@ default_galaxy_test_port_min = 10000 default_galaxy_test_port_max = 10999 default_galaxy_test_host = '127.0.0.1' -default_galaxy_master_api_key = "123456" +default_galaxy_master_api_key = None # should this serve static resources (scripts, images, styles, etc.) STATIC_ENABLED = True @@ -262,14 +262,14 @@ return passed_tests return [] -def deactivate_repository( app, repository_info_dict ): +def deactivate_repository( app, repository_dict ): # Clean out any generated tests. This is necessary for Twill. remove_generated_tests( app ) sa_session = app.model.context.current # The dict contains the only repository the app should have installed at this point. - repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( repository_info_dict[ 'name' ], - repository_info_dict[ 'owner' ], - repository_info_dict[ 'changeset_revision' ] ) + repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( repository_dict[ 'name' ], + repository_dict[ 'owner' ], + repository_dict[ 'changeset_revision' ] ) # We have to do this through Twill, in order to maintain app.toolbox and shed_tool_conf.xml in a state that is valid for future tests. for required_repository in repository.repository_dependencies: repository_dict = dict( name=required_repository.name, @@ -355,14 +355,17 @@ return url def get_latest_downloadable_changeset_revision( url, name, owner ): + error_message = '' api_url_parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ] params = urllib.urlencode( dict( name=name, owner=owner ) ) api_url = get_api_url( url, api_url_parts, params ) - changeset_revisions = json_from_url( api_url ) + changeset_revisions, error_message = json_from_url( api_url ) + if error_message: + return None, error_message if changeset_revisions: - return changeset_revisions[ -1 ] + return changeset_revisions[ -1 ], error_message else: - return '000000000000' + return suc.INITIAL_CHANGELOG_HASH, error_message def get_missing_tool_dependencies( repository ): log.debug( 'Checking %s repository %s for missing tool dependencies.' % ( repository.status, repository.name ) ) @@ -370,18 +373,24 @@ for tool_dependency in repository.tool_dependencies: log.debug( 'Tool dependency %s version %s has status %s.' % ( tool_dependency.name, tool_dependency.version, tool_dependency.status ) ) for repository_dependency in repository.repository_dependencies: - if not repository_dependency.includes_tool_dependencies: - continue - missing_tool_dependencies.extend( get_missing_tool_dependencies( repository_dependency ) ) + if repository_dependency.includes_tool_dependencies: + missing_tool_dependencies.extend( get_missing_tool_dependencies( repository_dependency ) ) return missing_tool_dependencies -def get_repository_info_from_api( url, repository_info_dict ): - parts = [ 'api', 'repositories', repository_info_dict[ 'repository_id' ] ] +def get_repository_info_from_api( url, repository_dict ): + error_message = '' + parts = [ 'api', 'repositories', repository_dict[ 'repository_id' ] ] api_url = get_api_url( base=url, parts=parts ) - extended_dict = json_from_url( api_url ) - latest_changeset_revision = get_latest_downloadable_changeset_revision( url, extended_dict[ 'name' ], extended_dict[ 'owner' ] ) + extended_dict, error_message = json_from_url( api_url ) + if error_message: + return None, error_message + latest_changeset_revision, error_message = get_latest_downloadable_changeset_revision( url, + extended_dict[ 'name' ], + extended_dict[ 'owner' ] ) + if error_message: + return None, error_message extended_dict[ 'latest_revision' ] = str( latest_changeset_revision ) - return extended_dict + return extended_dict, error_message def get_repository_tuple_from_elem( elem ): attributes = elem.attrib @@ -390,78 +399,84 @@ changeset_revision = attributes.get( 'changeset_revision', None ) return ( name, owner, changeset_revision ) -def get_repositories_to_install( tool_shed_url, latest_revision_only=True ): - ''' +def get_repositories_to_install( tool_shed_url ): + """ Get a list of repository info dicts to install. This method expects a json list of dicts with the following structure: - [ - { - "changeset_revision": <revision>, - "encoded_repository_id": <encoded repository id from the tool shed>, - "name": <name>, - "owner": <owner>, - "tool_shed_url": <url> - }, - ... - ] - NOTE: If the tool shed URL specified in any dict is not present in the tool_sheds_conf.xml, the installation will fail. - ''' - assert tool_shed_api_key is not None, 'Cannot proceed without tool shed API key.' + [{ "changeset_revision": <revision>, + "encoded_repository_id": <encoded repository id from the tool shed>, + "name": <name>, + "owner": <owner>, + "tool_shed_url": <url> }] + """ + error_message = '' + latest_revision_only = '-check_all_revisions' not in sys.argv + repository_dicts = [] + # We are testing deprecated repositories because it is possible that a deprecated repository contains valid tools that + # someone has previously installed. Deleted repositories have never been installed, so should not be tested. If they are + # undeleted, this script will then test them the next time it runs. We don't need to check if a repository has been deleted + # here because our call to the Tool Shed API filters by downloadable='true', in which case deleted will always be False. + if os.path.exists( exclude_list_file ): + log.debug( 'Loading the list of repositories excluded from testing from the file %s...', exclude_list_file ) + exclude_list = parse_exclude_list( exclude_list_file ) + results_dict[ 'exclude_list' ] = exclude_list params = urllib.urlencode( dict( do_not_test='false', downloadable='true', malicious='false', includes_tools='true', skip_tool_test='false' ) ) api_url = get_api_url( base=tool_shed_url, parts=[ 'repository_revisions' ], params=params ) - base_repository_list = json_from_url( api_url ) - log.info( 'The api returned %d metadata revisions.', len( base_repository_list ) ) - known_repository_ids = {} - detailed_repository_list = [] - for repository_to_install_dict in base_repository_list: + baseline_repository_dicts, error_message = json_from_url( api_url ) + if error_message: + return None, error_message + log.debug( 'The Tool Shed API returned %d metadata revisions for installation and testing.' % len( baseline_repository_dicts ) ) + for baseline_repository_dict in baseline_repository_dicts: # We need to get some details from the tool shed API, such as repository name and owner, to pass on to the # module that will generate the install methods. - repository_info_dict = get_repository_info_from_api( galaxy_tool_shed_url, repository_to_install_dict ) - if repository_info_dict[ 'latest_revision' ] == '000000000000': - continue - owner = repository_info_dict[ 'owner' ] - name = repository_info_dict[ 'name' ] - changeset_revision = repository_to_install_dict[ 'changeset_revision' ] - repository_id = repository_to_install_dict[ 'repository_id' ] - # We are testing deprecated repositories, because it is possible that a deprecated repository contains valid - # and functionally correct tools that someone has previously installed. Deleted repositories have never been installed, - # and therefore do not need to be checked. If they are undeleted, this script will then test them the next time it runs. - if repository_info_dict[ 'deleted' ]: - log.info( "Skipping revision %s of repository id %s (%s/%s) since the repository is deleted...", - changeset_revision, - repository_id, - name, - owner ) - continue - # Now merge the dict returned from /api/repository_revisions with the detailed dict we just retrieved. - if latest_revision_only: - if changeset_revision == repository_info_dict[ 'latest_revision' ]: - detailed_repository_list.append( dict( repository_info_dict.items() + repository_to_install_dict.items() ) ) + repository_dict, error_message = get_repository_info_from_api( galaxy_tool_shed_url, baseline_repository_dict ) + if error_message: + log.debug( 'Error getting additional details about repository %s from the API: %s' % ( str( name ), error_message ) ) else: - detailed_repository_list.append( dict( repository_info_dict.items() + repository_to_install_dict.items() ) ) - repositories_tested = len( detailed_repository_list ) + # Don't test empty repositories. + changeset_revision = baseline_repository_dict[ 'changeset_revision' ] + if changeset_revision != suc.INITIAL_CHANGELOG_HASH: + # Merge the dictionary returned from /api/repository_revisions with the detailed repository_dict and + # append it to the list of repository_dicts to install and test. + if latest_revision_only: + latest_revision = repository_dict[ 'latest_revision' ] + if changeset_revision == latest_revision: + repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) ) + else: + repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) ) if latest_revision_only: skipped_previous = ' and metadata revisions that are not the most recent' else: skipped_previous = '' if testing_single_repository: - log.info( 'Testing single repository with name %s and owner %s.', - testing_single_repository[ 'name' ], - testing_single_repository[ 'owner' ]) - for repository_to_install in detailed_repository_list: - if repository_to_install[ 'name' ] == testing_single_repository[ 'name' ] \ - and repository_to_install[ 'owner' ] == testing_single_repository[ 'owner' ]: - if testing_single_repository[ 'changeset_revision' ] is None: - return [ repository_to_install ] + tsr_name = testing_single_repository[ 'name' ] + tsr_owner = testing_single_repository[ 'owner' ] + tsr_changeset_revision = testing_single_repository[ 'changeset_revision' ] + log.debug( 'Testing single repository with name %s and owner %s.' % ( str( tsr_name ), str( tsr_owner ) ) ) + for repository_to_install in repository_dicts: + rti_name = repository_to_install[ 'name' ] + rti_owner = repository_to_install[ 'owner' ] + rti_changeset_revision = repository_to_install[ 'changeset_revision' ] + if rti_name == tsr_name and rti_owner == tsr_owner: + if tsr_changeset_revision is None: + return [ repository_to_install ], error_message else: - if testing_single_repository[ 'changeset_revision' ] == repository_to_install[ 'changeset_revision' ]: - return [ repository_to_install ] - return [] - log.info( 'After removing deleted repositories%s from the list, %d remain to be tested.', skipped_previous, repositories_tested ) - return detailed_repository_list + if tsr_changeset_revision == rti_changeset_revision: + return repository_dicts, error_message + return repository_dicts, error_message + # Get a list of repositories to test from the tool shed specified in the GALAXY_INSTALL_TEST_TOOL_SHED_URL environment variable. + log.debug( "Retrieved %d repositories from the API url: %s." % ( len( repository_dicts ), str( galaxy_tool_shed_url ) ) ) + if '-list_repositories' in sys.argv: + log.debug( "Repositories for testing:" ) + for repository_dict in repository_dicts: + log.debug( "Revision %s of repository %s owned by %s" % \ + ( str( repository_dict.get( 'changeset_revision', None ) ), \ + str( repository_dict.get( 'name', None ) ), \ + str( repository_dict.get( 'owner', None ) ) ) ) + return repository_dicts, error_message def get_tool_info_from_test_id( test_id ): ''' @@ -474,82 +489,289 @@ return tool_id, tool_version def get_tool_test_results_from_api( tool_shed_url, metadata_revision_id ): + error_message = '' api_path = [ 'api', 'repository_revisions', metadata_revision_id ] api_url = get_api_url( base=tool_shed_url, parts=api_path ) - repository_metadata = json_from_url( api_url ) + repository_metadata, error_message = json_from_url( api_url ) + if error_message: + return None, error_message tool_test_results = repository_metadata.get( 'tool_test_results', {} ) # If, for some reason, the script that checks for functional tests has not run, tool_test_results will be None. if tool_test_results is None: - return {} - return tool_test_results + return {}, error_message + return tool_test_results, error_message -def install_repository( repository_info_dict ): - """ - The repository_info_dict looks something like: - { - "changeset_revision": "13fa22a258b5", - "contents_url": "/api/repositories/529fd61ab1c6cc36/contents", - "deleted": false, - "deprecated": false, - "description": "Convert column case.", - "downloadable": true, - "id": "529fd61ab1c6cc36", - "long_description": "This tool takes the specified columns and converts them to uppercase or lowercase.", - "malicious": false, - "name": "change_case", - "owner": "test", - "private": false, - "repository_id": "529fd61ab1c6cc36", - "times_downloaded": 0, - "tool_shed_url": "http://toolshed.local:10001", - "url": "/api/repository_revisions/529fd61ab1c6cc36", - "user_id": "529fd61ab1c6cc36" - } - """ - data[ 'tool_shed_url' ] = repository_info_dict[ 'tool_shed_url' ] - data[ 'name' ] = repository_info_dict[ 'name' ] - data[ 'owner' ] = repository_info_dict[ 'owner' ] - data[ 'changeset_revision' ] = repository_info_dict[ 'changeset_revision' ] - data[ 'install_repository_dependencies' ] = True - data[ 'install_tool_dependencies' ] = True - submit( options.api, '%s%s' % ( galaxy_url.strip( '/' ), '/api/tool_shed_repositories/new/install_repository_revision' ), data ) +def handle_missing_dependencies( app, repository, metadata_revision_id, missing_tool_dependencies, repository_dict, repository_status_dict, results_dict ): + """Handle missing repository or tool dependencies for an installed repository.""" + # If a tool dependency fails to install correctly, this should be considered an installation error, + # and functional tests should be skipped, since the tool dependency needs to be correctly installed + # for the test to be considered reliable. + log.debug( 'The following dependencies of this repository are missing, so skipping functional tests.' ) + # In keeping with the standard display layout, add the error message to the dict for each tool individually. + for dependency in repository.missing_tool_dependencies: + log.debug( 'Missing tool dependency %s of type %s version %s: %s' % \ + ( str( dependency.name ), str( dependency.type ), str( dependency.version ), unicodify( dependency.error_message ) ) ) + test_result = dict( type=dependency.type, + name=dependency.name, + version=dependency.version, + error_message=dependency.error_message ) + repository_status_dict[ 'installation_errors' ][ 'tool_dependencies' ].append( test_result ) + for dependency in repository.missing_repository_dependencies: + log.debug( 'Missing repository dependency %s changeset revision %s owned by %s: %s' % \ + ( str( dependency.name ), str( dependency.changeset_revision ), str( dependency.owner ), unicodify( dependency.error_message ) ) ) + test_result = dict( tool_shed=dependency.tool_shed, + name=dependency.name, + owner=dependency.owner, + changeset_revision=dependency.changeset_revision, + error_message=dependency.error_message ) + repository_status_dict[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) + # Record the status of this repository in the tool shed. + params[ 'tools_functionally_correct' ] = False + params[ 'do_not_test' ] = False + params[ 'test_install_error' ] = True + register_test_result( galaxy_tool_shed_url, + metadata_revision_id, + repository_status_dict, + repository_dict, + params ) + # Since this repository is missing components, we do not want to test it. Deactivate this repository. + # The deactivate flag is set to True if the environment variable GALAXY_INSTALL_TEST_KEEP_TOOL_DEPENDENCIES + # is set to 'true'. + deactivate = asbool( os.environ.get( 'GALAXY_INSTALL_TEST_KEEP_TOOL_DEPENDENCIES', False ) ) + if deactivate: + log.debug( 'Due to the above missing tool dependencies, we are now uninstalling the following tool dependencies but not changing their repositories.' ) + for missing_tool_dependency in missing_tool_dependencies: + uninstall_tool_dependency( app, missing_tool_dependency ) + # We are deactivating this repository and all of its repository dependencies. + log.debug( 'Due to the above missing repository dependencies, we are now deactivating the following repositories.' ) + deactivate_repository( app, repository_dict ) + else: + # We are uninstalling this repository and all of its repository dependencies. + log.debug( 'Due to the above missing repository dependencies, we are now uninstalling the following repositories.' ) + uninstall_repository( app, repository_dict ) + results_dict[ 'repositories_failed_install' ].append( dict( name=str( repository.name ), + owner=str( repository.owner ), + changeset_revision=str( repository.changeset_revision ) ) ) + # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. + test_toolbox.toolbox = app.toolbox + return repository_status_dict, results_dict -def is_latest_downloadable_revision( url, repository_info_dict ): - latest_revision = get_latest_downloadable_changeset_revision( url, name=repository_info_dict[ 'name' ], owner=repository_info_dict[ 'owner' ] ) - return str( repository_info_dict[ 'changeset_revision' ] ) == str( latest_revision ) +def initialize_results_dict(): + # Initialize a dictionary for the summary that will be printed to stdout. + results_dict = {} + results_dict[ 'total_repositories_tested' ] = 0 + results_dict[ 'repositories_passed' ] = [] + results_dict[ 'repositories_failed' ] = [] + results_dict[ 'repositories_failed_install' ] = [] + results_dict[ 'exclude_list' ] = [] + return results_dict + +def install_repository( app, repository_dict ): + """Attempt to install a repository defined by the received repository_dict from the tool shed into Galaxy.""" + name = str( repository_dict[ 'name' ] ) + owner = str( repository_dict[ 'owner' ] ) + changeset_revision = str( repository_dict[ 'changeset_revision' ] ) + error_message = '' + repository = None + log.debug( "Installing revision %s of repository %s owned by %s..." % ( str( changeset_revision ), str( name ), str( owner ) ) ) + # Explicitly clear tests from twill's test environment. + remove_generated_tests( app ) + # Use the repository information dictionary to generate an install method that will install the repository into the + # embedded Galaxy application, with tool dependencies and repository dependencies, if any. + test_install_repositories.generate_install_method( repository_dict ) + # Configure nose to run the install method as a test. + test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) + test_config.configure( sys.argv ) + # Run the configured install method as a test. This method uses the embedded Galaxy application's web interface to + # install the specified repository with tool and repository dependencies also selected for installation. + result, _ = run_tests( test_config ) + try: + repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) + except Exception, e: + error_message = 'Error getting revision %s of installed repository %s owned by %s: %s' % \ + ( str( changeset_revision ), str( name ), str( owner ), str( e ) ) + log.exception( error_message ) + return repository, error_message + +def install_and_test_repositories( app, galaxy_shed_tools_dict, galaxy_shed_tool_conf_file ): + error_message = '' + # Allow the option to keep or delete tool dependencies when a repository has been tested. + deactivate = asbool( os.environ.get( 'GALAXY_INSTALL_TEST_KEEP_TOOL_DEPENDENCIES', False ) ) + # Initialize a dictionary for the summary that will be printed to stdout. + results_dict = initialize_results_dict() + total_repositories_tested = results_dict[ 'total_repositories_tested' ] + repositories_to_install, error_message = get_repositories_to_install( galaxy_tool_shed_url ) + if error_message: + return None, error_message + # Generate a test method that will use Twill to install each repository into the embedded Galaxy application that was + # started up, installing repository and tool dependencies. Upon successful installation, generate a test case for each + # functional test defined for each tool in the repository and execute the test cases. Record the result of the tests. + # The traceback and captured output of the tool that was run will be recored for test failures. After all tests have + # completed, the repository is uninstalled, so test cases don't interfere with the next repository's functional tests. + for repository_dict in repositories_to_install: + # Each repository_dict looks something like: + # { "changeset_revision": "13fa22a258b5", + # "contents_url": "/api/repositories/529fd61ab1c6cc36/contents", + # "deleted": false, + # "deprecated": false, + # "description": "Convert column case.", + # "downloadable": true, + # "id": "529fd61ab1c6cc36", + # "long_description": "This tool takes the specified columns and converts them to uppercase or lowercase.", + # "malicious": false, + # "name": "change_case", + # "owner": "test", + # "private": false, + # "repository_id": "529fd61ab1c6cc36", + # "times_downloaded": 0, + # "tool_shed_url": "http://toolshed.local:10001", + # "url": "/api/repository_revisions/529fd61ab1c6cc36", + # "user_id": "529fd61ab1c6cc36" } + params = {} + changeset_revision = repository_dict.get( 'changeset_revision', None ) + metadata_revision_id = repository_dict.get( 'id', None ) + # Add the URL for the tool shed we're installing from, so the automated installation methods go to the right place. + repository_dict[ 'tool_shed_url' ] = galaxy_tool_shed_url + # Get the name and owner out of the repository info dict. + name = str( repository_dict[ 'name' ] ) + owner = str( repository_dict[ 'owner' ] ) + # Populate the repository_status_dict. + repository_status_dict, error_message = get_tool_test_results_from_api( galaxy_tool_shed_url, metadata_revision_id ) + if error_message: + return None, error_message + if 'test_environment' not in repository_status_dict: + repository_status_dict[ 'test_environment' ] = {} + test_environment = get_test_environment( repository_status_dict[ 'test_environment' ] ) + test_environment[ 'galaxy_database_version' ] = get_database_version( app ) + test_environment[ 'galaxy_revision'] = get_repository_current_revision( os.getcwd() ) + repository_status_dict[ 'test_environment' ] = test_environment + repository_status_dict[ 'passed_tests' ] = [] + repository_status_dict[ 'failed_tests' ] = [] + repository_status_dict[ 'skip_reason' ] = None + repository_status_dict[ 'installation_errors' ] = dict( current_repository=[], repository_dependencies=[], tool_dependencies=[] ) + # Handle repositories not to be tested. + exclude_list = results_dict[ 'exclude_list' ] + # Entries in the exclude_list look something like this. + # { 'reason': The default reason or the reason specified in this section, + # 'repositories': + # [( name, owner, changeset revision if changeset revision else None ), + # ( name, owner, changeset revision if changeset revision else None )] } + # If changeset revision is None, that means the entire repository is excluded from testing, otherwise only the specified + # revision should be skipped. + skip_this_repository = False + skip_reason = None + for exclude_dict in exclude_list: + reason = exclude_dict[ 'reason' ] + exclude_repositories = exclude_dict[ 'repositories' ] + if ( name, owner, changeset_revision ) in exclude_repositories or ( name, owner, None ) in exclude_repositories: + skip_this_repository = True + skip_reason = reason + break + if skip_this_repository: + repository_status_dict[ 'not_tested' ] = dict( reason=skip_reason ) + params[ 'tools_functionally_correct' ] = False + params[ 'do_not_test' ] = False + register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status_dict, repository_dict, params ) + log.debug( "Not testing revision %s of repository %s owned by %s." % ( str( changeset_revision ), str( name ), str( owner ) ) ) + else: + repository, error_message = install_repository( app, repository_dict ) + if error_message: + repository_status_dict[ 'installation_errors' ][ 'current_repository' ] = error_message + + # Even if the repository failed to install, execute the uninstall method, in case a dependency did succeed. + log.debug( 'Attempting to uninstall repository %s' % str( name ) ) + try: + repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) + except: + log.exception( 'No installed repository found.' ) + error_message = '%s\n%s' % ( error_message, extract_log_data( result, from_tool_test=False ) ) + test_result = dict( tool_shed=galaxy_tool_shed_url, + name=name, + owner=owner, + changeset_revision=changeset_revision, + error_message=error_message ) + repository_status_dict[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) + params[ 'tools_functionally_correct' ] = False + params[ 'test_install_error' ] = True + params[ 'do_not_test' ] = False + register_test_result( galaxy_tool_shed_url, + metadata_revision_id, + repository_status_dict, + repository_dict, + params ) + try: + if deactivate: + # We are deactivating this repository and all of its repository dependencies. + success = deactivate_repository( app, repository_dict ) + else: + # We are uninstalling this repository and all of its repository dependencies. + success = uninstall_repository( app, repository_dict ) + except: + log.exception( 'Encountered error attempting to deactivate or uninstall %s.', repository_dict[ 'name' ] ) + success = False + if not success: + log.error( 'Repository %s failed to uninstall.', name ) + results_dict[ 'repositories_failed_install' ].append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + log.debug( 'Repository %s failed to install correctly.' % str( name ) ) + else: + # Configure and run functional tests for this repository. This is equivalent to sh run_functional_tests.sh -installed + remove_install_tests() + log.debug( 'Installation of %s succeeded, running all defined functional tests.' % str( repository.name ) ) + # Generate the shed_tools_dict that specifies the location of test data contained within this repository. If the repository + # does not have a test-data directory, this will return has_test_data = False, and we will set the do_not_test flag to True, + # and the tools_functionally_correct flag to False, as well as updating tool_test_results. + file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( {} ) ) + has_test_data, shed_tools_dict = parse_tool_panel_config( galaxy_shed_tool_conf_file, + from_json_string( file( galaxy_shed_tools_dict, 'r' ).read() ) ) + if 'missing_test_components' not in repository_status_dict: + repository_status_dict[ 'missing_test_components' ] = [] + missing_tool_dependencies = get_missing_tool_dependencies( repository ) + if missing_tool_dependencies or repository.missing_repository_dependencies: + repository_status_dict, results_dict = handle_missing_dependencies( app, + repository, + metadata_revision_id, + missing_tool_dependencies, + repository_dict, + repository_status_dict, + results_dict ) + else: + # If the repository has a test-data directory we write the generated shed_tools_dict to a file, so the functional + # test framework can find it. + file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( shed_tools_dict ) ) + log.debug( 'Saved generated shed_tools_dict to %s\nContents: %s' % ( str( galaxy_shed_tools_dict ), str( shed_tools_dict ) ) ) + params, repository_status_dict, results_dict = test_repository_tools( app, + repository, + params, + metadata_revision_id, + repository_dict, + repository_status_dict, + results_dict ) + total_repositories_tested += 1 + results_dict[ 'total_repositories_tested' ] = total_repositories_tested + return results_dict, error_message + +def is_latest_downloadable_revision( url, repository_dict ): + latest_revision = get_latest_downloadable_changeset_revision( url, name=repository_dict[ 'name' ], owner=repository_dict[ 'owner' ] ) + return str( repository_dict[ 'changeset_revision' ] ) == str( latest_revision ) def json_from_url( url ): + error_message = '' url_handle = urllib.urlopen( url ) url_contents = url_handle.read() try: parsed_json = from_json_string( url_contents ) - except: - log.exception( 'Error parsing JSON data.' ) - raise - return parsed_json + except Exception, e: + error_message = str( url_contents ) + log.exception( 'Error parsing JSON data in json_from_url(): %s.' % str( e ) ) + return None, error_message + return parsed_json, error_message def parse_exclude_list( xml_filename ): - ''' - This method should return a list with the following structure: - [ - { - 'reason': The default reason or the reason specified in this section, - 'repositories': - [ - ( name, owner, changeset revision if changeset revision else None ), - ( name, owner, changeset revision if changeset revision else None ) - ] - }, - { - 'reason': The default reason or the reason specified in this section, - 'repositories': - [ - ( name, owner, changeset revision if changeset revision else None ), - ( name, owner, changeset revision if changeset revision else None ) - ] - }, - ] - ''' + """Return a list of repositories to exclude from testing.""" + # This method should return a list with the following structure: + # [{ 'reason': The default reason or the reason specified in this section, + # 'repositories': [( name, owner, changeset revision if changeset revision else None ), + # ( name, owner, changeset revision if changeset revision else None )]}] exclude_list = [] exclude_verbose = [] xml_tree = parse_xml( xml_filename ) @@ -572,17 +794,17 @@ exclude_count += 1 exclude_dict[ 'repositories' ].append( repository_tuple ) exclude_list.append( exclude_dict ) - log.debug( '%d repositories excluded from testing...', exclude_count ) + log.debug( '%s repositories excluded from testing...' % str( exclude_count ) ) if '-list_repositories' in sys.argv: for name, owner, changeset_revision in exclude_verbose: if changeset_revision: - log.debug( 'Repository %s owned by %s, changeset revision %s.', name, owner, changeset_revision ) + log.debug( 'Repository %s owned by %s, changeset revision %s.' % ( str( name ), str( owner ), str( changeset_revision ) ) ) else: - log.debug( 'Repository %s owned by %s, all revisions.', name, owner ) + log.debug( 'Repository %s owned by %s, all revisions.' % ( str( name ), str( owner ) ) ) return exclude_list -def register_test_result( url, metadata_id, test_results_dict, repository_info_dict, params ): - '''Update the repository metadata tool_test_results and appropriate flags using the API.''' +def register_test_result( url, metadata_id, test_results_dict, repository_dict, params ): + """Update the repository metadata tool_test_results and appropriate flags using the API.""" params[ 'tool_test_results' ] = test_results_dict if '-info_only' in sys.argv or 'GALAXY_INSTALL_TEST_INFO_ONLY' in os.environ: return {} @@ -594,15 +816,17 @@ return_formatted=False ) def remove_generated_tests( app ): - # Delete any configured tool functional tests from the test_toolbox.__dict__, otherwise nose will find them - # and try to re-run the tests after uninstalling the repository, which will cause false failure reports, - # since the test data has been deleted from disk by now. + """ + Delete any configured tool functional tests from the test_toolbox.__dict__, otherwise nose will find them + and try to re-run the tests after uninstalling the repository, which will cause false failure reports, + since the test data has been deleted from disk by now. + """ tests_to_delete = [] tools_to_delete = [] global test_toolbox for key in test_toolbox.__dict__: if key.startswith( 'TestForTool_' ): - log.info( 'Tool test found in test_toolbox, deleting: %s', key ) + log.debug( 'Tool test found in test_toolbox, deleting: %s' % str( key ) ) # We can't delete this test just yet, we're still iterating over __dict__. tests_to_delete.append( key ) tool_id = key.replace( 'TestForTool_', '' ) @@ -615,15 +839,17 @@ for tool in tools_to_delete: del app.toolbox.tools_by_id[ tool ] -def remove_install_tests( app ): - # Delete any configured repository installation tests from the test_toolbox.__dict__, otherwise nose will find them - # and try to install the repository again while running tool functional tests. +def remove_install_tests(): + """ + Delete any configured repository installation tests from the test_toolbox.__dict__, otherwise nose will find them + and try to install the repository again while running tool functional tests. + """ tests_to_delete = [] global test_toolbox # Push all the toolbox tests to module level for key in test_install_repositories.__dict__: if key.startswith( 'TestInstallRepository_' ): - log.info( 'Repository installation process found, deleting: %s', key ) + log.debug( 'Repository installation process found, deleting: %s' % str( key ) ) # We can't delete this test just yet, we're still iterating over __dict__. tests_to_delete.append( key ) for key in tests_to_delete: @@ -646,9 +872,9 @@ result = test_runner.run( tests ) return result, test_config.plugins._plugins -def show_summary_output( repository_info_dicts ): +def show_summary_output( repository_dicts ): repositories_by_owner = {} - for repository in repository_info_dicts: + for repository in repository_dicts: if repository[ 'owner' ] not in repositories_by_owner: repositories_by_owner[ repository[ 'owner' ] ] = [] repositories_by_owner[ repository[ 'owner' ] ].append( repository ) @@ -657,29 +883,106 @@ for repository in repositories_by_owner[ owner ]: print "# %s owned by %s, changeset revision %s" % ( repository[ 'name' ], repository[ 'owner' ], repository[ 'changeset_revision' ] ) -def uninstall_repository( app, repository_info_dict ): +def test_repository_tools( app, repository, params, metadata_revision_id, repository_dict, repository_status_dict, results_dict ): + name = str( repository.name ) + owner = str( repository.owner ) + changeset_revision = str( repository.changeset_revision ) + # Set the module-level variable 'toolbox', so that test.functional.test_toolbox will generate the appropriate test methods. + test_toolbox.toolbox = app.toolbox + # Generate the test methods for this installed repository. We need to pass in True here, or it will look + # in $GALAXY_HOME/test-data for test data, which may result in missing or invalid test files. + test_toolbox.build_tests( testing_shed_tools=True, master_api_key=default_galaxy_master_api_key ) + # Set up nose to run the generated functional tests. + test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) + test_config.configure( sys.argv ) + # Run the configured tests. + result, test_plugins = run_tests( test_config ) + success = result.wasSuccessful() + # Use the ReportResults nose plugin to get a list of tests that passed. + for plugin in test_plugins: + if hasattr( plugin, 'getTestStatus' ): + test_identifier = '%s/%s' % ( owner, name ) + passed_tests = plugin.getTestStatus( test_identifier ) + break + repository_status_dict[ 'passed_tests' ] = [] + for test_id in passed_tests: + # Normalize the tool ID and version display. + tool_id, tool_version = get_tool_info_from_test_id( test_id ) + test_result = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version ) + repository_status_dict[ 'passed_tests' ].append( test_result ) + if success: + # This repository's tools passed all functional tests. Update the repository_metadata table in the tool shed's database + # to reflect that. Call the register_test_result method, which executes a PUT request to the repository_revisions API + # controller with the status of the test. This also sets the do_not_test and tools_functionally correct flags, and + # updates the time_last_tested field to today's date. + results_dict[ 'repositories_passed' ].append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + params[ 'tools_functionally_correct' ] = True + params[ 'do_not_test' ] = False + params[ 'test_install_error' ] = False + register_test_result( galaxy_tool_shed_url, + metadata_revision_id, + repository_status_dict, + repository_dict, + params ) + log.debug( 'Revision %s of repository %s installed and passed functional tests.' % ( str( changeset_revision ), str( name ) ) ) + else: + repository_status_dict[ 'failed_tests' ] = extract_log_data( result, from_tool_test=True ) + # Call the register_test_result method, which executes a PUT request to the repository_revisions API controller with the outcome + # of the tests, and updates tool_test_results with the relevant log data. + # This also sets the do_not_test and tools_functionally correct flags to the appropriate values, and updates the time_last_tested + # field to today's date. + results_dict[ 'repositories_failed' ].append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_dict ) + params[ 'tools_functionally_correct' ] = False + params[ 'test_install_error' ] = False + params[ 'do_not_test' ] = str( set_do_not_test ) + register_test_result( galaxy_tool_shed_url, + metadata_revision_id, + repository_status_dict, + repository_dict, + params ) + log.debug( 'Revision %s of repository %s installed successfully but did not pass functional tests.' % \ + ( str( changeset_revision ), str( name ) ) ) + # Run the uninstall method. This removes tool functional test methods from the test_toolbox module and uninstalls the + # repository using Twill. + deactivate = asbool( os.environ.get( 'GALAXY_INSTALL_TEST_KEEP_TOOL_DEPENDENCIES', False ) ) + if deactivate: + log.debug( 'Deactivating changeset revision %s of repository %s' % ( str( changeset_revision ), str( name ) ) ) + # We are deactivating this repository and all of its repository dependencies. + success = deactivate_repository( app, repository_dict ) + else: + log.debug( 'Uninstalling changeset revision %s of repository %s' % ( str( changeset_revision ), str( name ) ) ) + # We are uninstalling this repository and all of its repository dependencies. + success = uninstall_repository( app, repository_dict ) + if not success: + log.debug( 'Repository %s failed to uninstall.' % str( name ) ) + # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. + test_toolbox.toolbox = app.toolbox + return params, repository_status_dict, results_dict + +def uninstall_repository( app, repository_dict ): # Clean out any generated tests. This is necessary for Twill. remove_generated_tests( app ) sa_session = app.model.context.current # The dict contains the only repository the app should have installed at this point. - repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( repository_info_dict[ 'name' ], - repository_info_dict[ 'owner' ], - repository_info_dict[ 'changeset_revision' ] ) + repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( repository_dict[ 'name' ], + repository_dict[ 'owner' ], + repository_dict[ 'changeset_revision' ] ) # We have to do this through Twill, in order to maintain app.toolbox and shed_tool_conf.xml in a state that is valid for future tests. for required_repository in repository.repository_dependencies: - repository_dict = dict( name=required_repository.name, - owner=required_repository.owner, - changeset_revision=required_repository.changeset_revision ) + repository_dict = dict( name=str( required_repository.name ), + owner=str( required_repository.owner ), + changeset_revision=str( required_repository.changeset_revision ) ) # Generate a test method to uninstall this repository through the embedded Galaxy application's web interface. test_install_repositories.generate_deactivate_or_uninstall_method( repository_dict, deactivate=False ) log.debug( 'Changeset revision %s of %s repository %s selected for uninstallation.' % \ - ( required_repository.changeset_revision, required_repository.status, required_repository.name ) ) - repository_dict = dict( name=repository.name, - owner=repository.owner, - changeset_revision=repository.changeset_revision ) + ( str( required_repository.changeset_revision ), str( required_repository.status ), str( required_repository.name ) ) ) + repository_dict = dict( name=str( repository.name ), + owner=str( repository.owner ), + changeset_revision=str( repository.changeset_revision ) ) test_install_repositories.generate_deactivate_or_uninstall_method( repository_dict, deactivate=False ) log.debug( 'Changeset revision %s of %s repository %s selected for uninstallation.' % \ - ( repository.changeset_revision, repository.status, repository.name ) ) + ( str( repository.changeset_revision ), str( repository.status ), str( repository.name ) ) ) # Set up nose to run the generated uninstall method as a functional test. test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) test_config.configure( sys.argv ) @@ -708,11 +1011,16 @@ suc.remove_dir( tool_dependency_install_path ) def main(): + if tool_shed_api_key is None: + # If the tool shed URL specified in any dict is not present in the tool_sheds_conf.xml, the installation will fail. + log.debug( 'Cannot proceed without a valid tool shed API key set in the enviroment variable GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY.' ) + return 1 # ---- Configuration ------------------------------------------------------ - # TODO: refactor this very large main method into smaller, more maintainable components. galaxy_test_host = os.environ.get( 'GALAXY_INSTALL_TEST_HOST', default_galaxy_test_host ) + # Set the GALAXY_INSTALL_TEST_HOST variable so that Twill will have the Galaxy url to install repositories into. + os.environ[ 'GALAXY_INSTALL_TEST_HOST' ] = galaxy_test_host galaxy_test_port = os.environ.get( 'GALAXY_INSTALL_TEST_PORT', str( default_galaxy_test_port_max ) ) - + os.environ[ 'GALAXY_TEST_PORT' ] = galaxy_test_port tool_path = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_PATH', 'tools' ) if 'HTTP_ACCEPT_LANGUAGE' not in os.environ: os.environ[ 'HTTP_ACCEPT_LANGUAGE' ] = default_galaxy_locales @@ -722,9 +1030,6 @@ use_distributed_object_store = os.environ.get( 'GALAXY_INSTALL_TEST_USE_DISTRIBUTED_OBJECT_STORE', False ) if not os.path.isdir( galaxy_test_tmp_dir ): os.mkdir( galaxy_test_tmp_dir ) - galaxy_test_proxy_port = None - # Allow the option to keep or delete tool dependencies when a repository has been tested. - deactivate = asbool( os.environ.get( 'GALAXY_INSTALL_TEST_KEEP_TOOL_DEPENDENCIES', False ) ) # Set up the configuration files for the Galaxy instance. shed_tool_data_table_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_SHED_TOOL_DATA_TABLE_CONF', os.path.join( galaxy_test_tmp_dir, 'test_shed_tool_data_table_conf.xml' ) ) galaxy_tool_data_table_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_DATA_TABLE_CONF', tool_data_table_conf ) @@ -735,6 +1040,9 @@ galaxy_tool_sheds_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF', os.path.join( galaxy_test_tmp_dir, 'test_tool_sheds_conf.xml' ) ) galaxy_shed_tools_dict = os.environ.get( 'GALAXY_INSTALL_TEST_SHED_TOOL_DICT_FILE', os.path.join( galaxy_test_tmp_dir, 'shed_tool_dict' ) ) file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( {} ) ) + # Set the GALAXY_TOOL_SHED_TEST_FILE environment variable to the path of the shed_tools_dict file, so that test.base.twilltestcase.setUp + # will find and parse it properly. + os.environ[ 'GALAXY_TOOL_SHED_TEST_FILE' ] = galaxy_shed_tools_dict if 'GALAXY_INSTALL_TEST_TOOL_DATA_PATH' in os.environ: tool_data_path = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_DATA_PATH' ) else: @@ -768,9 +1076,7 @@ os.makedirs( dir ) except OSError: pass - print "Database connection: ", database_connection - # Generate the shed_tool_data_table_conf.xml file. file( shed_tool_data_table_conf_file, 'w' ).write( tool_data_table_conf_xml_template ) os.environ[ 'GALAXY_INSTALL_TEST_SHED_TOOL_DATA_TABLE_CONF' ] = shed_tool_data_table_conf_file @@ -839,9 +1145,7 @@ kwargs[ 'database_engine_option_pool_size' ] = '10' kwargs[ 'config_file' ] = galaxy_config_file app = UniverseApplication( **kwargs ) - - log.info( "Embedded Galaxy application started" ) - + log.debug( "Embedded Galaxy application started..." ) # ---- Run galaxy webserver ------------------------------------------------------ server = None global_conf = get_webapp_global_conf() @@ -850,7 +1154,6 @@ use_translogger=False, static_enabled=STATIC_ENABLED, app=app ) - # Serve the app on a specified or random port. if galaxy_test_port is not None: server = httpserver.serve( webapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False ) @@ -869,10 +1172,7 @@ else: raise Exception( "Unable to open a port between %s and %s to start Galaxy server" % \ ( default_galaxy_test_port_min, default_galaxy_test_port_max ) ) - if galaxy_test_proxy_port: - os.environ[ 'GALAXY_INSTALL_TEST_PORT' ] = galaxy_test_proxy_port - else: - os.environ[ 'GALAXY_INSTALL_TEST_PORT' ] = galaxy_test_port + os.environ[ 'GALAXY_INSTALL_TEST_PORT' ] = galaxy_test_port # Start the server. t = threading.Thread( target=server.serve_forever ) t.start() @@ -886,9 +1186,9 @@ time.sleep( 0.1 ) else: raise Exception( "Test HTTP server did not return '200 OK' after 10 tries" ) - log.info( "Embedded galaxy web server started" ) - log.info( "The embedded Galaxy application is running on %s:%s", galaxy_test_host, galaxy_test_port ) - log.info( "Repositories will be installed from the tool shed at %s", galaxy_tool_shed_url ) + log.debug( "Embedded galaxy web server started..." ) + log.debug( "The embedded Galaxy application is running on %s:%s" % ( str( galaxy_test_host ), str( galaxy_test_port ) ) ) + log.debug( "Repositories will be installed from the tool shed at %s" % str( galaxy_tool_shed_url ) ) success = False # If a tool_data_table_conf.test.xml file was found, add the entries from it into the app's tool data tables. if additional_tool_data_tables: @@ -896,452 +1196,68 @@ tool_data_path=additional_tool_data_path, shed_tool_data_table_config=None, persist=False ) - # Initialize some variables for the summary that will be printed to stdout. - repositories_passed = [] - repositories_failed = [] - repositories_failed_install = [] - exclude_list = [] - if os.path.exists( exclude_list_file ): - log.info( 'Loading the list of repositories excluded from testing from the file %s...', exclude_list_file ) - exclude_list = parse_exclude_list( exclude_list_file ) - try: - # Get a list of repositories to test from the tool shed specified in the GALAXY_INSTALL_TEST_TOOL_SHED_URL environment variable. - log.info( "Retrieving repositories to install from the URL:\n%s\n", str( galaxy_tool_shed_url ) ) - if '-check_all_revisions' not in sys.argv: - repositories_to_install = get_repositories_to_install( galaxy_tool_shed_url, latest_revision_only=True ) + now = strftime( "%Y-%m-%d %H:%M:%S" ) + print "####################################################################################" + print "# %s - running repository installation and testing script." % now + print "####################################################################################" + results_dict, error_message = install_and_test_repositories( app, galaxy_shed_tools_dict, galaxy_shed_tool_conf_file ) + if error_message: + log.debug( error_message ) + else: + total_repositories_tested = results_dict[ 'total_repositories_tested' ] + repositories_passed = results_dict[ 'repositories_passed' ] + repositories_failed = results_dict[ 'repositories_failed' ] + repositories_failed_install = results_dict[ 'repositories_failed_install' ] + exclude_list = results_dict[ 'exclude_list' ] + now = strftime( "%Y-%m-%d %H:%M:%S" ) + print "####################################################################################" + print "# %s - repository installation and testing script completed." % now + print "# Repository revisions tested: %s" % str( total_repositories_tested ) + if '-info_only' in sys.argv: + print "# -info_only set, not updating the tool shed." + if total_repositories_tested > 0: + if repositories_passed: + print '# ----------------------------------------------------------------------------------' + print "# %d repositories passed all tests:" % len( repositories_passed ) + show_summary_output( repositories_passed ) + if repositories_failed: + print '# ----------------------------------------------------------------------------------' + print "# %d repositories failed one or more tests:" % len( repositories_failed ) + show_summary_output( repositories_failed ) + if repositories_failed_install: + # Set success to False so that the return code will not be 0. + success = False + print '# ----------------------------------------------------------------------------------' + print "# %d repositories with installation errors:" % len( repositories_failed_install ) + show_summary_output( repositories_failed_install ) + else: + success = True else: - repositories_to_install = get_repositories_to_install( galaxy_tool_shed_url, latest_revision_only=False ) - log.info( "Retrieved %d repositories from the API.", len( repositories_to_install ) ) - if '-list_repositories' in sys.argv: - log.info( "The API returned the following repositories, not counting deleted:" ) - for repository_info_dict in repositories_to_install: - log.info( "%s owned by %s changeset revision %s", - repository_info_dict.get( 'name', None ), - repository_info_dict.get( 'owner', None ), - repository_info_dict.get( 'changeset_revision', None ) ) - repositories_tested = len( repositories_to_install ) - # This loop will iterate through the list of repositories generated by the above code, having already filtered out any - # that were marked as deleted. For each repository, it will generate a test method that will use Twill to install that - # repository into the embedded Galaxy application that was started up, selecting to install repository and tool - # dependencies if they are defined. If the installation completes successfully, it will then generate a test case for - # each functional test defined for each tool in the repository, and execute the generated test cases. When this completes, - # it will record the result of the tests, and if any failed, the traceback and captured output of the tool that was run. - # After all tests have completed, the repository is uninstalled, so that the previous test cases don't interfere with - # the next repository's functional tests. - for repository_info_dict in repositories_to_install: - """ - Each repository_info_dict looks something like: - { - "changeset_revision": "13fa22a258b5", - "contents_url": "/api/repositories/529fd61ab1c6cc36/contents", - "deleted": false, - "deprecated": false, - "description": "Convert column case.", - "downloadable": true, - "id": "529fd61ab1c6cc36", - "long_description": "This tool takes the specified columns and converts them to uppercase or lowercase.", - "malicious": false, - "name": "change_case", - "owner": "test", - "private": false, - "repository_id": "529fd61ab1c6cc36", - "times_downloaded": 0, - "tool_shed_url": "http://toolshed.local:10001", - "url": "/api/repository_revisions/529fd61ab1c6cc36", - "user_id": "529fd61ab1c6cc36" - } - """ - repository_status = {} - params = {} - repository_id = str( repository_info_dict.get( 'repository_id', None ) ) - changeset_revision = str( repository_info_dict.get( 'changeset_revision', None ) ) - metadata_revision_id = repository_info_dict.get( 'id', None ) - # Add the URL for the tool shed we're installing from, so the automated installation methods go to the right place. - repository_info_dict[ 'tool_shed_url' ] = galaxy_tool_shed_url - # Get the name and owner out of the repository info dict. - name = str( repository_info_dict[ 'name' ] ) - owner = str( repository_info_dict[ 'owner' ] ) - # Populate the repository_status dict now. - repository_status = get_tool_test_results_from_api( galaxy_tool_shed_url, metadata_revision_id ) - if 'test_environment' not in repository_status: - repository_status[ 'test_environment' ] = {} - test_environment = get_test_environment( repository_status[ 'test_environment' ] ) - test_environment[ 'galaxy_database_version' ] = get_database_version( app ) - test_environment[ 'galaxy_revision'] = get_repository_current_revision( os.getcwd() ) - repository_status[ 'test_environment' ] = test_environment - repository_status[ 'passed_tests' ] = [] - repository_status[ 'failed_tests' ] = [] - repository_status[ 'skip_reason' ] = None - # Iterate through the list of repositories defined not to be installed. This should be a list of dicts in the following format: - # { - # 'reason': The default reason or the reason specified in this section, - # 'repositories': - # [ - # ( name, owner, changeset revision if changeset revision else None ), - # ( name, owner, changeset revision if changeset revision else None ) - # ] - # }, - # If changeset revision is None, that means the entire repository is excluded from testing, otherwise only the specified - # revision should be skipped. - # TODO: When a repository is selected to be skipped, use the API to update the tool shed with the defined skip reason. - skip_this_repository = False - skip_because = None - for exclude_by_reason in exclude_list: - reason = exclude_by_reason[ 'reason' ] - exclude_repositories = exclude_by_reason[ 'repositories' ] - if ( name, owner, changeset_revision ) in exclude_repositories or ( name, owner, None ) in exclude_repositories: - skip_this_repository = True - skip_because = reason - break - if skip_this_repository: - repository_status[ 'not_tested' ] = dict( reason=skip_because ) - params[ 'tools_functionally_correct' ] = False - params[ 'do_not_test' ] = False - register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, repository_info_dict, params ) - log.info( "Not testing revision %s of repository %s owned by %s.", changeset_revision, name, owner ) - continue - else: - log.info( "Installing and testing revision %s of repository %s owned by %s...", changeset_revision, name, owner ) - # Explicitly clear tests from twill's test environment. - remove_generated_tests( app ) - # Use the repository information dict to generate an install method that will install the repository into the embedded - # Galaxy application, with tool dependencies and repository dependencies, if any. - test_install_repositories.generate_install_method( repository_info_dict ) - # Set the GALAXY_INSTALL_TEST_HOST variable so that Twill will have the Galaxy url to install repositories into. - os.environ[ 'GALAXY_INSTALL_TEST_HOST' ] = galaxy_test_host - # Configure nose to run the install method as a test. - test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) - test_config.configure( sys.argv ) - # Run the configured install method as a test. This method uses the embedded Galaxy application's web interface to install the specified - # repository, with tool and repository dependencies also selected for installation. - result, _ = run_tests( test_config ) - success = result.wasSuccessful() - repository_status[ 'installation_errors' ] = dict( current_repository=[], repository_dependencies=[], tool_dependencies=[] ) - try: - repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) - except: - log.exception( 'Error getting installed repository.' ) - success = False - pass - # If the installation succeeds, configure and run functional tests for this repository. This is equivalent to - # sh run_functional_tests.sh -installed - if success: - # Clean out repository installation methods. - remove_install_tests( app ) - log.debug( 'Installation of %s succeeded, running all defined functional tests.', name ) - # Generate the shed_tools_dict that specifies the location of test data contained within this repository. If the repository - # does not have a test-data directory, this will return has_test_data = False, and we will set the do_not_test flag to True, - # and the tools_functionally_correct flag to False, as well as updating tool_test_results. - file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( {} ) ) - has_test_data, shed_tools_dict = parse_tool_panel_config( galaxy_shed_tool_conf_file, from_json_string( file( galaxy_shed_tools_dict, 'r' ).read() ) ) - # The repository_status dict should always have the following structure: - # { - # "test_environment": - # { - # "galaxy_revision": "9001:abcd1234", - # "galaxy_database_version": "114", - # "tool_shed_revision": "9001:abcd1234", - # "tool_shed_mercurial_version": "2.3.1", - # "tool_shed_database_version": "17", - # "python_version": "2.7.2", - # "architecture": "x86_64", - # "system": "Darwin 12.2.0" - # }, - # "passed_tests": - # [ - # { - # "test_id": "The test ID, generated by twill", - # "tool_id": "The tool ID that was tested", - # "tool_version": "The tool version that was tested", - # }, - # ] - # "failed_tests": - # [ - # { - # "test_id": "The test ID, generated by twill", - # "tool_id": "The tool ID that was tested", - # "tool_version": "The tool version that was tested", - # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." - # "traceback": "The captured traceback." - # }, - # ] - # "installation_errors": - # { - # 'tool_dependencies': - # [ - # { - # 'type': 'Type of tool dependency, e.g. package, set_environment, etc.', - # 'name': 'Name of the tool dependency.', - # 'version': 'Version if this is a package, otherwise blank.', - # 'error_message': 'The error message returned when installation was attempted.', - # }, - # ], - # 'repository_dependencies': - # [ - # { - # 'tool_shed': 'The tool shed that this repository was installed from.', - # 'name': 'The name of the repository that failed to install.', - # 'owner': 'Owner of the failed repository.', - # 'changeset_revision': 'Changeset revision of the failed repository.', - # 'error_message': 'The error message that was returned when the repository failed to install.', - # }, - # ], - # 'current_repository': - # [ - # { - # 'tool_shed': 'The tool shed that this repository was installed from.', - # 'name': 'The name of the repository that failed to install.', - # 'owner': 'Owner of the failed repository.', - # 'changeset_revision': 'Changeset revision of the failed repository.', - # 'error_message': 'The error message that was returned when the repository failed to install.', - # }, - # ], - # { - # "name": "The name of the repository.", - # "owner": "The owner of the repository.", - # "changeset_revision": "The changeset revision of the repository.", - # "error_message": "The message stored in tool_dependency.error_message." - # }, - # } - # "missing_test_components": - # [ - # { - # "tool_id": "The tool ID that missing components.", - # "tool_version": "The version of the tool." - # "tool_guid": "The guid of the tool." - # "missing_components": "Which components are missing, e.g. the test data filename, or the test-data directory." - # }, - # ] - # "not_tested": - # { - # "reason": "The Galaxy development team has determined that this repository should not be installed and tested by the automated framework." - # } - # } - if 'missing_test_components' not in repository_status: - repository_status[ 'missing_test_components' ] = [] - missing_tool_dependencies = get_missing_tool_dependencies( repository ) - if missing_tool_dependencies or repository.missing_repository_dependencies: - # If a tool dependency fails to install correctly, this should be considered an installation error, - # and functional tests should be skipped, since the tool dependency needs to be correctly installed - # for the test to be considered reliable. - log.error( 'The following dependencies of this repository are missing, skipping functional tests.' ) - # In keeping with the standard display layout, add the error message to the dict for each tool individually. - for dependency in repository.missing_tool_dependencies: - log.error( 'Missing tool dependency %s of type %s version %s: %s' % \ - ( str( dependency.name ), str( dependency.type ), str( dependency.version ), unicodify( dependency.error_message ) ) ) - test_result = dict( type=dependency.type, - name=dependency.name, - version=dependency.version, - error_message=dependency.error_message ) - repository_status[ 'installation_errors' ][ 'tool_dependencies' ].append( test_result ) - for dependency in repository.missing_repository_dependencies: - log.error( 'Missing repository dependency %s changeset revision %s owned by %s: %s' % \ - ( str( dependency.name ), str( dependency.changeset_revision ), str( dependency.owner ), unicodify( dependency.error_message ) ) ) - test_result = dict( tool_shed=dependency.tool_shed, - name=dependency.name, - owner=dependency.owner, - changeset_revision=dependency.changeset_revision, - error_message=dependency.error_message ) - repository_status[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) - # Record the status of this repository in the tool shed. - params[ 'tools_functionally_correct' ] = False - params[ 'do_not_test' ] = False - params[ 'test_install_error' ] = True - register_test_result( galaxy_tool_shed_url, - metadata_revision_id, - repository_status, - repository_info_dict, - params ) - # Since this repository is missing components, we do not want to test it. Deactivate this repository. - # The deactivate flag is set to True if the environment variable GALAXY_INSTALL_TEST_KEEP_TOOL_DEPENDENCIES - # is set to 'true'. - if deactivate: - log.debug( 'Due to the above missing tool dependencies, we are now uninstalling the following tool dependencies, but not changing their repositories.' ) - for missing_tool_dependency in missing_tool_dependencies: - uninstall_tool_dependency( app, missing_tool_dependency ) - # We are deactivating this repository and all of its repository dependencies. - log.debug( 'Due to the above missing repository dependencies, we are now deactivating the following repositories.' ) - deactivate_repository( app, repository_info_dict ) - else: - # We are uninstalling this repository and all of its repository dependencies. - log.debug( 'Due to the above missing repository dependencies, we are now uninstalling the following repositories.' ) - uninstall_repository( app, repository_info_dict ) - # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. - test_toolbox.toolbox = app.toolbox - repositories_failed_install.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) - else: - # If the repository does have a test-data directory, we write the generated shed_tools_dict to a file, so the functional - # test framework can find it. - file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( shed_tools_dict ) ) - log.info( 'Saved generated shed_tools_dict to %s\nContents: %s', galaxy_shed_tools_dict, str( shed_tools_dict ) ) - # Set the GALAXY_TOOL_SHED_TEST_FILE environment variable to the path of the shed_tools_dict file, so that test.base.twilltestcase.setUp - # will find and parse it properly. - os.environ[ 'GALAXY_TOOL_SHED_TEST_FILE' ] = galaxy_shed_tools_dict - os.environ[ 'GALAXY_TEST_HOST' ] = galaxy_test_host - os.environ[ 'GALAXY_TEST_PORT' ] = galaxy_test_port - # Set the module-level variable 'toolbox', so that test.functional.test_toolbox will generate the appropriate test methods. - test_toolbox.toolbox = app.toolbox - # Generate the test methods for this installed repository. We need to pass in True here, or it will look - # in $GALAXY_HOME/test-data for test data, which may result in missing or invalid test files. - test_toolbox.build_tests( testing_shed_tools=True, master_api_key=default_galaxy_master_api_key ) - # Set up nose to run the generated functional tests. - test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) - test_config.configure( sys.argv ) - # Run the configured tests. - result, test_plugins = run_tests( test_config ) - success = result.wasSuccessful() - # Use the ReportResults nose plugin to get a list of tests that passed. - for plugin in test_plugins: - if hasattr( plugin, 'getTestStatus' ): - test_identifier = '%s/%s' % ( owner, name ) - passed_tests = plugin.getTestStatus( test_identifier ) - break - repository_status[ 'passed_tests' ] = [] - for test_id in passed_tests: - # Normalize the tool ID and version display. - tool_id, tool_version = get_tool_info_from_test_id( test_id ) - test_result = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version ) - repository_status[ 'passed_tests' ].append( test_result ) - if success: - # This repository's tools passed all functional tests. Update the repository_metadata table in the tool shed's database - # to reflect that. Call the register_test_result method, which executes a PUT request to the repository_revisions API - # controller with the status of the test. This also sets the do_not_test and tools_functionally correct flags, and - # updates the time_last_tested field to today's date. - repositories_passed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) - params[ 'tools_functionally_correct' ] = True - params[ 'do_not_test' ] = False - params[ 'test_install_error' ] = False - register_test_result( galaxy_tool_shed_url, - metadata_revision_id, - repository_status, - repository_info_dict, - params ) - log.debug( 'Revision %s of repository %s installed and passed functional tests.', changeset_revision, name ) - else: - repository_status[ 'failed_tests' ] = extract_log_data( result, from_tool_test=True ) - # Call the register_test_result method, which executes a PUT request to the repository_revisions API controller with the outcome - # of the tests, and updates tool_test_results with the relevant log data. - # This also sets the do_not_test and tools_functionally correct flags to the appropriate values, and updates the time_last_tested - # field to today's date. - repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) - set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_info_dict ) - params[ 'tools_functionally_correct' ] = False - params[ 'test_install_error' ] = False - params[ 'do_not_test' ] = str( set_do_not_test ) - register_test_result( galaxy_tool_shed_url, - metadata_revision_id, - repository_status, - repository_info_dict, - params ) - log.debug( 'Revision %s of repository %s installed successfully, but did not pass functional tests.', - changeset_revision, name ) - # Run the uninstall method. This removes tool functional test methods from the test_toolbox module and uninstalls the - # repository using Twill. - if deactivate: - log.debug( 'Deactivating changeset revision %s of repository %s', - repository_info_dict[ 'changeset_revision' ], - repository_info_dict[ 'name' ] ) - # We are deactivating this repository and all of its repository dependencies. - success = deactivate_repository( app, repository_info_dict ) - else: - log.debug( 'Uninstalling changeset revision %s of repository %s', - repository_info_dict[ 'changeset_revision' ], - repository_info_dict[ 'name' ] ) - # We are uninstalling this repository and all of its repository dependencies. - success = uninstall_repository( app, repository_info_dict ) - if not success: - log.error( 'Repository %s failed to uninstall.', repository_info_dict[ 'name' ] ) - # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. - test_toolbox.toolbox = app.toolbox - else: - # Even if the repository failed to install, execute the uninstall method, in case a dependency did succeed. - log.debug( 'Uninstalling repository %s', repository_info_dict[ 'name' ] ) - try: - repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) - except: - log.exception( 'Unable to uninstall, no installed repository found.' ) - test_result = dict( tool_shed=galaxy_tool_shed_url, - name=repository_info_dict[ 'name' ], - owner=repository_info_dict[ 'owner' ], - changeset_revision=repository_info_dict[ 'changeset_revision' ], - error_message=extract_log_data( result, from_tool_test=False ) ) - repository_status[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) - params[ 'tools_functionally_correct' ] = False - params[ 'test_install_error' ] = True - params[ 'do_not_test' ] = False - register_test_result( galaxy_tool_shed_url, - metadata_revision_id, - repository_status, - repository_info_dict, - params ) - try: - if deactivate: - # We are deactivating this repository and all of its repository dependencies. - success = deactivate_repository( app, repository_info_dict ) - else: - # We are uninstalling this repository and all of its repository dependencies. - success = uninstall_repository( app, repository_info_dict ) - except: - log.exception( 'Encountered error attempting to deactivate or uninstall %s.', repository_info_dict[ 'name' ] ) - success = False - if not success: - log.error( 'Repository %s failed to uninstall.', repository_info_dict[ 'name' ] ) - repositories_failed_install.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) - log.debug( 'Repository %s failed to install correctly.', repository_info_dict[ 'name' ] ) - except: - log.exception( "Failure running tests" ) - - log.info( "Shutting down" ) - # ---- Tear down ----------------------------------------------------------- + success = True + print "####################################################################################" + log.debug( "Shutting down..." ) # Gracefully shut down the embedded web server and UniverseApplication. if server: - log.info( "Shutting down embedded galaxy web server" ) + log.debug( "Shutting down embedded galaxy web server..." ) server.server_close() server = None - log.info( "Embedded galaxy server stopped" ) + log.debug( "Embedded galaxy server stopped..." ) if app: - log.info( "Shutting down galaxy application" ) + log.debug( "Shutting down galaxy application..." ) app.shutdown() app = None - log.info( "Embedded galaxy application stopped" ) + log.debug( "Embedded galaxy application stopped..." ) # Clean up test files unless otherwise specified. if 'GALAXY_INSTALL_TEST_NO_CLEANUP' not in os.environ: - try: - for dir in [ galaxy_test_tmp_dir ]: - if os.path.exists( dir ): - log.info( "Cleaning up temporary files in %s", dir ) + for dir in [ galaxy_test_tmp_dir ]: + if os.path.exists( dir ): + try: shutil.rmtree( dir ) - except: - pass + log.debug( "Cleaned up temporary files in %s", dir ) + except: + pass else: log.debug( 'GALAXY_INSTALL_TEST_NO_CLEANUP set, not cleaning up.' ) - - now = strftime( "%Y-%m-%d %H:%M:%S" ) - print "####################################################################################" - print "# %s - repository installation and testing script completed." % now - print "# Repository revisions tested: %d" % repositories_tested - if '-info_only' in sys.argv: - print "# -info_only set, not updating the tool shed." - if repositories_tested > 0: - if repositories_passed: - print '# ----------------------------------------------------------------------------------' - print "# %d repositories passed all tests:" % len( repositories_passed ) - show_summary_output( repositories_passed ) - if repositories_failed: - print '# ----------------------------------------------------------------------------------' - print "# %d repositories failed one or more tests:" % len( repositories_failed ) - show_summary_output( repositories_failed ) - if repositories_failed_install: - # Set success to False so that the return code will not be 0. - success = False - print '# ----------------------------------------------------------------------------------' - print "# %d repositories with installation errors:" % len( repositories_failed_install ) - show_summary_output( repositories_failed_install ) - else: - success = True - else: - success = True - print "####################################################################################" # Normally, the value of 'success' would determine whether this test suite is marked as passed or failed # in the automated buildbot framework. However, due to the procedure used here, we only want to report # failure if a repository fails to install correctly. Therefore, we have overriden the value of 'success' @@ -1352,8 +1268,87 @@ return 1 if __name__ == "__main__": - now = strftime( "%Y-%m-%d %H:%M:%S" ) - print "####################################################################################" - print "# %s - running repository installation and testing script." % now - print "####################################################################################" + # The repository_status_dict should always have the following structure: + # { + # "test_environment": + # { + # "galaxy_revision": "9001:abcd1234", + # "galaxy_database_version": "114", + # "tool_shed_revision": "9001:abcd1234", + # "tool_shed_mercurial_version": "2.3.1", + # "tool_shed_database_version": "17", + # "python_version": "2.7.2", + # "architecture": "x86_64", + # "system": "Darwin 12.2.0" + # }, + # "passed_tests": + # [ + # { + # "test_id": "The test ID, generated by twill", + # "tool_id": "The tool ID that was tested", + # "tool_version": "The tool version that was tested", + # }, + # ] + # "failed_tests": + # [ + # { + # "test_id": "The test ID, generated by twill", + # "tool_id": "The tool ID that was tested", + # "tool_version": "The tool version that was tested", + # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." + # "traceback": "The captured traceback." + # }, + # ] + # "installation_errors": + # { + # 'tool_dependencies': + # [ + # { + # 'type': 'Type of tool dependency, e.g. package, set_environment, etc.', + # 'name': 'Name of the tool dependency.', + # 'version': 'Version if this is a package, otherwise blank.', + # 'error_message': 'The error message returned when installation was attempted.', + # }, + # ], + # 'repository_dependencies': + # [ + # { + # 'tool_shed': 'The tool shed that this repository was installed from.', + # 'name': 'The name of the repository that failed to install.', + # 'owner': 'Owner of the failed repository.', + # 'changeset_revision': 'Changeset revision of the failed repository.', + # 'error_message': 'The error message that was returned when the repository failed to install.', + # }, + # ], + # 'current_repository': + # [ + # { + # 'tool_shed': 'The tool shed that this repository was installed from.', + # 'name': 'The name of the repository that failed to install.', + # 'owner': 'Owner of the failed repository.', + # 'changeset_revision': 'Changeset revision of the failed repository.', + # 'error_message': 'The error message that was returned when the repository failed to install.', + # }, + # ], + # { + # "name": "The name of the repository.", + # "owner": "The owner of the repository.", + # "changeset_revision": "The changeset revision of the repository.", + # "error_message": "The message stored in tool_dependency.error_message." + # }, + # } + # "missing_test_components": + # [ + # { + # "tool_id": "The tool ID that missing components.", + # "tool_version": "The version of the tool." + # "tool_guid": "The guid of the tool." + # "missing_components": "Which components are missing, e.g. the test data filename, or the test-data directory." + # }, + # ] + # "not_tested": + # { + # "reason": "The Galaxy development team has determined that this repository should not be installed and tested by the automated framework." + # } + # } sys.exit( main() ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: UI, History panel: fix assembly of popupmenus when rendering only body of hdas
by commits-noreply＠bitbucket.org 22 Nov '13

22 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d28d25d66095/ Changeset: d28d25d66095 User: carlfeberhard Date: 2013-11-22 18:53:30 Summary: UI, History panel: fix assembly of popupmenus when rendering only body of hdas Affected #: 2 files diff -r 54377b1aff65eb892d8e4067574c0cde552a2a57 -r d28d25d66095b84af1aaee1a487ee2b27c5a07ce static/scripts/mvc/dataset/hda-base.js --- a/static/scripts/mvc/dataset/hda-base.js +++ b/static/scripts/mvc/dataset/hda-base.js @@ -263,6 +263,8 @@ if( _.isFunction( renderFn ) ){ $body = renderFn.call( this ); } + this._setUpBehaviors( $body ); + // only render the body html if it's being shown if( this.expanded ){ $body.show(); diff -r 54377b1aff65eb892d8e4067574c0cde552a2a57 -r d28d25d66095b84af1aaee1a487ee2b27c5a07ce static/scripts/packed/mvc/dataset/hda-base.js --- a/static/scripts/packed/mvc/dataset/hda-base.js +++ b/static/scripts/packed/mvc/dataset/hda-base.js @@ -1,1 +1,1 @@ -define(["mvc/dataset/hda-model"],function(b){var a=Backbone.View.extend(LoggableMixin).extend({tagName:"div",className:"dataset hda history-panel-hda",id:function(){return"hda-"+this.model.get("id")},fxSpeed:"fast",initialize:function(c){if(c.logger){this.logger=this.model.logger=c.logger}this.log(this+".initialize:",c);this.defaultPrimaryActionButtonRenderers=[this._render_showParamsButton];this.selectable=false;this.selected=false;this.expanded=c.expanded||false;this._setUpListeners()},_setUpListeners:function(){this.model.on("change",function(d,c){if(this.model.changedAttributes().state&&this.model.inReadyState()&&this.expanded&&!this.model.hasDetails()){this.model.fetch()}else{this.render()}},this)},render:function(e){e=(e===undefined)?(true):(e);var c=this;this.$el.find("[title]").tooltip("destroy");this.urls=this.model.urls();var d=$(a.templates.skeleton(this.model.toJSON()));d.find(".dataset-primary-actions").append(this._render_titleButtons());d.children(".dataset-body").replaceWith(this._render_body());this._setUpBehaviors(d);if(e){$(c).queue(function(f){this.$el.fadeOut(c.fxSpeed,f)})}$(c).queue(function(f){this.$el.empty().attr("class",c.className).addClass("state-"+c.model.get("state")).append(d.children());f()});if(e){$(c).queue(function(f){this.$el.fadeIn(c.fxSpeed,f)})}$(c).queue(function(f){this.trigger("rendered",c);if(this.model.inReadyState()){this.trigger("rendered:ready",c)}if(this.selectable){this.showSelect()}f()});return this},_setUpBehaviors:function(c){c=c||this.$el;make_popup_menus(c);c.find("[title]").tooltip({placement:"bottom"})},_render_titleButtons:function(){return[this._render_displayButton()]},_render_displayButton:function(){if((this.model.get("state")===b.HistoryDatasetAssociation.STATES.NOT_VIEWABLE)||(this.model.get("state")===b.HistoryDatasetAssociation.STATES.DISCARDED)||(this.model.get("state")===b.HistoryDatasetAssociation.STATES.NEW)||(!this.model.get("accessible"))){return null}var d={target:"galaxy_main",classes:"dataset-display"};if(this.model.get("purged")){d.disabled=true;d.title=_l("Cannot display datasets removed from disk")}else{if(this.model.get("state")===b.HistoryDatasetAssociation.STATES.UPLOAD){d.disabled=true;d.title=_l("This dataset must finish uploading before it can be viewed")}else{d.title=_l("View data");d.href=this.urls.display;var c=this;d.onclick=function(){Galaxy.frame.add({title:"Data Viewer: "+c.model.get("name"),type:"url",target:"galaxy_main",content:c.urls.display})}}}d.faIcon="fa-eye";return faIconButton(d)},_render_downloadButton:function(){if(this.model.get("purged")||!this.model.hasData()){return null}var d=this.urls,e=this.model.get("meta_files");if(_.isEmpty(e)){return $(['<a href="'+d.download+'" title="'+_l("Download")+'" class="icon-btn">','<span class="fa fa-floppy-o"></span>',"</a>"].join(""))}var f="dataset-"+this.model.get("id")+"-popup",c=['<div popupmenu="'+f+'">','<a href="'+d.download+'">',_l("Download Dataset"),"</a>","<a>"+_l("Additional Files")+"</a>",_.map(e,function(g){return['<a class="action-button" href="',d.meta_download+g.file_type,'">',_l("Download")," ",g.file_type,"</a>"].join("")}).join("\n"),"</div>",'<div class="icon-btn-group">','<a href="'+d.download+'" title="'+_l("Download")+'" class="icon-btn">','<span class="fa fa-floppy-o"></span>','</a><a class="icon-btn popup" id="'+f+'">','<span class="fa fa-caret-down"></span>',"</a>","</div>"].join("\n");return $(c)},_render_showParamsButton:function(){return faIconButton({title:_l("View details"),href:this.urls.show_params,target:"galaxy_main",faIcon:"fa-info-circle"})},_render_body:function(){var d=$('<div>Error: unknown dataset state "'+this.model.get("state")+'".</div>'),c=this["_render_body_"+this.model.get("state")];if(_.isFunction(c)){d=c.call(this)}if(this.expanded){d.show()}return d},_render_stateBodyHelper:function(c,f){f=f||[];var d=this,e=$(a.templates.body(_.extend(this.model.toJSON(),{body:c})));e.find(".dataset-actions .left").append(_.map(f,function(g){return g.call(d)}));return e},_render_body_new:function(){return this._render_stateBodyHelper("<div>"+_l("This is a new dataset and not all of its data are available yet")+"</div>")},_render_body_noPermission:function(){return this._render_stateBodyHelper("<div>"+_l("You do not have permission to view this dataset")+"</div>")},_render_body_discarded:function(){return this._render_stateBodyHelper("<div>"+_l("The job creating this dataset was cancelled before completion")+"</div>",this.defaultPrimaryActionButtonRenderers)},_render_body_queued:function(){return this._render_stateBodyHelper("<div>"+_l("This job is waiting to run")+"</div>",this.defaultPrimaryActionButtonRenderers)},_render_body_upload:function(){return this._render_stateBodyHelper("<div>"+_l("This dataset is currently uploading")+"</div>")},_render_body_setting_metadata:function(){return this._render_stateBodyHelper("<div>"+_l("Metadata is being auto-detected")+"</div>")},_render_body_running:function(){return this._render_stateBodyHelper("<div>"+_l("This job is currently running")+"</div>",this.defaultPrimaryActionButtonRenderers)},_render_body_paused:function(){return this._render_stateBodyHelper("<div>"+_l('This job is paused. Use the "Resume Paused Jobs" in the history menu to resume')+"</div>",this.defaultPrimaryActionButtonRenderers)},_render_body_error:function(){var c=['<span class="help-text">',_l("An error occurred with this dataset"),":</span>",'<div class="job-error-text">',$.trim(this.model.get("misc_info")),"</div>"].join("");if(!this.model.get("purged")){c="<div>"+this.model.get("misc_blurb")+"</div>"+c}return this._render_stateBodyHelper(c,this.defaultPrimaryActionButtonRenderers.concat([this._render_downloadButton]))},_render_body_empty:function(){return this._render_stateBodyHelper("<div>"+_l("No data")+": <i>"+this.model.get("misc_blurb")+"</i></div>",this.defaultPrimaryActionButtonRenderers)},_render_body_failed_metadata:function(){var c=$('<div class="warningmessagesmall"></div>').append($("<strong/>").text(_l("An error occurred setting the metadata for this dataset"))),d=this._render_body_ok();d.prepend(c);return d},_render_body_ok:function(){var c=this,e=$(a.templates.body(this.model.toJSON())),d=[this._render_downloadButton].concat(this.defaultPrimaryActionButtonRenderers);e.find(".dataset-actions .left").append(_.map(d,function(f){return f.call(c)}));if(this.model.isDeletedOrPurged()){return e}return e},events:{"click .dataset-title-bar":"toggleBodyVisibility","click .dataset-selector":"select"},toggleBodyVisibility:function(d,c){var e=this.$el.find(".dataset-body");c=(c===undefined)?(!e.is(":visible")):(c);if(c){this.expandBody()}else{this.collapseBody()}return false},expandBody:function(){var c=this;function d(){c.$el.children(".dataset-body").replaceWith(c._render_body());c.$el.children(".dataset-body").slideDown(c.fxSpeed,function(){c.expanded=true;c.trigger("body-expanded",c.model.get("id"))})}if(this.model.inReadyState()&&!this.model.hasDetails()){this.model.fetch({silent:true}).always(function(e){d()})}else{d()}},collapseBody:function(){var c=this;this.$el.children(".dataset-body").slideUp(c.fxSpeed,function(){c.expanded=false;c.trigger("body-collapsed",c.model.get("id"))})},showSelect:function(e){if(this.$el.find(".dataset-selector").css("display")!=="none"){return}e=(e!==undefined)?(e):(this.fxSpeed);var d=this,c=32;this.$el.queue("fx",function(f){$(this).find(".dataset-primary-actions").fadeOut(e,f)});this.$el.queue("fx",function(f){$(this).find(".dataset-selector").show().css({height:$(this).find(".dataset-title-bar").innerHeight()}).animate({width:c},e,f);d.selectable=true;d.trigger("selectable",true,d)})},hideSelect:function(c){c=(c!==undefined)?(c):(this.fxSpeed);this.selectable=false;this.trigger("selectable",false,this);this.$el.queue("fx",function(d){$(this).find(".dataset-selector").animate({width:"0px"},c,function(){$(this).hide();d()})});this.$el.queue("fx",function(d){$(this).find(".dataset-primary-actions").fadeIn(c,d)})},select:function(d,c){if(c!==undefined&&c===this.selected){return false}this.$el.find(".dataset-selector span").toggleClass("fa-square-o fa-check-square-o");this.selected=!this.selected;this.trigger((this.selected)?("selected"):("de-selected"),this);return false},remove:function(d){var c=this;this.$el.fadeOut(c.fxSpeed,function(){c.$el.remove();c.off();if(d){d()}})},toString:function(){var c=(this.model)?(this.model+""):("(no model)");return"HDABaseView("+c+")"}});a.templates={skeleton:Handlebars.templates["template-hda-skeleton"],body:Handlebars.templates["template-hda-body"]};return{HDABaseView:a}}); \ No newline at end of file +define(["mvc/dataset/hda-model"],function(b){var a=Backbone.View.extend(LoggableMixin).extend({tagName:"div",className:"dataset hda history-panel-hda",id:function(){return"hda-"+this.model.get("id")},fxSpeed:"fast",initialize:function(c){if(c.logger){this.logger=this.model.logger=c.logger}this.log(this+".initialize:",c);this.defaultPrimaryActionButtonRenderers=[this._render_showParamsButton];this.selectable=false;this.selected=false;this.expanded=c.expanded||false;this._setUpListeners()},_setUpListeners:function(){this.model.on("change",function(d,c){if(this.model.changedAttributes().state&&this.model.inReadyState()&&this.expanded&&!this.model.hasDetails()){this.model.fetch()}else{this.render()}},this)},render:function(e){e=(e===undefined)?(true):(e);var c=this;this.$el.find("[title]").tooltip("destroy");this.urls=this.model.urls();var d=$(a.templates.skeleton(this.model.toJSON()));d.find(".dataset-primary-actions").append(this._render_titleButtons());d.children(".dataset-body").replaceWith(this._render_body());this._setUpBehaviors(d);if(e){$(c).queue(function(f){this.$el.fadeOut(c.fxSpeed,f)})}$(c).queue(function(f){this.$el.empty().attr("class",c.className).addClass("state-"+c.model.get("state")).append(d.children());f()});if(e){$(c).queue(function(f){this.$el.fadeIn(c.fxSpeed,f)})}$(c).queue(function(f){this.trigger("rendered",c);if(this.model.inReadyState()){this.trigger("rendered:ready",c)}if(this.selectable){this.showSelect()}f()});return this},_setUpBehaviors:function(c){c=c||this.$el;make_popup_menus(c);c.find("[title]").tooltip({placement:"bottom"})},_render_titleButtons:function(){return[this._render_displayButton()]},_render_displayButton:function(){if((this.model.get("state")===b.HistoryDatasetAssociation.STATES.NOT_VIEWABLE)||(this.model.get("state")===b.HistoryDatasetAssociation.STATES.DISCARDED)||(this.model.get("state")===b.HistoryDatasetAssociation.STATES.NEW)||(!this.model.get("accessible"))){return null}var d={target:"galaxy_main",classes:"dataset-display"};if(this.model.get("purged")){d.disabled=true;d.title=_l("Cannot display datasets removed from disk")}else{if(this.model.get("state")===b.HistoryDatasetAssociation.STATES.UPLOAD){d.disabled=true;d.title=_l("This dataset must finish uploading before it can be viewed")}else{d.title=_l("View data");d.href=this.urls.display;var c=this;d.onclick=function(){Galaxy.frame.add({title:"Data Viewer: "+c.model.get("name"),type:"url",target:"galaxy_main",content:c.urls.display})}}}d.faIcon="fa-eye";return faIconButton(d)},_render_downloadButton:function(){if(this.model.get("purged")||!this.model.hasData()){return null}var d=this.urls,e=this.model.get("meta_files");if(_.isEmpty(e)){return $(['<a href="'+d.download+'" title="'+_l("Download")+'" class="icon-btn">','<span class="fa fa-floppy-o"></span>',"</a>"].join(""))}var f="dataset-"+this.model.get("id")+"-popup",c=['<div popupmenu="'+f+'">','<a href="'+d.download+'">',_l("Download Dataset"),"</a>","<a>"+_l("Additional Files")+"</a>",_.map(e,function(g){return['<a class="action-button" href="',d.meta_download+g.file_type,'">',_l("Download")," ",g.file_type,"</a>"].join("")}).join("\n"),"</div>",'<div class="icon-btn-group">','<a href="'+d.download+'" title="'+_l("Download")+'" class="icon-btn">','<span class="fa fa-floppy-o"></span>','</a><a class="icon-btn popup" id="'+f+'">','<span class="fa fa-caret-down"></span>',"</a>","</div>"].join("\n");return $(c)},_render_showParamsButton:function(){return faIconButton({title:_l("View details"),href:this.urls.show_params,target:"galaxy_main",faIcon:"fa-info-circle"})},_render_body:function(){var d=$('<div>Error: unknown dataset state "'+this.model.get("state")+'".</div>'),c=this["_render_body_"+this.model.get("state")];if(_.isFunction(c)){d=c.call(this)}this._setUpBehaviors(d);if(this.expanded){d.show()}return d},_render_stateBodyHelper:function(c,f){f=f||[];var d=this,e=$(a.templates.body(_.extend(this.model.toJSON(),{body:c})));e.find(".dataset-actions .left").append(_.map(f,function(g){return g.call(d)}));return e},_render_body_new:function(){return this._render_stateBodyHelper("<div>"+_l("This is a new dataset and not all of its data are available yet")+"</div>")},_render_body_noPermission:function(){return this._render_stateBodyHelper("<div>"+_l("You do not have permission to view this dataset")+"</div>")},_render_body_discarded:function(){return this._render_stateBodyHelper("<div>"+_l("The job creating this dataset was cancelled before completion")+"</div>",this.defaultPrimaryActionButtonRenderers)},_render_body_queued:function(){return this._render_stateBodyHelper("<div>"+_l("This job is waiting to run")+"</div>",this.defaultPrimaryActionButtonRenderers)},_render_body_upload:function(){return this._render_stateBodyHelper("<div>"+_l("This dataset is currently uploading")+"</div>")},_render_body_setting_metadata:function(){return this._render_stateBodyHelper("<div>"+_l("Metadata is being auto-detected")+"</div>")},_render_body_running:function(){return this._render_stateBodyHelper("<div>"+_l("This job is currently running")+"</div>",this.defaultPrimaryActionButtonRenderers)},_render_body_paused:function(){return this._render_stateBodyHelper("<div>"+_l('This job is paused. Use the "Resume Paused Jobs" in the history menu to resume')+"</div>",this.defaultPrimaryActionButtonRenderers)},_render_body_error:function(){var c=['<span class="help-text">',_l("An error occurred with this dataset"),":</span>",'<div class="job-error-text">',$.trim(this.model.get("misc_info")),"</div>"].join("");if(!this.model.get("purged")){c="<div>"+this.model.get("misc_blurb")+"</div>"+c}return this._render_stateBodyHelper(c,this.defaultPrimaryActionButtonRenderers.concat([this._render_downloadButton]))},_render_body_empty:function(){return this._render_stateBodyHelper("<div>"+_l("No data")+": <i>"+this.model.get("misc_blurb")+"</i></div>",this.defaultPrimaryActionButtonRenderers)},_render_body_failed_metadata:function(){var c=$('<div class="warningmessagesmall"></div>').append($("<strong/>").text(_l("An error occurred setting the metadata for this dataset"))),d=this._render_body_ok();d.prepend(c);return d},_render_body_ok:function(){var c=this,e=$(a.templates.body(this.model.toJSON())),d=[this._render_downloadButton].concat(this.defaultPrimaryActionButtonRenderers);e.find(".dataset-actions .left").append(_.map(d,function(f){return f.call(c)}));if(this.model.isDeletedOrPurged()){return e}return e},events:{"click .dataset-title-bar":"toggleBodyVisibility","click .dataset-selector":"select"},toggleBodyVisibility:function(d,c){var e=this.$el.find(".dataset-body");c=(c===undefined)?(!e.is(":visible")):(c);if(c){this.expandBody()}else{this.collapseBody()}return false},expandBody:function(){var c=this;function d(){c.$el.children(".dataset-body").replaceWith(c._render_body());c.$el.children(".dataset-body").slideDown(c.fxSpeed,function(){c.expanded=true;c.trigger("body-expanded",c.model.get("id"))})}if(this.model.inReadyState()&&!this.model.hasDetails()){this.model.fetch({silent:true}).always(function(e){d()})}else{d()}},collapseBody:function(){var c=this;this.$el.children(".dataset-body").slideUp(c.fxSpeed,function(){c.expanded=false;c.trigger("body-collapsed",c.model.get("id"))})},showSelect:function(e){if(this.$el.find(".dataset-selector").css("display")!=="none"){return}e=(e!==undefined)?(e):(this.fxSpeed);var d=this,c=32;this.$el.queue("fx",function(f){$(this).find(".dataset-primary-actions").fadeOut(e,f)});this.$el.queue("fx",function(f){$(this).find(".dataset-selector").show().css({height:$(this).find(".dataset-title-bar").innerHeight()}).animate({width:c},e,f);d.selectable=true;d.trigger("selectable",true,d)})},hideSelect:function(c){c=(c!==undefined)?(c):(this.fxSpeed);this.selectable=false;this.trigger("selectable",false,this);this.$el.queue("fx",function(d){$(this).find(".dataset-selector").animate({width:"0px"},c,function(){$(this).hide();d()})});this.$el.queue("fx",function(d){$(this).find(".dataset-primary-actions").fadeIn(c,d)})},select:function(d,c){if(c!==undefined&&c===this.selected){return false}this.$el.find(".dataset-selector span").toggleClass("fa-square-o fa-check-square-o");this.selected=!this.selected;this.trigger((this.selected)?("selected"):("de-selected"),this);return false},remove:function(d){var c=this;this.$el.fadeOut(c.fxSpeed,function(){c.$el.remove();c.off();if(d){d()}})},toString:function(){var c=(this.model)?(this.model+""):("(no model)");return"HDABaseView("+c+")"}});a.templates={skeleton:Handlebars.templates["template-hda-skeleton"],body:Handlebars.templates["template-hda-body"]};return{HDABaseView:a}}); \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Fix metadata_util.py for recent changes to representation of ToolTestBuilder.inputs.
by commits-noreply＠bitbucket.org 22 Nov '13

22 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/54377b1aff65/ Changeset: 54377b1aff65 User: jmchilton Date: 2013-11-22 17:36:01 Summary: Fix metadata_util.py for recent changes to representation of ToolTestBuilder.inputs. Thanks to Greg for catching the error. Inputs display to the user will now be fully qualified - we should do some extra work I guess and display a prettified tree structure to match the new expressiveness of test inputs allowed. Affected #: 1 file diff -r 211f30207d48101a1250983d4f0f55c937f00a2e -r 54377b1aff65eb892d8e4067574c0cde552a2a57 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -940,10 +940,7 @@ for required_file in ttb.required_files: value, extra = required_file required_files.append( ( value ) ) - inputs = [] - for input in ttb.inputs: - name, value, extra = input - inputs.append( ( name, value ) ) + inputs = [ input for input in ttb.inputs ] outputs = [] for output in ttb.outputs: name, file_name, extra = output Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Extend tools API to allow use of data params with multiple='true' with multiple values.
by commits-noreply＠bitbucket.org 22 Nov '13

22 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/211f30207d48/ Changeset: 211f30207d48 User: jmchilton Date: 2013-11-22 09:09:28 Summary: Extend tools API to allow use of data params with multiple='true' with multiple values. Likewise extend API driven tool testing framework to support these, add sample test cases. Also fixup english on a log warning. Affected #: 4 files diff -r 4ddfe2108c00f3ba6ff4eb097986297a2b5c98f2 -r 211f30207d48101a1250983d4f0f55c937f00a2e lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1739,7 +1739,13 @@ if isinstance( value, str ) and value.find( "," ) > 0: value = [ int( value_part ) for value_part in value.split( "," ) ] if isinstance( value, list ): - rval = [ trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( v ) for v in value ] + rval = [] + for single_value in value: + if isinstance( single_value, dict ): + assert single_value['src'] == 'hda' + rval.append( trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( trans.app.security.decode_id( single_value[ 'id' ] ) ) ) + else: + rval.append( trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( single_value ) ) elif isinstance( value, trans.app.model.HistoryDatasetAssociation ): rval = value elif isinstance( value, dict ) and 'src' in value and 'id' in value: diff -r 4ddfe2108c00f3ba6ff4eb097986297a2b5c98f2 -r 211f30207d48101a1250983d4f0f55c937f00a2e lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -179,10 +179,12 @@ raw_input = context.extract_value( raw_inputs ) if raw_input: (name, param_value, param_extra) = raw_input + param_value = self.__split_if_str( param_value ) if isinstance( value, basic.DataToolParameter ): - processed_value = [ self.__add_uploaded_dataset( context.for_state(), param_value, param_extra, value ) ] + if not isinstance(param_value, list): + param_value = [ param_value ] + processed_value = [ self.__add_uploaded_dataset( context.for_state(), v, param_extra, value ) for v in param_value ] else: - param_value = self.__split_if_str( param_value ) processed_value = param_value expanded_inputs[ context.for_state() ] = processed_value return expanded_inputs @@ -309,7 +311,7 @@ for metadata_elem in output_elem.findall( 'metadata' ): metadata[ metadata_elem.get('name') ] = metadata_elem.get( 'value' ) if not (assert_list or file or extra_files or metadata): - raise Exception( "Test output defines not checks (e.g. must have a 'file' check against, assertions to check, etc...)") + raise Exception( "Test output defines nothting to check (e.g. must have a 'file' check against, assertions to check, etc...)") attributes['assert_list'] = assert_list attributes['extra_files'] = extra_files attributes['metadata'] = metadata diff -r 4ddfe2108c00f3ba6ff4eb097986297a2b5c98f2 -r 211f30207d48101a1250983d4f0f55c937f00a2e test/base/interactor.py --- a/test/base/interactor.py +++ b/test/base/interactor.py @@ -133,9 +133,13 @@ inputs_tree = testdef.inputs.copy() for key, value in inputs_tree.iteritems(): values = [value] if not isinstance(value, list) else value + new_values = [] for value in values: if value in self.uploads: - inputs_tree[ key ] = self.uploads[ value ] + new_values.append( self.uploads[ value ] ) + else: + new_values.append( value ) + inputs_tree[ key ] = new_values # # HACK: Flatten single-value lists. Required when using expand_grouping for key, value in inputs_tree.iteritems(): diff -r 4ddfe2108c00f3ba6ff4eb097986297a2b5c98f2 -r 211f30207d48101a1250983d4f0f55c937f00a2e test/functional/tools/multi_data_param.xml --- a/test/functional/tools/multi_data_param.xml +++ b/test/functional/tools/multi_data_param.xml @@ -18,13 +18,23 @@ <output name="out1" file="simple_line.txt" /><output name="out2" file="simple_line_alternative.txt" /></test> - <test><param name="f1" value="simple_line.txt,simple_line.txt" /><param name="f2" value="simple_line_alternative.txt" /><output name="out1" file="simple_line_x2.txt" /> - <output name="out1" file="simple_line_alternative.txt" /> + <output name="out2" file="simple_line_alternative.txt" /></test> - --></tests></tool> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Add test case for data param with multiple=True even if just 1 file used.
by commits-noreply＠bitbucket.org 22 Nov '13

22 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/4ddfe2108c00/ Changeset: 4ddfe2108c00 User: jmchilton Date: 2013-11-22 08:27:47 Summary: Add test case for data param with multiple=True even if just 1 file used. Add fix to twill driver to fix this use case. Affected #: 3 files diff -r b2feff0cd2e7dbc3d9ba4b5bed55bd744406c6b1 -r 4ddfe2108c00f3ba6ff4eb097986297a2b5c98f2 test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -1305,7 +1305,15 @@ control.get( name=elem ).selected = True else: for elem in control_value: - control.get( name=elem ).selected = True + try: + # Doubt this case would ever work, but want + # to preserve backward compat. + control.get( name=elem ).selected = True + except Exception: + # ... anyway this is really what we want to + # do, probably even want to try the len( + # elem ) > 30 check below. + control.get( label=elem ).selected = True else: # control.is_of_kind( "singlelist" ) for elem in control_value: try: diff -r b2feff0cd2e7dbc3d9ba4b5bed55bd744406c6b1 -r 4ddfe2108c00f3ba6ff4eb097986297a2b5c98f2 test/functional/tools/multi_data_param.xml --- /dev/null +++ b/test/functional/tools/multi_data_param.xml @@ -0,0 +1,30 @@ +<tool id="multi_data_param" name="multi_data_param" version="0.1.0"> + <command> + cat #for $f in $f1# ${f} #end for# >> $out1; + cat #for $f in $f2# ${f} #end for# >> $out2 + </command> + <inputs> + <param name="f1" type="data" format="txt" multiple="true" /> + <param name="f2" type="data" format="txt" multiple="true" /> + </inputs> + <outputs> + <data format="txt" name="out1" /> + <data format="txt" name="out2" /> + </outputs> + <tests> + <test> + <param name="f1" value="simple_line.txt" /> + <param name="f2" value="simple_line_alternative.txt" /> + <output name="out1" file="simple_line.txt" /> + <output name="out2" file="simple_line_alternative.txt" /> + </test> +  + </tests> +</tool> diff -r b2feff0cd2e7dbc3d9ba4b5bed55bd744406c6b1 -r 4ddfe2108c00f3ba6ff4eb097986297a2b5c98f2 test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -14,4 +14,5 @@ <tool file="disambiguate_repeats.xml" /><tool file="min_repeat.xml" /><tool file="implicit_default_conds.xml" /> + <tool file="multi_data_param.xml" /></toolbox> \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Allow API driven tests via install_and_test_tool_shed_repositories framework.
by commits-noreply＠bitbucket.org 22 Nov '13

22 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b2feff0cd2e7/ Changeset: b2feff0cd2e7 User: jmchilton Date: 2013-11-22 07:50:37 Summary: Allow API driven tests via install_and_test_tool_shed_repositories framework. Greg/Dave - I don't have the infrastructure/knowledge to test this changeset, so sorry if it breaks something, but this is what I believe is nessecary if you want to set GALAXY_TEST_DEFAULT_INTERACTOR=api or if anyone posts a tool to the tool shed that forces the issue by marking a test case with 'interactor="api"'. Affected #: 1 file diff -r d896ca45280fbc7764fa2fe82fe10d7537d858dc -r b2feff0cd2e7dbc3d9ba4b5bed55bd744406c6b1 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -91,6 +91,7 @@ default_galaxy_test_port_min = 10000 default_galaxy_test_port_max = 10999 default_galaxy_test_host = '127.0.0.1' +default_galaxy_master_api_key = "123456" # should this serve static resources (scripts, images, styles, etc.) STATIC_ENABLED = True @@ -792,6 +793,7 @@ # Write the embedded web application's specific configuration to a temporary file. This is necessary in order for # the external metadata script to find the right datasets. kwargs = dict( admin_users = 'test(a)bx.psu.edu', + master_api_key = default_galaxy_master_api_key, allow_user_creation = True, allow_user_deletion = True, allow_library_path_paste = True, @@ -1181,7 +1183,7 @@ test_toolbox.toolbox = app.toolbox # Generate the test methods for this installed repository. We need to pass in True here, or it will look # in $GALAXY_HOME/test-data for test data, which may result in missing or invalid test files. - test_toolbox.build_tests( testing_shed_tools=True ) + test_toolbox.build_tests( testing_shed_tools=True, master_api_key=default_galaxy_master_api_key ) # Set up nose to run the generated functional tests. test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) test_config.configure( sys.argv ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Fixes related to implicit defaults of param values.
by commits-noreply＠bitbucket.org 22 Nov '13

22 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d896ca45280f/ Changeset: d896ca45280f User: jmchilton Date: 2013-11-22 06:37:22 Summary: Fixes related to implicit defaults of param values. Logic errors related to them being contained in repeat blocks and to picking the top value in a select by default when no other value is marked as default. Other small adjustments - add another sample tool demonstrating the problem and clean up error messages. Affected #: 3 files diff -r 1c306a40b1c3f6f3abc51f44d2bc77a5bb1d6695 -r d896ca45280fbc7764fa2fe82fe10d7537d858dc lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -78,13 +78,15 @@ if selected: default_option = name else: - default_option = test_param.static_options[0] + first_option = test_param.static_options[0] + first_option_value = first_option[1] + default_option = first_option_value matches_declared_value = lambda case_value: case_value == default_option else: # No explicit value for this param and cannot determine a # default - give up. Previously this would just result in a key # error exception. - msg = "Failed to find test parameter specification required for conditional %s" % cond + msg = "Failed to find test parameter value specification required for conditional %s" % cond.name raise Exception( msg ) # Check the tool's defined cases against predicate to determine @@ -93,7 +95,9 @@ if matches_declared_value( case.value ): return case else: - log.info("Failed to find case matching test parameter specification for cond %s. Remainder of test behavior is unspecified." % cond) + msg_template = "%s - Failed to find case matching value (%s) for test parameter specification for conditional %s. Remainder of test behavior is unspecified." + msg = msg_template % ( self.tool.id, declared_value, cond.name ) + log.info( msg ) def __split_if_str( self, value ): split = isinstance(value, str) @@ -143,10 +147,20 @@ case_value = raw_input[ 1 ] if raw_input else None case = self.__matching_case_for_value( value, case_value ) if case: - expanded_value = self.__split_if_str( case.value ) - expanded_inputs[ case_context.for_state() ] = expanded_value for input_name, input_value in case.inputs.items(): - expanded_inputs.update( self.__process_raw_inputs( { input_name: input_value }, raw_inputs, parent_context=cond_context ) ) + case_inputs = self.__process_raw_inputs( { input_name: input_value }, raw_inputs, parent_context=cond_context ) + expanded_inputs.update( case_inputs ) + expanded_case_value = self.__split_if_str( case.value ) + if case_value is not None: + # A bit tricky here - we are growing inputs with value + # that may be implicit (i.e. not defined by user just + # a default defined in tool). So we do not want to grow + # expanded_inputs and risk repeat block viewing this + # as a new instance with value defined and hence enter + # an infinite loop - hence the "case_value is not None" + # check. + expanded_inputs[ case_context.for_state() ] = expanded_case_value + elif isinstance( value, grouping.Repeat ): repeat_index = 0 while True: diff -r 1c306a40b1c3f6f3abc51f44d2bc77a5bb1d6695 -r d896ca45280fbc7764fa2fe82fe10d7537d858dc test/functional/tools/implicit_default_conds.xml --- /dev/null +++ b/test/functional/tools/implicit_default_conds.xml @@ -0,0 +1,49 @@ +<tool id="implicit_default_conds" name="implicit_default_conds"> + <command> + echo "$param_group[0].p1.val" >> $out_file1; + echo "$param_group[0].p2.val" >> $out_file1; + </command> + <inputs> + <repeat name="param_group" title="Param Group" min="1"> + <conditional name="p1"> + <param name="type" type="select"> + <option value="default">THE DEFAULT</option> + <option value="different">A different value</option> + </param> + <when value="default"> + <param name="val" value="p1default" type="text" /> + </when> + <when value="different"> + <param name="val" value="p1different" type="text" /> + </when> + </conditional> + <conditional name="p2"> + <param name="type" type="select"> + <option value="default">THE DEFAULT</option> + <option value="different" selected="true">A different value</option> + </param> + <when value="default"> + <param name="val" value="p2default" type="text" /> + </when> + <when value="different"> + <param name="val" value="p2different" type="text" /> + </when> + </conditional> + <param name="int_param" type="integer" value="8" /> + </repeat> + </inputs> + <outputs> + <data name="out_file1" format="txt" /> + </outputs> + <tests> + <test> + <param name="int_param" value="7" /> + <output name="out_file1"> + <assert_contents> + <has_line line="p1default" /> + <has_line line="p2different" /> + </assert_contents> + </output> + </test> + </tests> +</tool> diff -r 1c306a40b1c3f6f3abc51f44d2bc77a5bb1d6695 -r d896ca45280fbc7764fa2fe82fe10d7537d858dc test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -13,4 +13,5 @@ <tool file="output_order.xml" /><tool file="disambiguate_repeats.xml" /><tool file="min_repeat.xml" /> + <tool file="implicit_default_conds.xml" /></toolbox> \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Clean up tool framework test tools.
by commits-noreply＠bitbucket.org 22 Nov '13

22 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1c306a40b1c3/ Changeset: 1c306a40b1c3 User: jmchilton Date: 2013-11-22 04:17:04 Summary: Clean up tool framework test tools. Add example of nested conditional disambiugation. Affected #: 3 files diff -r 4615f7424dcb4d2b61be5f388c761f45a1302a03 -r 1c306a40b1c3f6f3abc51f44d2bc77a5bb1d6695 test/functional/tools/composite.xml --- a/test/functional/tools/composite.xml +++ b/test/functional/tools/composite.xml @@ -1,5 +1,4 @@ -<tool id="velvet_inspired" name="velvet_inspired" version="1.0.0"> - <description>Velvet sequence assembler for very short reads</description> +<tool id="composite" version="1.0.0"><command>cat '$input.extra_files_path/Sequences' > $output</command><inputs><param name="input" type="data" format="velvet" label="Velvet Dataset" help="Prepared by velveth."/> diff -r 4615f7424dcb4d2b61be5f388c761f45a1302a03 -r 1c306a40b1c3f6f3abc51f44d2bc77a5bb1d6695 test/functional/tools/disambiguate_cond.xml --- a/test/functional/tools/disambiguate_cond.xml +++ b/test/functional/tools/disambiguate_cond.xml @@ -1,6 +1,4 @@ -<tool id="handle_cond" name="handle_cond"> - <description>tail-to-head</description> -  +<tool id="disambiguate_cond" name="disambiguate_cond"><command> echo "$p1.p1v $p2.p2v $p3.p3v" > $out_file1; cat "$files.p4.file" >> $out_file1; </command> @@ -61,5 +59,29 @@ </assert_contents></output></test> + <test> + <conditional name="p1"> + <param name="use" value="False"/> + </conditional> + <conditional name="p2"> + <param name="use" value="True"/> + </conditional> + <conditional name="p3"> + <param name="use" value="False"/> + </conditional> + <conditional name="files"> + <param name="attach_files" value="True" /> + <conditional name="p4"> + <param name="use" value="True"/> + <param name="file" value="simple_line_alternative.txt" /> + </conditional> + </conditional> + <output name="out_file1"> + <assert_contents> + <has_line line="7 4 7" /> + <has_line line="This is a different line of text." /> + </assert_contents> + </output> + </test></tests></tool> diff -r 4615f7424dcb4d2b61be5f388c761f45a1302a03 -r 1c306a40b1c3f6f3abc51f44d2bc77a5bb1d6695 test/functional/tools/multi_repeats.xml --- a/test/functional/tools/multi_repeats.xml +++ b/test/functional/tools/multi_repeats.xml @@ -1,5 +1,4 @@ -<tool id="multirepeat" name="multirepeat"> - <description>tail-to-head</description> +<tool id="multi_repeats" name="multi_repeats"><command> cat $input1 #for $q in $queries# ${q.input2} #end for# #for $q in $more_queries# ${q.more_queries_input} #end for# > $out_file1 </command> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0