- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: Dave Bouvier: Enhance the install and test functional testing framework to:
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/89b5f6866eeb/ Changeset: 89b5f6866eeb User: Dave Bouvier Date: 2013-05-15 21:14:55 Summary: Enhance the install and test functional testing framework to: Skip any repository revision that has been flagged by the owner not to be tested. Set the appropriate flags for failed tests, installations, and missing components. Limit the flagging to revisions that are not the most recent. Affected #: 1 file diff -r 0005fa6a989143068c626cc63103aba910ba22f8 -r 89b5f6866eeb2fcb955d4e773d5b5328dd214610 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -253,7 +253,8 @@ params = urllib.urlencode( dict( do_not_test='false', downloadable='true', malicious='false', - includes_tools='true' ) ) + includes_tools='true', + skip_tool_test='false' ) ) api_url = get_api_url( base=location, parts=[ 'repository_revisions' ], params=params ) if format == 'json': return json_from_url( api_url ) @@ -284,23 +285,32 @@ tool_test_results = repository_metadata.get( 'tool_test_errors', {} ) return tool_test_results +def is_latest_downloadable_revision( url, repository_info_dict ): + api_url_parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ] + params = urllib.urlencode( dict( name=repository_info_dict[ 'name' ], owner=repository_info_dict[ 'owner' ] ) ) + api_url = get_api_url( url, api_url_parts, params ) + # TODO: Remove this try/catch wrapping when the main tool shed is updated. + try: + changeset_revisions = json_from_url( api_url ) + except Exception, e: + return True + latest_revision = changeset_revisions.pop() + return str( repository_info_dict[ 'changeset_revision' ] ) == str( latest_revision ) + def json_from_url( url ): url_handle = urllib.urlopen( url ) url_contents = url_handle.read() - return from_json_string( url_contents ) + try: + parsed_json = from_json_string( url_contents ) + except: + log.exception( 'Error parsing JSON data.' ) + raise + return parsed_json -def register_test_result( url, metadata_id, test_results_dict, passed_tests=False ): +def register_test_result( url, metadata_id, test_results_dict, repository_info_dict, params ): ''' - This script should never set do_not_test = True, because the repositories should always be re-tested - against the most recent code. + Update the repository metadata tool_test_results and appropriate flags using the API. ''' - params = {} - if passed_tests: - params[ 'tools_functionally_correct' ] = 'true' - params[ 'do_not_test' ] = 'false' - else: - params[ 'tools_functionally_correct' ] = 'false' - params[ 'do_not_test' ] = 'false' params[ 'tool_test_results' ] = test_results_dict # BEGIN compatibility code. # TODO: The repository_revisions API controller ignores any received parameter that is not present in the database schema, @@ -602,6 +612,7 @@ } """ repository_status = dict() + params = dict() repository_id = repository_info_dict.get( 'repository_id', None ) changeset_revision = repository_info_dict.get( 'changeset_revision', None ) metadata_revision_id = repository_info_dict.get( 'id', None ) @@ -677,21 +688,50 @@ # }, # ] # "installation_errors": - # [ + # { + # 'tool_dependencies': + # [ + # { + # 'type': 'Type of tool dependency, e.g. package, set_environment, etc.', + # 'name': 'Name of the tool dependency.', + # 'version': 'Version if this is a package, otherwise blank.', + # 'error_message': 'The error message returned when installation was attempted.', + # }, + # ], + # 'repository_dependencies': + # [ + # { + # 'tool_shed': 'The tool shed that this repository was installed from.', + # 'name': 'The name of the repository that failed to install.', + # 'owner': 'Owner of the failed repository.', + # 'changeset_revision': 'Changeset revision of the failed repository.', + # 'error_message': 'The error message that was returned when the repository failed to install.', + # }, + # ], + # 'current_repository': + # [ + # { + # 'tool_shed': 'The tool shed that this repository was installed from.', + # 'name': 'The name of the repository that failed to install.', + # 'owner': 'Owner of the failed repository.', + # 'changeset_revision': 'Changeset revision of the failed repository.', + # 'error_message': 'The error message that was returned when the repository failed to install.', + # }, + # ], # { # "name": "The name of the repository.", # "owner": "The owner of the repository.", # "changeset_revision": "The changeset revision of the repository.", # "error_message": "The message stored in tool_dependency.error_message." # }, - # ] + # } # "missing_test_components": # [ # { - # "tool_id": "The tool ID that does not have functional tests defined.", + # "tool_id": "The tool ID that missing components.", # "tool_version": "The version of the tool." # "tool_guid": "The guid of the tool." - # "missing_components": "A short explanation of what is invalid. + # "missing_components": "Which components are missing, e.g. the test data filename, or the test-data directory." # }, # ] # } @@ -714,7 +754,15 @@ if missing_components not in repository_status[ 'missing_test_components' ]: repository_status[ 'missing_test_components' ].append( missing_components ) # Record the status of this repository in the tool shed. - register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=False ) + set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_info_dict ) + params[ 'tools_functionally_correct' ] = 'false' + params[ 'missing_test_components' ] = 'true' + params[ 'do_not_test' ] = str( set_do_not_test ) + register_test_result( galaxy_tool_shed_url, + metadata_revision_id, + repository_status, + repository_info_dict, + params ) # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the # repository using Twill. execute_uninstall_method( app ) @@ -742,13 +790,21 @@ error_message=dependency.error_message ) repository_status[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) # Record the status of this repository in the tool shed. - register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=False ) + set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_info_dict ) + params[ 'tools_functionally_correct' ] = 'false' + params[ 'do_not_test' ] = str( set_do_not_test ) + params[ 'test_install_error' ] = 'true' + register_test_result( galaxy_tool_shed_url, + metadata_revision_id, + repository_status, + repository_info_dict, + params ) # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the # repository using Twill. execute_uninstall_method( app ) # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. test_toolbox.toolbox = app.toolbox - repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + repositories_failed_install.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) else: # If the repository does have a test-data directory, we write the generated shed_tools_dict to a file, so the functional # test framework can find it. @@ -788,7 +844,13 @@ # controller with the status of the test. This also sets the do_not_test and tools_functionally correct flags, and # updates the time_last_tested field to today's date. repositories_passed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) - register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=True ) + params[ 'tools_functionally_correct' ] = 'true' + params[ 'do_not_test' ] = 'false' + register_test_result( galaxy_tool_shed_url, + metadata_revision_id, + repository_status, + repository_info_dict, + params ) log.debug( 'Revision %s of repository %s installed and passed functional tests.', changeset_revision, name ) else: # If the functional tests fail, log the output and update the failed changeset revision's metadata record in the tool shed via the API. @@ -833,7 +895,14 @@ # This also sets the do_not_test and tools_functionally correct flags to the appropriate values, and updates the time_last_tested # field to today's date. repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) - register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=False ) + set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_info_dict ) + params[ 'tools_functionally_correct' ] = 'false' + params[ 'do_not_test' ] = str( set_do_not_test ) + register_test_result( galaxy_tool_shed_url, + metadata_revision_id, + repository_status, + repository_info_dict, + params ) log.debug( 'Revision %s of repository %s installed successfully, but did not pass functional tests.', changeset_revision, name ) # Run the uninstall method. This removes tool functional test methods from the test_toolbox module and uninstalls the @@ -856,7 +925,15 @@ changeset_revision=repository.changeset_revision, error_message=repository.error_message ) repository_status[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) - register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=False ) + set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_info_dict ) + params[ 'tools_functionally_correct' ] = 'false' + params[ 'test_install_error' ] = 'true' + params[ 'do_not_test' ] = str( set_do_not_test ) + register_test_result( galaxy_tool_shed_url, + metadata_revision_id, + repository_status, + repository_info_dict, + params ) success = execute_uninstall_method( app ) if not success: log.error( 'Repository %s failed to uninstall.', repository_info_dict[ 'name' ] ) @@ -899,17 +976,17 @@ if repositories_tested > 0: if repositories_passed: print '# ----------------------------------------------------------------------------------' - print "# %d repositories passed:" % len( repositories_passed ) + print "# %d repositories passed all tests:" % len( repositories_passed ) show_summary_output( repositories_passed ) if repositories_failed: print '# ----------------------------------------------------------------------------------' - print "# %d repositories failed:" % len( repositories_failed ) + print "# %d repositories failed one or more tests:" % len( repositories_failed ) show_summary_output( repositories_failed ) if repositories_failed_install: # Set success to False so that the return code will not be 0. success = False print '# ----------------------------------------------------------------------------------' - print "# %d repositories not installed correctly:" % len( repositories_failed_install ) + print "# %d repositories with installation errors:" % len( repositories_failed_install ) show_summary_output( repositories_failed_install ) else: success = True Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Add option to filter by repository installation error status in the repository metadata API controller.
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0005fa6a9891/ Changeset: 0005fa6a9891 User: Dave Bouvier Date: 2013-05-15 21:00:15 Summary: Add option to filter by repository installation error status in the repository metadata API controller. Affected #: 1 file diff -r 31616ab57a240c155ac2d5e1272bba28731552be -r 0005fa6a989143068c626cc63103aba910ba22f8 lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -54,6 +54,10 @@ includes_tools = kwd.get( 'includes_tools', None ) if includes_tools is not None: clause_list.append( trans.model.RepositoryMetadata.table.c.includes_tools == util.string_as_bool( includes_tools ) ) + # Filter by test_install_error if received. + test_install_error = kwd.get( 'test_install_error', None ) + if test_install_error is not None: + clause_list.append( trans.model.RepositoryMetadata.table.c.test_install_error == util.string_as_bool( test_install_error ) ) # Filter by skip_tool_test if received. skip_tool_test = kwd.get( 'skip_tool_test', None ) if skip_tool_test is not None: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Unicode fixes for toolshed repositories.
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/31616ab57a24/ Changeset: 31616ab57a24 User: dan Date: 2013-05-15 20:50:19 Summary: Unicode fixes for toolshed repositories. Affected #: 3 files diff -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 -r 31616ab57a240c155ac2d5e1272bba28731552be lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -690,7 +690,7 @@ contents = [] for item in os.listdir( folder_path ): # Skip .hg directories - if str( item ).startswith( '.hg' ): + if item.startswith( '.hg' ): continue if os.path.isdir( os.path.join( folder_path, item ) ): # Append a '/' character so that our jquery dynatree will function properly. diff -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 -r 31616ab57a240c155ac2d5e1272bba28731552be templates/webapps/tool_shed/repository/view_changelog.mako --- a/templates/webapps/tool_shed/repository/view_changelog.mako +++ b/templates/webapps/tool_shed/repository/view_changelog.mako @@ -88,7 +88,7 @@ %endif <div class="form-row"><label>Description:</label> - <a href="${h.url_for( controller='repository', action='view_changeset', id=trans.security.encode_id( repository.id ), ctx_str=ctx_str )}">${changeset[ 'description' ] | h}</a> + <a href="${h.url_for( controller='repository', action='view_changeset', id=trans.security.encode_id( repository.id ), ctx_str=ctx_str )}">${ util.unicodify( changeset[ 'description' ] ) | h}</a></div><div class="form-row"><label>Commit:</label> diff -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 -r 31616ab57a240c155ac2d5e1272bba28731552be templates/webapps/tool_shed/repository/view_changeset.mako --- a/templates/webapps/tool_shed/repository/view_changeset.mako +++ b/templates/webapps/tool_shed/repository/view_changeset.mako @@ -74,7 +74,7 @@ <tr><td><b>Commit message:</b> - <br/>${ctx.description() | h}<br/> + <br/>${ util.unicodify( ctx.description() ) | h}<br/></td></tr> %if modified: @@ -82,7 +82,7 @@ <td><b>modified:</b> %for item in modified: - <br/><a href="#${item}">${item | h}</a> + <br/><a href="#${ util.unicodify( item ) }">${ util.unicodify( item ) | h}</a> %endfor </td></tr> @@ -92,7 +92,7 @@ <td><b>added:</b> %for item in added: - <br/><a href="#${item}">${item | h}</a> + <br/><a href="#${ util.unicodify( item ) }">${ util.unicodify( item ) | h}</a> %endfor </td></tr> @@ -102,7 +102,7 @@ <td><b>removed:</b> %for item in removed: - <br/><a href="#${item}">${item | h}</a> + <br/><a href="#${ util.unicodify( item ) }">${ util.unicodify( item ) | h}</a> %endfor </td></tr> @@ -112,7 +112,7 @@ <td><b>deleted:</b> %for item in deleted: - <br/><a href="#${item}">${item | h}</a> + <br/><a href="#${ util.unicodify( item ) }">${ util.unicodify( item ) | h}</a> %endfor </td></tr> @@ -122,7 +122,7 @@ <td><b>unknown:</b> %for item in unknown: - <br/><a href="#${item}">${item | h}</a> + <br/><a href="#${ util.unicodify( item ) }">${ util.unicodify( item ) | h}</a> %endfor }</td></tr> @@ -132,7 +132,7 @@ <td><b>ignored:</b> %for item in ignored: - <br/><a href="#${item}">${item | h}</a> + <br/><a href="#${ util.unicodify( item ) }">${ util.unicodify( item ) | h}</a> %endfor </td></tr> @@ -142,7 +142,7 @@ <td> clean: %for item in clean: - <br/><a href="#${item}">${item | h}</a> + <br/><a href="#${ util.unicodify( item ) }">${ util.unicodify( item ) | h}</a> %endfor </td></tr> @@ -162,8 +162,8 @@ break ctr += 1 %> - <tr><td bgcolor="#E0E0E0">${anchor_str}</td></tr> - <tr><td>${diff}</td></tr> + <tr><td bgcolor="#E0E0E0">${ util.unicodify( anchor_str ) }</td></tr> + <tr><td>${ util.unicodify( diff ) }</td></tr> %endfor </table></div> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add the ability for a tool shed repository owner to set a flag to skip testing a specified repository revision. Add a new test_install_error column to the repository_metadata table, and add additional filtered list of the latest revision of repositories using these new filters.
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a3e1fb22f5b5/ Changeset: a3e1fb22f5b5 User: greg Date: 2013-05-15 20:03:43 Summary: Add the ability for a tool shed repository owner to set a flag to skip testing a specified repository revision. Add a new test_install_error column to the repository_metadata table, and add additional filtered list of the latest revision of repositories using these new filters. Affected #: 9 files diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -54,11 +54,25 @@ includes_tools = kwd.get( 'includes_tools', None ) if includes_tools is not None: clause_list.append( trans.model.RepositoryMetadata.table.c.includes_tools == util.string_as_bool( includes_tools ) ) + # Filter by skip_tool_test if received. + skip_tool_test = kwd.get( 'skip_tool_test', None ) + if skip_tool_test is not None: + skip_tool_test = util.string_as_bool( skip_tool_test ) + # Generate and execute the query. try: - query = trans.sa_session.query( trans.app.model.RepositoryMetadata ) \ - .filter( and_( *clause_list ) ) \ - .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id ) \ - .all() + if skip_tool_test: + # The skip_tool_test filter was received as True. + query = trans.sa_session.query( trans.app.model.RepositoryMetadata ) \ + .join( trans.model.SkipToolTest ) \ + .filter( and_( *clause_list ) ) \ + .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id ) \ + .all() + else: + # The skip_tool_test filter was not received or it was received as False. + query = trans.sa_session.query( trans.app.model.RepositoryMetadata ) \ + .filter( and_( *clause_list ) ) \ + .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id ) \ + .all() for repository_metadata in query: repository_metadata_dict = repository_metadata.get_api_value( view='collection', value_mapper=default_value_mapper( trans, repository_metadata ) ) diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -64,6 +64,8 @@ my_writable_repositories_with_failing_tool_tests_grid = repository_grids.MyWritableRepositoriesWithFailingToolTestsGrid() my_writable_repositories_with_invalid_tools_grid = repository_grids.MyWritableRepositoriesWithInvalidToolsGrid() my_writable_repositories_with_no_failing_tool_tests_grid = repository_grids.MyWritableRepositoriesWithNoFailingToolTestsGrid() + my_writable_repositories_with_skip_tests_checked_grid = repository_grids.MyWritableRepositoriesWithSkipTestsCheckedGrid() + my_writable_repositories_with_test_install_errors_grid = repository_grids.MyWritableRepositoriesWithTestInstallErrorsGrid() repositories_by_user_grid = repository_grids.RepositoriesByUserGrid() repositories_i_own_grid = repository_grids.RepositoriesIOwnGrid() repositories_in_category_grid = repository_grids.RepositoriesInCategoryGrid() @@ -71,6 +73,8 @@ repositories_with_failing_tool_tests_grid = repository_grids.RepositoriesWithFailingToolTestsGrid() repositories_with_invalid_tools_grid = repository_grids.RepositoriesWithInvalidToolsGrid() repositories_with_no_failing_tool_tests_grid = repository_grids.RepositoriesWithNoFailingToolTestsGrid() + repositories_with_skip_tests_checked_grid = repository_grids.RepositoriesWithSkipTestsCheckedGrid() + repositories_with_test_install_errors_grid = repository_grids.RepositoriesWithTestInstallErrorsGrid() repository_dependencies_grid = repository_grids.RepositoryDependenciesGrid() repository_grid = repository_grids.RepositoryGrid() # The repository_metadata_grid is not currently displayed, but is sub-classed by several grids. @@ -222,6 +226,7 @@ message += '<li>you are authorized to update them</li>' message += '<li>the latest installable revision contains at least 1 tool</li>' message += '<li>the latest installable revision is not missing any tool test components</li>' + message += '<li>the latest installable revision has no installation errors</li>' message += '<li>the latest installable revision has at least 1 tool test that fails</li>' message += '</ul>' kwd[ 'message' ] = message @@ -286,6 +291,63 @@ return self.my_writable_repositories_with_no_failing_tool_tests_grid( trans, **kwd ) @web.expose + def browse_my_writable_repositories_with_install_errors( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd[ 'operation' ].lower() + if operation == "view_or_manage_repository": + return trans.response.send_redirect( web.url_for( controller='repository', + action='view_or_manage_repository', + **kwd ) ) + elif operation == "repositories_by_user": + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories_by_user', + **kwd ) ) + elif operation in [ 'mark as deprecated', 'mark as not deprecated' ]: + kwd[ 'mark_deprecated' ] = operation == 'mark as deprecated' + return trans.response.send_redirect( web.url_for( controller='repository', + action='deprecate', + **kwd ) ) + if 'message' not in kwd: + message = 'This list contains repositories that match the following criteria:<br>' + message += '<ul>' + message += '<li>you are authorized to update them</li>' + message += '<li>the latest installable revision contains at least 1 tool</li>' + message += '<li>the latest installable revision is not missing any tool test components</li>' + message += '<li>the latest installable revision has installation errors (the repository itself, repository dependencies or tool dependencies)</li>' + message += '</ul>' + kwd[ 'message' ] = message + kwd[ 'status' ] = 'warning' + return self.my_writable_repositories_with_test_install_errors_grid( trans, **kwd ) + + @web.expose + def browse_my_writable_repositories_with_skip_tool_test_checked( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd[ 'operation' ].lower() + if operation == "view_or_manage_repository": + return trans.response.send_redirect( web.url_for( controller='repository', + action='view_or_manage_repository', + **kwd ) ) + elif operation == "repositories_by_user": + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories_by_user', + **kwd ) ) + elif operation in [ 'mark as deprecated', 'mark as not deprecated' ]: + kwd[ 'mark_deprecated' ] = operation == 'mark as deprecated' + return trans.response.send_redirect( web.url_for( controller='repository', + action='deprecate', + **kwd ) ) + if 'message' not in kwd: + message = 'This list contains repositories that match the following criteria:<br>' + message += '<ul>' + message += '<li>you are authorized to update them</li>' + message += '<li>the latest installable revision contains at least 1 tool</li>' + message += '<li>the latest installable revision has <b>Skip tool tests</b> checked</li>' + message += '</ul>' + kwd[ 'message' ] = message + kwd[ 'status' ] = 'warning' + return self.my_writable_repositories_with_skip_tests_checked_grid( trans, **kwd ) + + @web.expose def browse_repositories( self, trans, **kwd ): # We add params to the keyword dict in this method in order to rename the param with an "f-" prefix, simulating filtering by clicking a search # link. We have to take this approach because the "-" character is illegal in HTTP requests. @@ -484,6 +546,34 @@ return self.repositories_with_failing_tool_tests_grid( trans, **kwd ) @web.expose + def browse_repositories_with_install_errors( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd[ 'operation' ].lower() + if operation == "view_or_manage_repository": + return trans.response.send_redirect( web.url_for( controller='repository', + action='view_or_manage_repository', + **kwd ) ) + elif operation == "repositories_by_user": + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories_by_user', + **kwd ) ) + elif operation in [ 'mark as deprecated', 'mark as not deprecated' ]: + kwd[ 'mark_deprecated' ] = operation == 'mark as deprecated' + return trans.response.send_redirect( web.url_for( controller='repository', + action='deprecate', + **kwd ) ) + if 'message' not in kwd: + message = 'This list contains repositories that match the following criteria:<br>' + message += '<ul>' + message += '<li>the latest installable revision contains at least 1 tool</li>' + message += '<li>the latest installable revision is not missing any tool test components</li>' + message += '<li>the latest installable revision has installation errors (the repository itself, repository dependencies or tool dependencies)</li>' + message += '</ul>' + kwd[ 'message' ] = message + kwd[ 'status' ] = 'warning' + return self.repositories_with_test_install_errors_grid( trans, **kwd ) + + @web.expose def browse_repositories_with_invalid_tools( self, trans, **kwd ): if 'operation' in kwd: operation = kwd[ 'operation' ].lower() @@ -539,6 +629,33 @@ return self.repositories_with_no_failing_tool_tests_grid( trans, **kwd ) @web.expose + def browse_repositories_with_skip_tool_test_checked( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd[ 'operation' ].lower() + if operation == "view_or_manage_repository": + return trans.response.send_redirect( web.url_for( controller='repository', + action='view_or_manage_repository', + **kwd ) ) + elif operation == "repositories_by_user": + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories_by_user', + **kwd ) ) + elif operation in [ 'mark as deprecated', 'mark as not deprecated' ]: + kwd[ 'mark_deprecated' ] = operation == 'mark as deprecated' + return trans.response.send_redirect( web.url_for( controller='repository', + action='deprecate', + **kwd ) ) + if 'message' not in kwd: + message = 'This list contains repositories that match the following criteria:<br>' + message += '<ul>' + message += '<li>the latest installable revision contains at least 1 tool</li>' + message += '<li>the latest installable revision has <b>Skip tool tests</b> checked</li>' + message += '</ul>' + kwd[ 'message' ] = message + kwd[ 'status' ] = 'warning' + return self.repositories_with_skip_tests_checked_grid( trans, **kwd ) + + @web.expose def browse_repository( self, trans, id, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/galaxy/webapps/tool_shed/model/__init__.py --- a/lib/galaxy/webapps/tool_shed/model/__init__.py +++ b/lib/galaxy/webapps/tool_shed/model/__init__.py @@ -219,13 +219,14 @@ class RepositoryMetadata( object, APIItem ): api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'has_repository_dependencies', 'includes_datatypes', 'includes_tools', 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) - api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'tools_functionally_correct', - 'do_not_test', 'time_last_tested', 'tool_test_results', 'has_repository_dependencies', 'includes_datatypes', 'includes_tools', - 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) + api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'tools_functionally_correct', 'do_not_test', + 'test_install_error', 'time_last_tested', 'tool_test_results', 'has_repository_dependencies', 'includes_datatypes', + 'includes_tools', 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) def __init__( self, id=None, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False, downloadable=False, - missing_test_components=None, tools_functionally_correct=False, do_not_test=False, time_last_tested=None, tool_test_results=None, - has_repository_dependencies=False, includes_datatypes=False, includes_tools=False, includes_tool_dependencies=False, includes_workflows=False ): + missing_test_components=None, tools_functionally_correct=False, do_not_test=False, test_install_error=False, time_last_tested=None, + tool_test_results=None, has_repository_dependencies=False, includes_datatypes=False, includes_tools=False, includes_tool_dependencies=False, + includes_workflows=False ): self.id = id self.repository_id = repository_id self.changeset_revision = changeset_revision @@ -236,6 +237,7 @@ self.missing_test_components = missing_test_components self.tools_functionally_correct = tools_functionally_correct self.do_not_test = do_not_test + self.test_install_error = test_install_error self.time_last_tested = time_last_tested self.tool_test_results = tool_test_results self.has_repository_dependencies = has_repository_dependencies @@ -274,6 +276,37 @@ return rval +class SkipToolTest( object, APIItem ): + api_collection_visible_keys = ( 'id', 'repository_metadata_id', 'initial_changeset_revision' ) + api_element_visible_keys = ( 'id', 'repository_metadata_id', 'initial_changeset_revision', 'comment' ) + + def __init__( self, id=None, repository_metadata_id=None, initial_changeset_revision=None, comment=None ): + self.id = id + self.repository_metadata_id = repository_metadata_id + self.initial_changeset_revision = initial_changeset_revision + self.comment = comment + + def as_dict( self, value_mapper=None ): + return self.get_api_value( view='element', value_mapper=value_mapper ) + + def get_api_value( self, view='collection', value_mapper=None ): + if value_mapper is None: + value_mapper = {} + rval = {} + try: + visible_keys = self.__getattribute__( 'api_' + view + '_visible_keys' ) + except AttributeError: + raise Exception( 'Unknown API view: %s' % view ) + for key in visible_keys: + try: + rval[ key ] = self.__getattribute__( key ) + if key in value_mapper: + rval[ key ] = value_mapper.get( key, rval[ key ] ) + except AttributeError: + rval[ key ] = None + return rval + + class RepositoryReview( object, APIItem ): api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating', 'deleted' ) api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating', 'deleted' ) diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/galaxy/webapps/tool_shed/model/mapping.py --- a/lib/galaxy/webapps/tool_shed/model/mapping.py +++ b/lib/galaxy/webapps/tool_shed/model/mapping.py @@ -132,6 +132,7 @@ Column( "downloadable", Boolean, default=True ), Column( "tools_functionally_correct", Boolean, default=False, index=True ), Column( "do_not_test", Boolean, default=False, index=True ), + Column( "test_install_error", Boolean, default=False, index=True ), Column( "time_last_tested", DateTime, default=None, nullable=True ), Column( "missing_test_components", Boolean, default=False, index=True ), Column( "tool_test_results", JSONType, nullable=True ), @@ -141,6 +142,14 @@ Column( "includes_tool_dependencies", Boolean, default=False, index=True ), Column( "includes_workflows", Boolean, default=False, index=True ) ) +SkipToolTest.table = Table( "skip_tool_test", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "repository_metadata_id", Integer, ForeignKey( "repository_metadata.id" ), index=True ), + Column( "initial_changeset_revision", TrimmedString( 255 ), index=True ), + Column( "comment" , TEXT ) ) + RepositoryReview.table = Table( "repository_review", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -267,10 +276,14 @@ foreign_keys=[ RepositoryMetadata.table.c.repository_id, RepositoryMetadata.table.c.changeset_revision ], primaryjoin=( ( RepositoryMetadata.table.c.repository_id == RepositoryReview.table.c.repository_id ) & ( RepositoryMetadata.table.c.changeset_revision == RepositoryReview.table.c.changeset_revision ) ) ) ) ) +mapper( SkipToolTest, SkipToolTest.table, + properties=dict( repository_revision=relation( RepositoryMetadata, + backref='skip_tool_tests' ) ) ) + mapper( RepositoryReview, RepositoryReview.table, properties=dict( repository=relation( Repository, primaryjoin=( RepositoryReview.table.c.repository_id == Repository.table.c.id ) ), - # Take case when using the mapper below! It should be used only when a new review is being created for a repository change set revision. + # Take care when using the mapper below! It should be used only when a new review is being created for a repository change set revision. # Keep in mind that repository_metadata records can be removed from the database for certain change set revisions when metadata is being # reset on a repository! repository_metadata=relation( RepositoryMetadata, diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/galaxy/webapps/tool_shed/model/migrate/versions/0019_add_skip_tool_test_table_and_test_install_error_column.py --- /dev/null +++ b/lib/galaxy/webapps/tool_shed/model/migrate/versions/0019_add_skip_tool_test_table_and_test_install_error_column.py @@ -0,0 +1,82 @@ +""" +Migration script to add the skip_tool_test table and add the test_install_error column to the repository_metadata table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * +import datetime + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +now = datetime.datetime.utcnow + +metadata = MetaData() + +SkipToolTest_table = Table( "skip_tool_test", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "repository_metadata_id", Integer, ForeignKey( "repository_metadata.id" ), index=True ), + Column( "initial_changeset_revision", TrimmedString( 255 ), index=True ), + Column( "comment" , TEXT ) ) + +def upgrade( migrate_engine ): + print __doc__ + metadata.bind = migrate_engine + metadata.reflect() + # Initialize. + if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + default_false = "0" + elif migrate_engine.name in [ 'postgresql', 'postgres' ]: + default_false = "false" + + try: + RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True ) + except NoSuchTableError: + RepositoryMetadata_table = None + log.debug( "Failed loading table repository_metadata." ) + + if RepositoryMetadata_table is not None: + # Create the test_install_error column. + c = Column( "test_install_error", Boolean, default=False, index=True ) + try: + c.create( RepositoryMetadata_table, index_name="ix_repository_metadata_ttie") + assert c is RepositoryMetadata_table.c.test_install_error + migrate_engine.execute( "UPDATE repository_metadata SET test_install_error=%s" % default_false ) + except Exception, e: + print "Adding test_install_error column to the repository_metadata table failed: %s" % str( e ) + + # Create skip_tool_test table. + try: + SkipToolTest_table.create() + except Exception, e: + print "Creating the skip_tool_test table failed: %s" % str( e ) + +def downgrade( migrate_engine ): + metadata.bind = migrate_engine + metadata.reflect() + + # Drop the skip_tool_test table. + try: + SkipToolTest_table.drop() + except Exception, e: + print "Dropping the skip_tool_test table failed: %s" % str( e ) + + # Drop test_install_error column from the repository_metadata table. + RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True ) + try: + RepositoryMetadata_table.c.test_install_error.drop() + except Exception, e: + print "Dropping column test_install_error from the repository_metadata table failed: %s" % str( e ) diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/tool_shed/grids/repository_grids.py --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -604,6 +604,180 @@ .filter( model.Repository.table.c.id < 0 ) +class RepositoriesWithTestInstallErrorsGrid( RepositoryGrid ): + # This grid displays only the latest installable revision of each repository. + title = "Repositories with tool test installation errors" + columns = [ + RepositoryGrid.NameColumn( "Name", + key="name", + link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), + attach_popup=False ), + RepositoryGrid.LatestInstallableRevisionColumn( "Latest Installable Revision" ), + RepositoryGrid.UserColumn( "Owner", + key="User.username", + model_class=model.User, + link=( lambda item: dict( operation="repositories_by_user", id=item.id ) ), + attach_popup=False ) + ] + columns.append( grids.MulticolFilterColumn( "Search repository name", + cols_to_filter=[ columns[0] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + operations = [] + use_paging = False + + def build_initial_query( self, trans, **kwd ): + # Filter by latest installable revisions that contain tools with missing tool test components. + revision_clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ): + changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ) + if changeset_revision: + revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) + if revision_clause_list: + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ) \ + .join( model.RepositoryMetadata ) \ + .filter( or_( *revision_clause_list ) ) \ + .join( model.User.table ) + # Return an empty query. + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.table.c.id < 0 ) + + +class MyWritableRepositoriesWithTestInstallErrorsGrid( RepositoriesWithTestInstallErrorsGrid ): + # This grid displays only the latest installable revision of each repository. + title = "Repositories I can change with tool test installation errors" + columns = [ col for col in RepositoriesWithTestInstallErrorsGrid.columns ] + operations = [] + use_paging = False + + def build_initial_query( self, trans, **kwd ): + # First get all repositories that the current user is authorized to update. + username = trans.user.username + user_clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ): + allow_push = repository.allow_push( trans.app ) + if allow_push: + allow_push_usernames = allow_push.split( ',' ) + if username in allow_push_usernames: + user_clause_list.append( model.Repository.table.c.id == repository.id ) + if user_clause_list: + # We have the list of repositories that the current user is authorized to update, so filter further by latest installable revisions that contain + # tools with missing tool test components. + revision_clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ) \ + .filter( or_( *user_clause_list ) ): + changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ) + if changeset_revision: + revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) + if revision_clause_list: + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ) \ + .join( model.User.table ) \ + .filter( or_( *user_clause_list ) ) \ + .join( model.RepositoryMetadata ) \ + .filter( or_( *revision_clause_list ) ) + # Return an empty query. + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.table.c.id < 0 ) + + +class RepositoriesWithSkipTestsCheckedGrid( RepositoryGrid ): + # This grid displays only the latest installable revision of each repository. + title = "Repositories with skip tool tests checked" + columns = [ + RepositoryGrid.NameColumn( "Name", + key="name", + link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), + attach_popup=False ), + RepositoryGrid.LatestInstallableRevisionColumn( "Latest Installable Revision" ), + RepositoryGrid.UserColumn( "Owner", + key="User.username", + model_class=model.User, + link=( lambda item: dict( operation="repositories_by_user", id=item.id ) ), + attach_popup=False ) + ] + columns.append( grids.MulticolFilterColumn( "Search repository name", + cols_to_filter=[ columns[0] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + operations = [] + use_paging = False + + def build_initial_query( self, trans, **kwd ): + # Filter by latest installable revisions that contain tools with missing tool test components. + revision_clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ): + changeset_revision = filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ) + if changeset_revision: + revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) + if revision_clause_list: + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ) \ + .join( model.RepositoryMetadata ) \ + .filter( or_( *revision_clause_list ) ) \ + .join( model.User.table ) + # Return an empty query. + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.table.c.id < 0 ) + + +class MyWritableRepositoriesWithSkipTestsCheckedGrid( RepositoriesWithSkipTestsCheckedGrid ): + # This grid displays only the latest installable revision of each repository. + title = "Repositories I can change with skip tool tests checked" + columns = [ col for col in RepositoriesWithSkipTestsCheckedGrid.columns ] + operations = [] + use_paging = False + + def build_initial_query( self, trans, **kwd ): + # First get all repositories that the current user is authorized to update. + username = trans.user.username + user_clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ): + allow_push = repository.allow_push( trans.app ) + if allow_push: + allow_push_usernames = allow_push.split( ',' ) + if username in allow_push_usernames: + user_clause_list.append( model.Repository.table.c.id == repository.id ) + if user_clause_list: + # We have the list of repositories that the current user is authorized to update, so filter further by latest installable revisions that contain + # tools with missing tool test components. + revision_clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ) \ + .filter( or_( *user_clause_list ) ): + changeset_revision = filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ) + if changeset_revision: + revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) + if revision_clause_list: + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ) \ + .join( model.User.table ) \ + .filter( or_( *user_clause_list ) ) \ + .join( model.RepositoryMetadata ) \ + .filter( or_( *revision_clause_list ) ) + # Return an empty query. + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.table.c.id < 0 ) + + class DeprecatedRepositoriesIOwnGrid( RepositoriesIOwnGrid ): title = "Deprecated repositories I own" columns = [ @@ -1475,7 +1649,10 @@ repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) repository_metadata = get_latest_installable_repository_metadata_if_it_includes_tools( trans, repository ) - if repository_metadata and not repository_metadata.missing_test_components and not repository_metadata.tools_functionally_correct: + if repository_metadata \ + and not repository_metadata.missing_test_components \ + and not repository_metadata.tools_functionally_correct \ + and not repository_metadata.test_install_error: return repository_metadata.changeset_revision return None @@ -1516,6 +1693,32 @@ return repository_metadata.changeset_revision return None +def filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ): + """ + Inspect the latest installable changeset revision for the received repository to see if it has tool test installation errors. + """ + encoded_repository_id = trans.security.encode_id( repository.id ) + repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) + tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) + repository_metadata = get_latest_installable_repository_metadata_if_it_includes_tools( trans, repository ) + if repository_metadata \ + and repository_metadata.test_install_error \ + and not repository_metadata.missing_test_components: + return repository_metadata.changeset_revision + return None + +def filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ): + """ + Inspect the latest installable changeset revision for the received repository to see if skip tests is checked. + """ + encoded_repository_id = trans.security.encode_id( repository.id ) + repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) + tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) + repository_metadata = get_latest_installable_repository_metadata_if_it_includes_tools( trans, repository ) + if repository_metadata and repository_metadata.skip_tool_tests: + return repository_metadata.changeset_revision + return None + def get_latest_repository_metadata_if_it_includes_invalid_tools( trans, repository ): """Return the latest repository_metadata record for the received repository that contains invalid tools if one exists.""" encoded_repository_id = trans.security.encode_id( repository.id ) diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -259,6 +259,8 @@ downloadable = has_repository_dependencies or includes_datatypes or includes_tools or includes_tool_dependencies or includes_workflows repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) if repository_metadata: + # A repository metadata record already exists with the received changeset_revision, so we don't need to check the skip_tool_test table. + check_skip_tool_test = False repository_metadata.metadata = metadata_dict repository_metadata.downloadable = downloadable repository_metadata.has_repository_dependencies = has_repository_dependencies @@ -267,6 +269,8 @@ repository_metadata.includes_tool_dependencies = includes_tool_dependencies repository_metadata.includes_workflows = includes_workflows else: + # No repository_metadata record exists for the received changeset_revision, so we may need to update the skip_tool_test table. + check_skip_tool_test = True repository_metadata = trans.model.RepositoryMetadata( repository_id=repository.id, changeset_revision=changeset_revision, metadata=metadata_dict, @@ -279,11 +283,38 @@ # Always set the default values for the following columns. When resetting all metadata on a repository, this will reset the values. repository_metadata.tools_functionally_correct = False repository_metadata.missing_test_components = False + repository_metadata.test_install_error = False repository_metadata.do_not_test = False repository_metadata.time_last_tested = None repository_metadata.tool_test_results = None trans.sa_session.add( repository_metadata ) trans.sa_session.flush() + if check_skip_tool_test: + # Since we created a new repository_metadata record, we may need to update the skip_tool_test table to point to it. Inspect each + # changeset revision in the received repository's changelog (up to the received changeset revision) to see if it is contained in the + # skip_tool_test table. If it is, but is not associated with a repository_metadata record, reset that skip_tool_test record to the + # newly created repository_metadata record. + repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) + for changeset in repo.changelog: + changeset_hash = str( repo.changectx( changeset ) ) + skip_tool_test = suc.get_skip_tool_test_by_changeset_revision( trans, changeset_hash ) + if skip_tool_test: + # We found a skip_tool_test record associated with the changeset_revision, so see if it has a valid repository_revision. + try: + repository_revision = skip_tool_test.repository_revision + # The skip_tool_test record is associated with a valid repository_metadata record, so proceed. + continue + except: + # We found a skip_tool_test record that is associated with an invalid repository_metadata record, so update it to point to + # the newly created repository_metadata record. In some special cases there may be multiple skip_tool_test records that + # require updating, so we won't break here, we'll continue to inspect the rest of the changelog up to the received + # changeset_revision. + skip_tool_test.repository_revision = repository_metadata + trans.sa_session.add( skip_tool_test ) + trans.sa_session.flush() + if changeset_hash == changeset_revision: + # Proceed no further than the received changeset_revision. + break return repository_metadata def generate_data_manager_metadata( app, repository, repo_dir, data_manager_config_filename, metadata_dict, shed_config_dict=None ): diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -793,6 +793,13 @@ if shed_tool_conf == file_name: return index, shed_tool_conf_dict +def get_skip_tool_test_by_changeset_revision( trans, changeset_revision ): + """Return a skip_tool_test record whose initial_changeset_revision is the received changeset_revision.""" + # There should only be one, but we'll use first() so callers won't have to handle exceptions. + return trans.sa_session.query( trans.model.SkipToolTest ) \ + .filter( trans.model.SkipToolTest.table.c.initial_changeset_revision == changeset_revision ) \ + .first() + def get_tool_panel_config_tool_path_install_dir( app, repository ): """ Return shed-related tool panel config, the tool_path configured in it, and the relative path to the directory where the repository is installed. diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 templates/webapps/tool_shed/index.mako --- a/templates/webapps/tool_shed/index.mako +++ b/templates/webapps/tool_shed/index.mako @@ -116,9 +116,15 @@ <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_missing_tool_test_components' )}">Latest revision: missing tool tests</a></div><div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_install_errors' )}">Latest revision: installation errors</a> + </div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_failing_tool_tests' )}">Latest revision: failing tool tests</a></div><div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_skip_tool_test_checked' )}">Latest revision: skip tool tests</a> + </div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_no_failing_tool_tests' )}">Latest revision: all tool tests pass</a></div><div class="toolTitle"> @@ -167,9 +173,15 @@ <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories_missing_tool_test_components' )}">Latest revision: missing tool tests</a></div><div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories_with_install_errors' )}">Latest revision: installation errors</a> + </div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories_with_failing_tool_tests' )}">Latest revision: failing tool tests</a></div><div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories_with_skip_tool_test_checked' )}">Latest revision: skip tool tests</a> + </div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories_with_no_failing_tool_tests' )}">Latest revision: all tool tests pass</a></div><div class="toolTitle"> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Update functional tests to reflect recent interface upgrades and enhancements.
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/036f57f93b23/ Changeset: 036f57f93b23 User: Dave Bouvier Date: 2013-05-15 19:44:53 Summary: Update functional tests to reflect recent interface upgrades and enhancements. Affected #: 4 files diff -r 93013dcdc4a558791743839d90262e713aba66fd -r 036f57f93b23a62205b8541b6fd773561879827e test/tool_shed/functional/test_0000_basic_repository_features.py --- a/test/tool_shed/functional/test_0000_basic_repository_features.py +++ b/test/tool_shed/functional/test_0000_basic_repository_features.py @@ -1,6 +1,7 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os import tool_shed.base.test_db_util as test_db_util - +import logging +log = logging.getLogger(__name__) repository_name = 'filtering_0000' repository_description = "Galaxy's filtering tool for test 0000" repository_long_description = "Long description of Galaxy's filtering tool for test 0000" @@ -342,8 +343,16 @@ repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) changeset_revision = self.get_repository_tip( repository ) repository_id = self.security.encode_id( repository.id ) + changelog_tuples = self.get_repository_changelog_tuples( repository ) + revision_number = -1 + revision_hash = '000000000000' + for numeric_changeset, changeset_hash in changelog_tuples: + if str( changeset_hash ) == str( changeset_revision ): + revision_number = numeric_changeset + revision_hash = changeset_hash + break # Check for the changeset revision, repository name, owner username, 'repos' in the clone url, and the captured # unicode decoding error message. - strings_displayed = [ 'Changeset %s' % changeset_revision, 'filtering_0000', 'user1', 'repos', 'added:', + strings_displayed = [ 'Changeset %d:%s' % ( revision_number, revision_hash ), 'filtering_0000', 'user1', 'repos', 'added:', 'Error decoding string:', "codec can't decode byte" ] self.load_changeset_in_tool_shed( repository_id, changeset_revision, strings_displayed=strings_displayed ) \ No newline at end of file diff -r 93013dcdc4a558791743839d90262e713aba66fd -r 036f57f93b23a62205b8541b6fd773561879827e test/tool_shed/functional/test_0420_citable_urls_for_repositories.py --- a/test/tool_shed/functional/test_0420_citable_urls_for_repositories.py +++ b/test/tool_shed/functional/test_0420_citable_urls_for_repositories.py @@ -148,9 +148,6 @@ test_user_1 = test_db_util.get_user( common.test_user_1_email ) encoded_user_id = self.security.encode_id( test_user_1.id ) encoded_repository_id = self.security.encode_id( repository.id ) - # We are checking the changeset revision pointed to by first_changeset_hash, stored in a global variable at the end of - # test_0005. The tip changeset hash should not be displayed here, but first_changeset_hash should. - tip_changeset_hash = self.get_repository_tip( repository ) # Since twill does not load the contents of an iframe, we need to check that the iframe has been generated correctly, # then directly load the url that the iframe should be loading and check for the expected strings. # The iframe should point to /repository/view_repository?id=<encoded repository ID> @@ -158,7 +155,7 @@ strings_displayed_in_iframe = [ 'user1', 'filtering_0420', 'Galaxy filtering tool for test 0420', first_changeset_hash ] strings_displayed_in_iframe.append( 'Sharable link to this repository revision:' ) strings_displayed_in_iframe.append( '%s/view/user1/filtering_0420/%s' % ( self.url, first_changeset_hash ) ) - strings_not_displayed_in_iframe = [ tip_changeset_hash ] + strings_not_displayed_in_iframe = [] self.load_citable_url( username='user1', repository_name='filtering_0420', changeset_revision=first_changeset_hash, @@ -173,20 +170,20 @@ test_user_1 = test_db_util.get_user( common.test_user_1_email ) encoded_user_id = self.security.encode_id( test_user_1.id ) encoded_repository_id = self.security.encode_id( repository.id ) - changeset_hash = '!!invalid!!' + invalid_changeset_hash = 'invalid' tip_revision = self.get_repository_tip( repository ) # Since twill does not load the contents of an iframe, we need to check that the iframe has been generated correctly, # then directly load the url that the iframe should be loading and check for the expected strings. # The iframe should point to /repository/view_repository?id=<encoded repository ID>&status=error strings_displayed = [ '/repository', 'view_repository', 'id=' + encoded_repository_id ] - strings_displayed.extend( [ 'The+change+log', 'does+not+include+revision', '%21%21invalid%21%21', 'status=error' ] ) + strings_displayed.extend( [ 'The+change+log', 'does+not+include+revision', invalid_changeset_hash, 'status=error' ] ) strings_displayed_in_iframe = [ 'user1', 'filtering_0420', 'Galaxy filtering tool for test 0420' ] strings_displayed_in_iframe.append( 'Sharable link to this repository revision:' ) - strings_displayed_in_iframe.append( '%s/view/user1/filtering_0420/%s' % ( self.url, changeset_hash ) ) - strings_not_displayed_in_iframe = [ tip_revision ] + strings_displayed_in_iframe.append( '%s/view/user1/filtering_0420/%s' % ( self.url, invalid_changeset_hash ) ) + strings_not_displayed_in_iframe = [] self.load_citable_url( username='user1', repository_name='filtering_0420', - changeset_revision=changeset_hash, + changeset_revision=invalid_changeset_hash, encoded_user_id=encoded_user_id, encoded_repository_id=encoded_repository_id, strings_displayed=strings_displayed, diff -r 93013dcdc4a558791743839d90262e713aba66fd -r 036f57f93b23a62205b8541b6fd773561879827e test/tool_shed/functional/test_1090_install_tool_from_tool_search.py --- a/test/tool_shed/functional/test_1090_install_tool_from_tool_search.py +++ b/test/tool_shed/functional/test_1090_install_tool_from_tool_search.py @@ -296,7 +296,7 @@ exact_matches=False, from_galaxy=True, strings_displayed=[ 'bwa_color_0090', 'bwa_base_0090', bwa_base_revision, bwa_color_revision ] ) strings_displayed=[ 'freebayes_0090', 'emboss_0090', 'filtering_0090' ] - strings_displayed.extend( [ 'bwa_color_0090', 'bwa_base_0090', 'emboss_datatypes_0090' ] ) + strings_displayed.extend( [ 'bwa_color_0090', 'bwa_base_0090' ] ) strings_displayed.extend( [ 'bwa', 'Handle', 'tool dependencies' ] ) repositories_to_install = [ bwa_color_repository, bwa_base_repository ] # BWA is a good candidate for testing the installation of tool dependencies, but it is a core requirement of functional diff -r 93013dcdc4a558791743839d90262e713aba66fd -r 036f57f93b23a62205b8541b6fd773561879827e test/tool_shed/functional/test_1300_reset_all_metadata.py --- a/test/tool_shed/functional/test_1300_reset_all_metadata.py +++ b/test/tool_shed/functional/test_1300_reset_all_metadata.py @@ -400,5 +400,14 @@ for repository in repositories: test_db_util.ga_refresh( repository ) old_metadata = repository_metadata[ self.security.encode_id( repository.id ) ] - assert repository.metadata == old_metadata, 'Metadata for installed repository %s changed after reset.' % repository.name + # When a repository with tools to be displayed in a tool panel section is deactivated and reinstalled, + # the tool panel section remains in the repository metadata. However, when the repository's metadata + # is subsequently reset, the tool panel section is removed from the repository metadata. While this + # is normal and expected behavior, the functional tests assume that repository metadata will not change + # in any way after a reset. A workaround is to remove the tool panel section from the stored repository + # metadata dict, in order to eliminate the misleading detection of changed metadata. + if 'tool_panel_section' in old_metadata and 'tool_panel_section' not in repository.metadata: + del old_metadata[ 'tool_panel_section' ] + assert repository.metadata == old_metadata, 'Metadata for %s repository %s changed after reset. \nOld: %s\nNew: %s' % \ + ( repository.status, repository.name, old_metadata, repository.metadata ) \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Fix for displaying a tool in the tool shed.
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/93013dcdc4a5/ Changeset: 93013dcdc4a5 User: Dave Bouvier Date: 2013-05-15 19:26:12 Summary: Fix for displaying a tool in the tool shed. Affected #: 1 file diff -r 413211aaecc51eb38fae668326d4defefda2b8f7 -r 93013dcdc4a558791743839d90262e713aba66fd templates/webapps/tool_shed/repository/tool_form.mako --- a/templates/webapps/tool_shed/repository/tool_form.mako +++ b/templates/webapps/tool_shed/repository/tool_form.mako @@ -107,7 +107,7 @@ %if trans.webapp.name == 'galaxy': ${render_galaxy_repository_actions( repository=repository )} %else: - ${render_tool_shed_repository_actions( repository, repo=None, metadata=None, changeset_revision=None )} + ${render_tool_shed_repository_actions( repository, metadata=None, changeset_revision=None )} %endif %if message: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Undo 69240f7da05e
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/413211aaecc5/ Changeset: 413211aaecc5 User: dan Date: 2013-05-15 19:23:45 Summary: Undo 69240f7da05e Affected #: 1 file diff -r ca0daccbb90f2a2eabfb5f64156a7c270cbd3eca -r 413211aaecc51eb38fae668326d4defefda2b8f7 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -422,10 +422,7 @@ Returns a unicode string or None """ if isinstance( value, unicode ): - try: - return value.encode( encoding, error ) - except: - return value + return value try: return unicode( value, encoding, error ) except: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Enhance util.unicodify to attempt to force encoding when already an unicode object.
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ca0daccbb90f/ Changeset: ca0daccbb90f User: dan Date: 2013-05-15 19:07:07 Summary: Enhance util.unicodify to attempt to force encoding when already an unicode object. Affected #: 1 file diff -r 69240f7da05e4ae42803b67a507103d201186306 -r ca0daccbb90f2a2eabfb5f64156a7c270cbd3eca lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -422,7 +422,10 @@ Returns a unicode string or None """ if isinstance( value, unicode ): - return value + try: + return value.encode( encoding, error ) + except: + return value try: return unicode( value, encoding, error ) except: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8701e8440e57/ Changeset: 8701e8440e57 User: dan Date: 2013-05-15 17:09:32 Summary: Unicode fixes for workflow names in Tool Menu. Affected #: 1 file diff -r 866ed8fd31e146792980c61489d57cc18a5036b5 -r 8701e8440e57bc8bc0b173297e60d33d4eab3734 templates/webapps/galaxy/root/tool_menu.mako --- a/templates/webapps/galaxy/root/tool_menu.mako +++ b/templates/webapps/galaxy/root/tool_menu.mako @@ -10,7 +10,7 @@ <div class="toolTitleNoSection"> %endif <% encoded_id = key.lstrip( 'workflow_' ) %> - <a id="link-${workflow.id}" href="${ h.url_for( controller='workflow', action='run', id=encoded_id )}" target="_parent">${_(workflow.name)}</a> + <a id="link-${workflow.id}" href="${ h.url_for( controller='workflow', action='run', id=encoded_id )}" target="_parent">${_( util.unicodify( workflow.name ) ) }</a></div></%def> @@ -122,7 +122,7 @@ %if t.user.stored_workflow_menu_entries: %for m in t.user.stored_workflow_menu_entries: <div class="toolTitle"> - <a href="${h.url_for( controller='workflow', action='run', id=trans.security.encode_id(m.stored_workflow_id) )}" target="galaxy_main">${m.stored_workflow.name}</a> + <a href="${h.url_for( controller='workflow', action='run', id=trans.security.encode_id(m.stored_workflow_id) )}" target="galaxy_main">${ util.unicodify( m.stored_workflow.name ) }</a></div> %endfor %endif https://bitbucket.org/galaxy/galaxy-central/commits/69240f7da05e/ Changeset: 69240f7da05e User: dan Date: 2013-05-15 17:13:59 Summary: Merge Affected #: 1 file diff -r 2e387c1384f1f7429a86e77e9a23b5329ca97b94 -r 69240f7da05e4ae42803b67a507103d201186306 templates/webapps/galaxy/root/tool_menu.mako --- a/templates/webapps/galaxy/root/tool_menu.mako +++ b/templates/webapps/galaxy/root/tool_menu.mako @@ -10,7 +10,7 @@ <div class="toolTitleNoSection"> %endif <% encoded_id = key.lstrip( 'workflow_' ) %> - <a id="link-${workflow.id}" href="${ h.url_for( controller='workflow', action='run', id=encoded_id )}" target="_parent">${_(workflow.name)}</a> + <a id="link-${workflow.id}" href="${ h.url_for( controller='workflow', action='run', id=encoded_id )}" target="_parent">${_( util.unicodify( workflow.name ) ) }</a></div></%def> @@ -122,7 +122,7 @@ %if t.user.stored_workflow_menu_entries: %for m in t.user.stored_workflow_menu_entries: <div class="toolTitle"> - <a href="${h.url_for( controller='workflow', action='run', id=trans.security.encode_id(m.stored_workflow_id) )}" target="galaxy_main">${m.stored_workflow.name}</a> + <a href="${h.url_for( controller='workflow', action='run', id=trans.security.encode_id(m.stored_workflow_id) )}" target="galaxy_main">${ util.unicodify( m.stored_workflow.name ) }</a></div> %endfor %endif https://bitbucket.org/galaxy/galaxy-central/commits/0f51ae4f1c84/ Changeset: 0f51ae4f1c84 Branch: stable User: dan Date: 2013-05-15 17:09:32 Summary: Unicode fixes for workflow names in Tool Menu. Affected #: 1 file diff -r 6861705712161c545398e6db1a7cdadda931fc7a -r 0f51ae4f1c84c5bab6cd1c93ae837696be9565e1 templates/webapps/galaxy/root/tool_menu.mako --- a/templates/webapps/galaxy/root/tool_menu.mako +++ b/templates/webapps/galaxy/root/tool_menu.mako @@ -10,7 +10,7 @@ <div class="toolTitleNoSection"> %endif <% encoded_id = key.lstrip( 'workflow_' ) %> - <a id="link-${workflow.id}" href="${ h.url_for( controller='workflow', action='run', id=encoded_id )}" target="_parent">${_(workflow.name)}</a> + <a id="link-${workflow.id}" href="${ h.url_for( controller='workflow', action='run', id=encoded_id )}" target="_parent">${_( util.unicodify( workflow.name ) ) }</a></div></%def> @@ -122,7 +122,7 @@ %if t.user.stored_workflow_menu_entries: %for m in t.user.stored_workflow_menu_entries: <div class="toolTitle"> - <a href="${h.url_for( controller='workflow', action='run', id=trans.security.encode_id(m.stored_workflow_id) )}" target="galaxy_main">${m.stored_workflow.name}</a> + <a href="${h.url_for( controller='workflow', action='run', id=trans.security.encode_id(m.stored_workflow_id) )}" target="galaxy_main">${ util.unicodify( m.stored_workflow.name ) }</a></div> %endfor %endif Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Add support for specifying the target filename in the download_file action. Set a tool dependency to the error state if the actions list specifies an unsupported action.
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/2e387c1384f1/ Changeset: 2e387c1384f1 User: Dave Bouvier Date: 2013-05-15 17:07:49 Summary: Add support for specifying the target filename in the download_file action. Set a tool dependency to the error state if the actions list specifies an unsupported action. Affected #: 2 files diff -r 866ed8fd31e146792980c61489d57cc18a5036b5 -r 2e387c1384f1f7429a86e77e9a23b5329ca97b94 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -92,7 +92,10 @@ # Download a single file to the working directory. filtered_actions = actions[ 1: ] url = action_dict[ 'url' ] - filename = url.split( '/' )[ -1 ] + if action_dict[ 'target_filename' ]: + filename = action_dict[ 'target_filename' ] + else: + filename = url.split( '/' )[ -1 ] common_util.url_download( work_dir, filename, url ) dir = os.path.curdir else: @@ -150,7 +153,10 @@ elif action_type == 'download_file': # Download a single file to the current directory. url = action_dict[ 'url' ] - filename = url.split( '/' )[ -1 ] + if action_dict[ 'target_filename' ]: + filename = action_dict[ 'target_filename' ] + else: + filename = url.split( '/' )[ -1 ] common_util.url_download( current_dir, filename, url ) def log_results( command, fabric_AttributeString, file_path ): diff -r 866ed8fd31e146792980c61489d57cc18a5036b5 -r 2e387c1384f1f7429a86e77e9a23b5329ca97b94 lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -345,6 +345,7 @@ # <action type="download_file">http://effectors.org/download/version/TTSS_GUI-1.0.1.jar</action> if action_elem.text: action_dict[ 'url' ] = action_elem.text + action_dict[ 'target_filename' ] = action_elem.attrib.get( 'target_filename', None ) else: continue elif action_type == 'make_directory': @@ -402,8 +403,8 @@ else: continue else: - log.debug( "Skipping unsupported action type '%s'." % str( action_type ) ) - continue + log.debug( "Unsupported action type '%s'. Not proceeding." % str( action_type ) ) + raise Exception( "Unsupported action type '%s' in tool dependency definition." % str( action_type ) ) actions.append( ( action_type, action_dict ) ) if actions: actions_dict[ 'actions' ] = actions Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Trackster: unify draw_helper functions for performance and clarity.
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/866ed8fd31e1/ Changeset: 866ed8fd31e1 User: jgoecks Date: 2013-05-15 16:09:55 Summary: Trackster: unify draw_helper functions for performance and clarity. Affected #: 1 file diff -r eb04cd61c978821c88f793ca3bc51783fd550c05 -r 866ed8fd31e146792980c61489d57cc18a5036b5 static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -3082,6 +3082,13 @@ }, /** + * Returns a list of drawables to draw. Defaults to current track. + */ + _get_drawables: function() { + return [ this ]; + }, + + /** * Retrieves from cache, draws, or sets up drawing for a single tile. Returns either a Tile object or a * jQuery.Deferred object that is fulfilled when tile can be drawn again. */ @@ -3090,6 +3097,7 @@ if (!kwargs) { kwargs = {}; } var track = this, + drawables = this._get_drawables(), key = this._gen_tile_cache_key(w_scale, region), is_tile = function(o) { return (o && 'track' in o); }, mode = kwargs.mode || track.mode; @@ -3104,17 +3112,19 @@ } // Function that returns data/Deferreds needed to draw tile. - var get_data = function() { - // Get the track data, maybe a deferred - var tile_data = track.data_manager.get_data(region, mode, resolution, track.data_url_extra_params); - - // Get reference data if needed, maybe a deferred - var seq_data; - if ( view.reference_track ) { - seq_data = view.reference_track.data_manager.get_data(region, mode, resolution, view.reference_track.data_url_extra_params); - } + var get_tile_data = function() { + // Map drawable object to data needed for drawing. + var tile_data = _.map(drawables, function(d) { + // Get the track data/promise. + return d.data_manager.get_data(region, mode, resolution, track.data_url_extra_params); + }); - return [tile_data, seq_data]; + // Get reference data/promise. + if (view.reference_track) { + tile_data.push(view.reference_track.data_manager.get_data(region, mode, resolution, view.reference_track.data_url_extra_params)); + } + + return tile_data; }; // @@ -3122,37 +3132,54 @@ // var tile_drawn = $.Deferred(); track.tile_cache.set_elt(key, tile_drawn); - $.when.apply($, get_data()).then( function() { - var data = get_data(), - tile_data = data[0], - seq_data = data[1]; - - // Set up and draw tile. - extend(tile_data, kwargs[ 'more_tile_data' ] ); - + $.when.apply($, get_tile_data()).then( function() { + var tile_data = get_tile_data(), + tracks_data = tile_data.slice(0, tile_data.length - 1), + seq_data = tile_data[tile_data.length - 1]; + // If sequence data is available, subset to get only data in region. - if (view.reference_track) { + if (seq_data) { seq_data = view.reference_track.data_manager.subset_entry(seq_data, region); } - // HACK: this is FeatureTrack-specific. - // If track mode is Auto, determine mode and update. - if (mode === "Auto" && track.get_mode) { - mode = track.get_mode(tile_data); - track.update_auto_mode(mode); - } - - // Draw canvas. + // Get drawing modes, heights for all tracks. + var drawing_modes = [], + drawing_heights = []; + + _.each(drawables, function(d, i) { + var mode = d.mode, + data = tracks_data[i]; + if (mode === "Auto") { + mode = d.get_mode(data); + d.update_auto_mode(mode); + } + drawing_modes.push(mode); + drawing_heights.push(d.get_canvas_height(data, mode, w_scale, width)); + }) + var canvas = track.view.canvas_manager.new_canvas(), tile_low = region.get('start'), tile_high = region.get('end'), + all_data_index = 0, width = Math.ceil( (tile_high - tile_low) * w_scale ) + track.left_offset, - height = track.get_canvas_height(tile_data, mode, w_scale, width); + height = _.max(drawing_heights), + tile; + + // + // Draw all tracks on tile. + // canvas.width = width; - canvas.height = height; + // Height is specified in kwargs or is the height found above. + canvas.height = (kwargs.height ? kwargs.height : height); var ctx = canvas.getContext('2d'); ctx.translate(track.left_offset, 0); - var tile = track.draw_tile(tile_data, ctx, mode, resolution, region, w_scale, seq_data); + if (drawables.length > 1) { + ctx.globalAlpha = 0.5; + ctx.globalCompositeOperation = "source-over"; + } + _.each(drawables, function(d, i) { + tile = d.draw_tile(tracks_data[i], ctx, drawing_modes[i], resolution, region, w_scale, seq_data); + }); // Don't cache, show if no tile. if (tile !== undefined) { @@ -3470,126 +3497,8 @@ can_draw: Drawable.prototype.can_draw, - draw_helper: function(force, region, resolution, parent_element, w_scale, kwargs) { - // FIXME: this function is similar to TiledTrack.draw_helper -- can the two be merged/refactored? - - // SHARED block with TiledTrack.draw_helper(): - - var track = this, - key = this._gen_tile_cache_key(w_scale, region), - is_tile = function(o) { return (o && 'track' in o); }; - - // Init kwargs if necessary to avoid having to check if kwargs defined. - if (!kwargs) { kwargs = {}; } - - // Check tile cache, if found show existing tile in correct position - var tile = (force ? undefined : track.tile_cache.get_elt(key)); - if (tile) { - if (is_tile(tile)) { - track.show_tile(tile, parent_element, w_scale); - } - return tile; - } - - // UNIQUE block. - - // Try to get drawables' data. - var all_data = [], - track, - // Flag to track whether we can draw everything now - can_draw_now = true, - tile_data, - seq_data; - for (var i = 0; i < this.drawables.length; i++) { - track = this.drawables[i]; - // Get the track data, maybe a deferred. - tile_data = track.data_manager.get_data(region, track.mode, resolution, track.data_url_extra_params); - if ( is_deferred( tile_data ) ) { - can_draw_now = false; - } - all_data.push(tile_data); - - // Get seq data if needed, maybe a deferred. - seq_data = null; - if ( view.reference_track && w_scale > view.canvas_manager.char_width_px ) { - seq_data = view.reference_track.data_manager.get_data(region, track.mode, resolution, view.reference_track.data_url_extra_params); - if ( is_deferred( seq_data ) ) { - can_draw_now = false; - } - } - all_data.push(seq_data); - } - - // If we can draw now, do so. - if ( can_draw_now ) { - // Set up and draw tile. - extend(tile_data, kwargs[ 'more_tile_data' ] ); - - var canvas = track.view.canvas_manager.new_canvas(), - tile_low = region.get('start'), - tile_high = region.get('end'), - all_data_index = 0, - width = Math.ceil( (tile_high - tile_low) * w_scale ) + this.left_offset, - height = 0, - track_modes = [], - i; - - // Get max height for all tracks and record track modes. - var track_canvas_height = 0; - for (i = 0; i < this.drawables.length; i++, all_data_index += 2) { - track = this.drawables[i]; - tile_data = all_data[ all_data_index ]; - - // HACK: this is FeatureTrack-specific. - // If track mode is Auto, determine mode and update. - var mode = track.mode; - if (mode === "Auto") { - mode = track.get_mode(tile_data); - track.update_auto_mode(mode); - } - track_modes.push(mode); - - track_canvas_height = track.get_canvas_height(tile_data, mode, w_scale, width); - if (track_canvas_height > height) { height = track_canvas_height; } - } - - // - // Draw all tracks on a single tile. - // - canvas.width = width; - // Height is specified in kwargs or is the height found above. - canvas.height = (kwargs.height ? kwargs.height : height); - all_data_index = 0; - var ctx = canvas.getContext('2d'); - ctx.translate(this.left_offset, 0); - ctx.globalAlpha = 0.5; - ctx.globalCompositeOperation = "source-over"; - for (i = 0; i < this.drawables.length; i++, all_data_index += 2) { - track = this.drawables[i]; - tile_data = all_data[ all_data_index ]; - seq_data = all_data[ all_data_index + 1 ]; - tile = track.draw_tile(tile_data, ctx, track_modes[i], resolution, region, w_scale, seq_data); - } - - // Don't cache, show if no tile. - this.tile_cache.set_elt(key, tile); - this.show_tile(tile, parent_element, w_scale); - return tile; - } - - // SHARED (somewhat) block with TiledTrack.draw_helper() - - // Can't draw now, so put Deferred in cache and draw tile when data is available. - var tile_drawn = $.Deferred(); - track = this; - track.tile_cache.set_elt(key, tile_drawn); - $.when.apply($, all_data).then(function() { - // Draw tile--force to clear Deferred from cache--and resolve. - tile = track.draw_helper(true, region, resolution, parent_element, w_scale, kwargs); - tile_drawn.resolve(tile); - }); - - return tile_drawn; + _get_drawables: function() { + return this.drawables; }, /** Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Trackster bug fix: use computed mode rather than track mode.
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/eb04cd61c978/ Changeset: eb04cd61c978 User: jgoecks Date: 2013-05-15 15:17:41 Summary: Trackster bug fix: use computed mode rather than track mode. Affected #: 1 file diff -r d6125e3f660caa376323218546b72e4f953dc669 -r eb04cd61c978821c88f793ca3bc51783fd550c05 static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -3106,12 +3106,12 @@ // Function that returns data/Deferreds needed to draw tile. var get_data = function() { // Get the track data, maybe a deferred - var tile_data = track.data_manager.get_data(region, track.mode, resolution, track.data_url_extra_params); + var tile_data = track.data_manager.get_data(region, mode, resolution, track.data_url_extra_params); // Get reference data if needed, maybe a deferred var seq_data; if ( view.reference_track ) { - seq_data = view.reference_track.data_manager.get_data(region, track.mode, resolution, view.reference_track.data_url_extra_params); + seq_data = view.reference_track.data_manager.get_data(region, mode, resolution, view.reference_track.data_url_extra_params); } return [tile_data, seq_data]; Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Kyle Ellrott: Starting work on 'read-only': object stores that can be added to the distributed object store system and provide remote access to additional data sources
by commits-noreply＠bitbucket.org 14 May '13

14 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/73858b780fdc/ Changeset: 73858b780fdc Branch: federate User: Kyle Ellrott Date: 2013-05-15 01:53:32 Summary: Starting work on 'read-only': object stores that can be added to the distributed object store system and provide remote access to additional data sources Affected #: 4 files diff -r 6d72b2db32c0d84032ee40f2cd86b7b09485c441 -r 73858b780fdcede620cb0e42409a856367e330b4 distributed_object_store_conf.xml --- /dev/null +++ b/distributed_object_store_conf.xml @@ -0,0 +1,16 @@ +<?xml version="1.0"?> +<backends> + <backend id="files1" type="disk" weight="1"> + <files_dir path="database/files1"/> + <extra_dir type="temp" path="database/tmp1"/> + <extra_dir type="job_work" path="database/job_working_directory1"/> + </backend> + <backend id="files2" type="disk" weight="1"> + <files_dir path="database/files2"/> + <extra_dir type="temp" path="database/tmp2"/> + <extra_dir type="job_work" path="database/job_working_directory2"/> + </backend> + <backend id="galaxy_main" type="galaxy.objectstore.remote:RemoteGalaxyStore"> + <url>https://main.g2.bx.psu.edu/</url> + </backend> +</backends> diff -r 6d72b2db32c0d84032ee40f2cd86b7b09485c441 -r 73858b780fdcede620cb0e42409a856367e330b4 distributed_object_store_conf.xml.sample --- a/distributed_object_store_conf.xml.sample +++ b/distributed_object_store_conf.xml.sample @@ -10,4 +10,7 @@ <extra_dir type="temp" path="database/tmp2"/><extra_dir type="job_work" path="database/job_working_directory2"/></backend> + </backends> diff -r 6d72b2db32c0d84032ee40f2cd86b7b09485c441 -r 73858b780fdcede620cb0e42409a856367e330b4 lib/galaxy/objectstore/__init__.py --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -36,6 +36,7 @@ """ ObjectStore abstract interface """ + is_readonly = False def __init__(self): self.running = True self.extra_dirs = {} @@ -895,7 +896,8 @@ id = elem.get('id') weight = int(elem.get('weight', 1)) maxpctfull = float(elem.get('maxpctfull', 0)) - if elem.get('type', 'disk'): + ostype = elem.get('type', 'disk') + if ostype == 'disk': path = None extra_dirs = {} for sub in elem: @@ -911,6 +913,20 @@ log.debug(" Extra directories:") for type, dir in extra_dirs.items(): log.debug(" %s: %s" % (type, dir)) + else: + fields = ostype.split( ':' ) + ostype_module = fields[0] + ostype_class_name = fields[1] + mfields = ostype_module.split( '.' ) + module = __import__( ostype_module ) + for mod in mfields[1:]: + module = getattr( module, mod ) + ostype_class = getattr( module, ostype_class_name ) + obj = ostype_class({}) + self.backends[id] = obj + if obj.is_readonly: + weight = 0 + for i in range(0, weight): # The simplest way to do weighting: add backend ids to a # sequence the number of times equalling weight, then randomly diff -r 6d72b2db32c0d84032ee40f2cd86b7b09485c441 -r 73858b780fdcede620cb0e42409a856367e330b4 lib/galaxy/objectstore/remote.py --- /dev/null +++ b/lib/galaxy/objectstore/remote.py @@ -0,0 +1,8 @@ + +from galaxy.objectstore import ObjectStore + +class RemoteGalaxyStore(ObjectStore): + is_readonly = True + def __init__(self, params): + print "Remote Galaxy Store Loaded!!!!!" + \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 14 May '13

14 May '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e99b00df40b2/ Changeset: e99b00df40b2 User: Dave Bouvier Date: 2013-05-14 22:54:12 Summary: Fix for reinstalling an uninstalled repository when leaving the no changes checkbox checked and entering text into the new tool panel section field. Affected #: 2 files diff -r 6d72b2db32c0d84032ee40f2cd86b7b09485c441 -r e99b00df40b2895ee1a36d356dc5c0b968f80f8c lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -445,7 +445,8 @@ try: tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ] except KeyError: - log.debug( 'Invalid tool_panel_section_key "%s" specified. Tools will be loaded outside of sections in the tool panel.' ) + log.debug( 'Invalid tool_panel_section_key "%s" specified. Tools will be loaded outside of sections in the tool panel.', + str( tool_panel_section_key ) ) tool_section = None else: tool_section = None diff -r 6d72b2db32c0d84032ee40f2cd86b7b09485c441 -r e99b00df40b2895ee1a36d356dc5c0b968f80f8c lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -617,6 +617,9 @@ if 'tools' in metadata: # This forces everything to be loaded into the same section (or no section) in the tool panel. if no_changes_checked: + # Make sure the no_changes checkbox overrides the new_tool_panel_section if the user checked the checkbox and + # entered something into the field. + new_tool_panel_section = None if 'tool_panel_section' in metadata: tool_panel_dict = metadata[ 'tool_panel_section' ] if not tool_panel_dict: https://bitbucket.org/galaxy/galaxy-central/commits/d6125e3f660c/ Changeset: d6125e3f660c User: Dave Bouvier Date: 2013-05-14 22:55:16 Summary: Merge. Affected #: 1 file diff -r e99b00df40b2895ee1a36d356dc5c0b968f80f8c -r d6125e3f660caa376323218546b72e4f953dc669 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -362,7 +362,7 @@ data_tables.append( data_table_name ) if bad_data_table: continue - data_manager_metadata_tool_file = os.path.join( relative_data_manager_dir, tool_file ) + data_manager_metadata_tool_file = os.path.normpath( os.path.join( relative_data_manager_dir, tool_file ) ) tool_metadata_tool_file = os.path.join( repo_files_directory, data_manager_metadata_tool_file ) tool = tools.get( tool_metadata_tool_file, None ) if tool is None: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Fix for determining Data Manager Tool Filename.
by commits-noreply＠bitbucket.org 14 May '13

14 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8483f0ef8288/ Changeset: 8483f0ef8288 User: dan Date: 2013-05-14 22:52:58 Summary: Fix for determining Data Manager Tool Filename. Affected #: 1 file diff -r 6d72b2db32c0d84032ee40f2cd86b7b09485c441 -r 8483f0ef8288b6eecf65a556c75b65e944a00bb0 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -362,7 +362,7 @@ data_tables.append( data_table_name ) if bad_data_table: continue - data_manager_metadata_tool_file = os.path.join( relative_data_manager_dir, tool_file ) + data_manager_metadata_tool_file = os.path.normpath( os.path.join( relative_data_manager_dir, tool_file ) ) tool_metadata_tool_file = os.path.join( repo_files_directory, data_manager_metadata_tool_file ) tool = tools.get( tool_metadata_tool_file, None ) if tool is None: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Ehancements for the tool shed repository View changeset page: display the commit message, display both the ctx rev and the changeset hash in the title bar, and add Previous changeset and Next changeset buttons to allow for easier scrolling through the repository changelog.
by commits-noreply＠bitbucket.org 14 May '13

14 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6d72b2db32c0/ Changeset: 6d72b2db32c0 User: greg Date: 2013-05-14 21:58:09 Summary: Ehancements for the tool shed repository View changeset page: display the commit message, display both the ctx rev and the changeset hash in the title bar, and add Previous changeset and Next changeset buttons to allow for easier scrolling through the repository changelog. Affected #: 2 files diff -r c1f961549cca546214ff4430dd6a01c42cfeaa63 -r 6d72b2db32c0d84032ee40f2cd86b7b09485c441 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -2681,17 +2681,39 @@ id=id, message=message, status=status ) ) - ctx_parent = ctx.parents()[0] + ctx_parent = ctx.parents()[ 0 ] + if ctx.children(): + ctx_child = ctx.children()[ 0 ] + else: + ctx_child = None modified, added, removed, deleted, unknown, ignored, clean = repo.status( node1=ctx_parent.node(), node2=ctx.node() ) anchors = modified + added + removed + deleted + unknown + ignored + clean diffs = [] for diff in patch.diff( repo, node1=ctx_parent.node(), node2=ctx.node() ): diffs.append( suc.to_safe_string( diff, to_html=True ) ) metadata = self.get_metadata( trans, id, ctx_str ) + # For rendering the prev button. + if ctx_parent: + ctx_parent_rev = ctx_parent.rev() + if ctx_parent_rev < 0: + prev = None + else: + prev = "%s:%s" % ( ctx_parent_rev, ctx_parent ) + else: + prev = None + if ctx_child: + ctx_child_rev = ctx_child.rev() + next = "%s:%s" % ( ctx_child_rev, ctx_child ) + else: + next = None return trans.fill_template( '/webapps/tool_shed/repository/view_changeset.mako', repository=repository, metadata=metadata, + prev=prev, + next=next, ctx=ctx, + ctx_parent=ctx_parent, + ctx_child=ctx_child, anchors=anchors, modified=modified, added=added, diff -r c1f961549cca546214ff4430dd6a01c42cfeaa63 -r 6d72b2db32c0d84032ee40f2cd86b7b09485c441 templates/webapps/tool_shed/repository/view_changeset.mako --- a/templates/webapps/tool_shed/repository/view_changeset.mako +++ b/templates/webapps/tool_shed/repository/view_changeset.mako @@ -50,13 +50,33 @@ <div class="toolForm"><% if can_download: - title_str = 'Changeset %s' % ctx + title_str = 'Changeset %s:%s' % ( ctx.rev(), ctx ) else: - title_str = '%s changeset %s' % ( repository.name, ctx ) + title_str = '%s changeset %s:%s' % ( repository.name, ctx.rev(), ctx ) %> - <div class="toolFormTitle">${title_str | h}</div> + <div class="toolFormTitle"> + ${title_str | h} + </div><div class="toolFormBody"><table class="grid"> + %if prev or next: + <tr> + <td> + %if prev: + <a class="action-button" href="${h.url_for( controller='repository', action='view_changeset', id=trans.security.encode_id( repository.id ), ctx_str=ctx_parent )}">Previous changeset ${prev | h}</a> + %endif + %if next: + <a class="action-button" href="${h.url_for( controller='repository', action='view_changeset', id=trans.security.encode_id( repository.id ), ctx_str=ctx_child )}">Next changeset ${next | h}</a> + %endif + </td> + </tr> + %endif + <tr> + <td> + <b>Commit message:</b> + <br/>${ctx.description() | h}<br/> + </td> + </tr> %if modified: <tr><td> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Fix for detecting job error in data manager tools; needed as a result of recent job flow changes.
by commits-noreply＠bitbucket.org 14 May '13

14 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c1f961549cca/ Changeset: c1f961549cca User: dan Date: 2013-05-14 21:54:33 Summary: Fix for detecting job error in data manager tools; needed as a result of recent job flow changes. Affected #: 1 file diff -r 4573880b2d9b7a76abcbe2ccac824ab9338df802 -r c1f961549cca546214ff4430dd6a01c42cfeaa63 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -3157,6 +3157,10 @@ #process results of tool if job and job.state == job.states.ERROR: return + #Job state may now be 'running' instead of previous 'error', but datasets are still set to e.g. error + for dataset in out_data.itervalues(): + if dataset.state != dataset.states.OK: + return data_manager_id = job.data_manager_association.data_manager_id data_manager = self.app.data_managers.get_manager( data_manager_id, None ) assert data_manager is not None, "Invalid data manager (%s) requested. It may have been removed before the job completed." % ( data_manager_id ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 4 new changesets
by commits-noreply＠bitbucket.org 14 May '13

14 May '13

4 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/2c0c1070891c/ Changeset: 2c0c1070891c User: jgoecks Date: 2013-05-14 19:15:02 Summary: Trackster: LinePainter improvements to use block_color when drawing coverage plots and refactor min/max finding to us underscore functions. Affected #: 1 file diff -r 8882e45504a37ab9f87f2872df2a6749673bd741 -r 2c0c1070891cdef382f32c0c3780a06d9169a10d static/scripts/viz/trackster/painters.js --- a/static/scripts/viz/trackster/painters.js +++ b/static/scripts/viz/trackster/painters.js @@ -135,20 +135,11 @@ var LinePainter = function(data, view_start, view_end, prefs, mode) { Painter.call( this, data, view_start, view_end, prefs, mode ); - var i, len; if ( this.prefs.min_value === undefined ) { - var min_value = Infinity; - for (i = 0, len = this.data.length; i < len; i++) { - min_value = Math.min( min_value, this.data[i][1] ); - } - this.prefs.min_value = min_value; + this.prefs.min_value = _.min( _.map(this.data, function(d) { return d[1]; }) ) || 0; } if ( this.prefs.max_value === undefined ) { - var max_value = -Infinity; - for (i = 0, len = this.data.length; i < len; i++) { - max_value = Math.max( max_value, this.data[i][1] ); - } - this.prefs.max_value = max_value; + this.prefs.max_value = _.max( _.map(this.data, function(d) { return d[1]; }) ) || 0; } }; @@ -184,15 +175,17 @@ delta_x_px = 10; } - // Extract RGB from preference color. - var pref_color = parseInt( this.prefs.color.slice(1), 16 ), + // Painter color can be in either block_color (FeatureTrack) or color pref (LineTrack). + var painter_color = this.prefs.block_color || this.prefs.color, + // Extract RGB from preference color. + pref_color = parseInt( painter_color.slice(1), 16 ), pref_r = (pref_color & 0xff0000) >> 16, pref_g = (pref_color & 0x00ff00) >> 8, pref_b = pref_color & 0x0000ff; // Paint track. for (var i = 0, len = data.length; i < len; i++) { - ctx.fillStyle = ctx.strokeStyle = this.prefs.color; + ctx.fillStyle = ctx.strokeStyle = painter_color; // -0.5 to offset drawing between bases. x_scaled = Math.round((data[i][0] - view_start - 0.5) * w_scale); y = data[i][1]; @@ -261,7 +254,7 @@ ctx.fillRect(x_scaled, height_px - 3, overflow_x, 3); } } - ctx.fillStyle = this.prefs.color; + ctx.fillStyle = painter_color; } if (mode === "Filled") { if (in_path) { https://bitbucket.org/galaxy/galaxy-central/commits/f4f596595970/ Changeset: f4f596595970 User: jgoecks Date: 2013-05-14 20:56:09 Summary: Add interval-to-bigwig converter, which creates a bigwig of coverage data for an interval dataset. Affected #: 2 files diff -r 2c0c1070891cdef382f32c0c3780a06d9169a10d -r f4f59659597023ad3b68b631da3b085dc98b15ed datatypes_conf.xml.sample --- a/datatypes_conf.xml.sample +++ b/datatypes_conf.xml.sample @@ -17,7 +17,6 @@ </datatype><datatype extension="bed" type="galaxy.datatypes.interval:Bed" display_in_upload="true"><converter file="bed_to_gff_converter.xml" target_datatype="gff"/> - <converter file="interval_to_coverage.xml" target_datatype="coverage"/><converter file="bed_to_bgzip_converter.xml" target_datatype="bgzip"/><converter file="bed_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/><converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/> diff -r 2c0c1070891cdef382f32c0c3780a06d9169a10d -r f4f59659597023ad3b68b631da3b085dc98b15ed lib/galaxy/datatypes/converters/interval_to_bigwig_converter.xml --- /dev/null +++ b/lib/galaxy/datatypes/converters/interval_to_bigwig_converter.xml @@ -0,0 +1,33 @@ +<tool id="CONVERTER_interval_to_bigwig_0" name="Convert Genomic Intervals To Coverage"> +  +  + <command> + + ## Remove comments and sort by chromosome. + grep -v '^#' $input1 | sort -k${input1.metadata.chromCol},${input1.metadata.chromCol} | + + ## Create simple BED by cutting chrom, start, and end columns. + awk -v OFS=' ' '{print $${input1.metadata.chromCol},$${input1.metadata.startCol},$${input1.metadata.endCol} }' | + + ## Generate coverage bedgraph. + bedtools genomecov -bg -split -i stdin -g $chromInfo + + ## Streaming the bedgraph file to wigToBigWig is fast but very memory intensive; hence, this + ## should only be used on systems with large RAM. + ## | wigToBigWig stdin $chromInfo $output + + ## This can be used anywhere. + > temp.bg ; bedGraphToBigWig temp.bg $chromInfo $output + + </command> + <inputs> + <page> + <param format="interval" name="input1" type="data" label="Choose intervals"/> + </page> + </inputs> + <outputs> + <data format="bigwig" name="output"/> + </outputs> + <help> + </help> +</tool> https://bitbucket.org/galaxy/galaxy-central/commits/15224c9c15d9/ Changeset: 15224c9c15d9 User: jgoecks Date: 2013-05-14 21:46:27 Summary: Trackster: use deferred rather than flags to manage tile drawing. Affected #: 1 file diff -r f4f59659597023ad3b68b631da3b085dc98b15ed -r 15224c9c15d9de6323892207fff44520a23e3084 static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -3102,34 +3102,39 @@ } return tile; } - - // Flag to track whether we can draw everything now - var can_draw_now = true; - - // Get the track data, maybe a deferred - var tile_data = track.data_manager.get_data(region, mode, resolution, track.data_url_extra_params); - if ( is_deferred( tile_data ) ) { - can_draw_now = false; - } - - // Get reference data if needed, maybe a deferred - var seq_data; - if ( view.reference_track ) { - seq_data = view.reference_track.data_manager.get_data(region, mode, resolution, view.reference_track.data_url_extra_params); - if ( is_deferred( seq_data ) ) { - can_draw_now = false; + + // Function that returns data/Deferreds needed to draw tile. + var get_data = function() { + // Get the track data, maybe a deferred + var tile_data = track.data_manager.get_data(region, track.mode, resolution, track.data_url_extra_params); + + // Get reference data if needed, maybe a deferred + var seq_data; + if ( view.reference_track ) { + seq_data = view.reference_track.data_manager.get_data(region, track.mode, resolution, view.reference_track.data_url_extra_params); + } + + return [tile_data, seq_data]; + }; + + // + // When data is available, draw tile. + // + var tile_drawn = $.Deferred(); + track.tile_cache.set_elt(key, tile_drawn); + $.when.apply($, get_data()).then( function() { + var data = get_data(), + tile_data = data[0], + seq_data = data[1]; + + // Set up and draw tile. + extend(tile_data, kwargs[ 'more_tile_data' ] ); + + // If sequence data is available, subset to get only data in region. + if (view.reference_track) { + seq_data = view.reference_track.data_manager.subset_entry(seq_data, region); } - else { - // Sequence data is available, subset to get only data in region. - seq_data = view.reference_track.data_manager.subset_entry(seq_data, region); - } - } - - // If we can draw now, do so. - if ( can_draw_now ) { - // Set up and draw tile. - extend(tile_data, kwargs[ 'more_tile_data' ] ); - + // HACK: this is FeatureTrack-specific. // If track mode is Auto, determine mode and update. if (mode === "Auto" && track.get_mode) { @@ -3138,8 +3143,7 @@ } // Draw canvas. - var - canvas = track.view.canvas_manager.new_canvas(), + var canvas = track.view.canvas_manager.new_canvas(), tile_low = region.get('start'), tile_high = region.get('end'), width = Math.ceil( (tile_high - tile_low) * w_scale ) + track.left_offset, @@ -3147,26 +3151,18 @@ canvas.width = width; canvas.height = height; var ctx = canvas.getContext('2d'); - ctx.translate(this.left_offset, 0); + ctx.translate(track.left_offset, 0); var tile = track.draw_tile(tile_data, ctx, mode, resolution, region, w_scale, seq_data); - + // Don't cache, show if no tile. if (tile !== undefined) { track.tile_cache.set_elt(key, tile); track.show_tile(tile, parent_element, w_scale); } - return tile; - } - - // Can't draw now, so put Deferred in cache and draw tile when data is available. - var tile_drawn = $.Deferred(); - track.tile_cache.set_elt(key, tile_drawn); - $.when( tile_data, seq_data ).then( function() { - // Draw tile--force to clear Deferred from cache--and resolve. - tile = track.draw_helper(true, region, resolution, parent_element, w_scale, kwargs); + tile_drawn.resolve(tile); }); - + return tile_drawn; }, https://bitbucket.org/galaxy/galaxy-central/commits/4573880b2d9b/ Changeset: 4573880b2d9b User: jgoecks Date: 2013-05-14 21:47:42 Summary: Automated merge. Affected #: 4 files diff -r 1a4b56a78124b26103c61f8e6e736b088e65aa6d -r 4573880b2d9b7a76abcbe2ccac824ab9338df802 datatypes_conf.xml.sample --- a/datatypes_conf.xml.sample +++ b/datatypes_conf.xml.sample @@ -17,7 +17,6 @@ </datatype><datatype extension="bed" type="galaxy.datatypes.interval:Bed" display_in_upload="true"><converter file="bed_to_gff_converter.xml" target_datatype="gff"/> - <converter file="interval_to_coverage.xml" target_datatype="coverage"/><converter file="bed_to_bgzip_converter.xml" target_datatype="bgzip"/><converter file="bed_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/><converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/> diff -r 1a4b56a78124b26103c61f8e6e736b088e65aa6d -r 4573880b2d9b7a76abcbe2ccac824ab9338df802 lib/galaxy/datatypes/converters/interval_to_bigwig_converter.xml --- /dev/null +++ b/lib/galaxy/datatypes/converters/interval_to_bigwig_converter.xml @@ -0,0 +1,33 @@ +<tool id="CONVERTER_interval_to_bigwig_0" name="Convert Genomic Intervals To Coverage"> +  +  + <command> + + ## Remove comments and sort by chromosome. + grep -v '^#' $input1 | sort -k${input1.metadata.chromCol},${input1.metadata.chromCol} | + + ## Create simple BED by cutting chrom, start, and end columns. + awk -v OFS=' ' '{print $${input1.metadata.chromCol},$${input1.metadata.startCol},$${input1.metadata.endCol} }' | + + ## Generate coverage bedgraph. + bedtools genomecov -bg -split -i stdin -g $chromInfo + + ## Streaming the bedgraph file to wigToBigWig is fast but very memory intensive; hence, this + ## should only be used on systems with large RAM. + ## | wigToBigWig stdin $chromInfo $output + + ## This can be used anywhere. + > temp.bg ; bedGraphToBigWig temp.bg $chromInfo $output + + </command> + <inputs> + <page> + <param format="interval" name="input1" type="data" label="Choose intervals"/> + </page> + </inputs> + <outputs> + <data format="bigwig" name="output"/> + </outputs> + <help> + </help> +</tool> diff -r 1a4b56a78124b26103c61f8e6e736b088e65aa6d -r 4573880b2d9b7a76abcbe2ccac824ab9338df802 static/scripts/viz/trackster/painters.js --- a/static/scripts/viz/trackster/painters.js +++ b/static/scripts/viz/trackster/painters.js @@ -135,20 +135,11 @@ var LinePainter = function(data, view_start, view_end, prefs, mode) { Painter.call( this, data, view_start, view_end, prefs, mode ); - var i, len; if ( this.prefs.min_value === undefined ) { - var min_value = Infinity; - for (i = 0, len = this.data.length; i < len; i++) { - min_value = Math.min( min_value, this.data[i][1] ); - } - this.prefs.min_value = min_value; + this.prefs.min_value = _.min( _.map(this.data, function(d) { return d[1]; }) ) || 0; } if ( this.prefs.max_value === undefined ) { - var max_value = -Infinity; - for (i = 0, len = this.data.length; i < len; i++) { - max_value = Math.max( max_value, this.data[i][1] ); - } - this.prefs.max_value = max_value; + this.prefs.max_value = _.max( _.map(this.data, function(d) { return d[1]; }) ) || 0; } }; @@ -184,15 +175,17 @@ delta_x_px = 10; } - // Extract RGB from preference color. - var pref_color = parseInt( this.prefs.color.slice(1), 16 ), + // Painter color can be in either block_color (FeatureTrack) or color pref (LineTrack). + var painter_color = this.prefs.block_color || this.prefs.color, + // Extract RGB from preference color. + pref_color = parseInt( painter_color.slice(1), 16 ), pref_r = (pref_color & 0xff0000) >> 16, pref_g = (pref_color & 0x00ff00) >> 8, pref_b = pref_color & 0x0000ff; // Paint track. for (var i = 0, len = data.length; i < len; i++) { - ctx.fillStyle = ctx.strokeStyle = this.prefs.color; + ctx.fillStyle = ctx.strokeStyle = painter_color; // -0.5 to offset drawing between bases. x_scaled = Math.round((data[i][0] - view_start - 0.5) * w_scale); y = data[i][1]; @@ -261,7 +254,7 @@ ctx.fillRect(x_scaled, height_px - 3, overflow_x, 3); } } - ctx.fillStyle = this.prefs.color; + ctx.fillStyle = painter_color; } if (mode === "Filled") { if (in_path) { diff -r 1a4b56a78124b26103c61f8e6e736b088e65aa6d -r 4573880b2d9b7a76abcbe2ccac824ab9338df802 static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -3102,34 +3102,39 @@ } return tile; } - - // Flag to track whether we can draw everything now - var can_draw_now = true; - - // Get the track data, maybe a deferred - var tile_data = track.data_manager.get_data(region, mode, resolution, track.data_url_extra_params); - if ( is_deferred( tile_data ) ) { - can_draw_now = false; - } - - // Get reference data if needed, maybe a deferred - var seq_data; - if ( view.reference_track ) { - seq_data = view.reference_track.data_manager.get_data(region, mode, resolution, view.reference_track.data_url_extra_params); - if ( is_deferred( seq_data ) ) { - can_draw_now = false; + + // Function that returns data/Deferreds needed to draw tile. + var get_data = function() { + // Get the track data, maybe a deferred + var tile_data = track.data_manager.get_data(region, track.mode, resolution, track.data_url_extra_params); + + // Get reference data if needed, maybe a deferred + var seq_data; + if ( view.reference_track ) { + seq_data = view.reference_track.data_manager.get_data(region, track.mode, resolution, view.reference_track.data_url_extra_params); + } + + return [tile_data, seq_data]; + }; + + // + // When data is available, draw tile. + // + var tile_drawn = $.Deferred(); + track.tile_cache.set_elt(key, tile_drawn); + $.when.apply($, get_data()).then( function() { + var data = get_data(), + tile_data = data[0], + seq_data = data[1]; + + // Set up and draw tile. + extend(tile_data, kwargs[ 'more_tile_data' ] ); + + // If sequence data is available, subset to get only data in region. + if (view.reference_track) { + seq_data = view.reference_track.data_manager.subset_entry(seq_data, region); } - else { - // Sequence data is available, subset to get only data in region. - seq_data = view.reference_track.data_manager.subset_entry(seq_data, region); - } - } - - // If we can draw now, do so. - if ( can_draw_now ) { - // Set up and draw tile. - extend(tile_data, kwargs[ 'more_tile_data' ] ); - + // HACK: this is FeatureTrack-specific. // If track mode is Auto, determine mode and update. if (mode === "Auto" && track.get_mode) { @@ -3138,8 +3143,7 @@ } // Draw canvas. - var - canvas = track.view.canvas_manager.new_canvas(), + var canvas = track.view.canvas_manager.new_canvas(), tile_low = region.get('start'), tile_high = region.get('end'), width = Math.ceil( (tile_high - tile_low) * w_scale ) + track.left_offset, @@ -3147,26 +3151,18 @@ canvas.width = width; canvas.height = height; var ctx = canvas.getContext('2d'); - ctx.translate(this.left_offset, 0); + ctx.translate(track.left_offset, 0); var tile = track.draw_tile(tile_data, ctx, mode, resolution, region, w_scale, seq_data); - + // Don't cache, show if no tile. if (tile !== undefined) { track.tile_cache.set_elt(key, tile); track.show_tile(tile, parent_element, w_scale); } - return tile; - } - - // Can't draw now, so put Deferred in cache and draw tile when data is available. - var tile_drawn = $.Deferred(); - track.tile_cache.set_elt(key, tile_drawn); - $.when( tile_data, seq_data ).then( function() { - // Draw tile--force to clear Deferred from cache--and resolve. - tile = track.draw_helper(true, region, resolution, parent_element, w_scale, kwargs); + tile_drawn.resolve(tile); }); - + return tile_drawn; }, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Display the Reset all metadata on repository option in the Resporitory Actions popup menu in more pages in the tool shed than the Manage repsitory page.
by commits-noreply＠bitbucket.org 14 May '13

14 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1a4b56a78124/ Changeset: 1a4b56a78124 User: greg Date: 2013-05-14 20:30:08 Summary: Display the Reset all metadata on repository option in the Resporitory Actions popup menu in more pages in the tool shed than the Manage repsitory page. Affected #: 2 files diff -r 1cabd9fd107c14caa379e591ff3fea66c3cc5ce4 -r 1a4b56a78124b26103c61f8e6e736b088e65aa6d templates/webapps/tool_shed/common/repository_actions_menu.mako --- a/templates/webapps/tool_shed/common/repository_actions_menu.mako +++ b/templates/webapps/tool_shed/common/repository_actions_menu.mako @@ -1,6 +1,6 @@ <%inherit file="/base.mako"/> -<%def name="render_tool_shed_repository_actions( repository, repo=None, metadata=None, changeset_revision=None )"> +<%def name="render_tool_shed_repository_actions( repository, metadata=None, changeset_revision=None )"><% from tool_shed.util.review_util import can_browse_repository_reviews, changeset_revision_reviewed_by_user, get_review_by_repository_id_changeset_revision_user_id from tool_shed.util.shed_util_common import changeset_is_malicious @@ -54,11 +54,8 @@ else: can_download = False - if not is_deprecated: - if repo and len( repo ) > 0: - can_reset_all_metadata = True - else: - can_reset_all_metadata = False + if ( is_admin or ( trans.user and trans.user == repository.user ) ) and not repository.deleted and not repository.deprecated and not is_new: + can_reset_all_metadata = True else: can_reset_all_metadata = False diff -r 1cabd9fd107c14caa379e591ff3fea66c3cc5ce4 -r 1a4b56a78124b26103c61f8e6e736b088e65aa6d templates/webapps/tool_shed/repository/manage_repository.mako --- a/templates/webapps/tool_shed/repository/manage_repository.mako +++ b/templates/webapps/tool_shed/repository/manage_repository.mako @@ -88,7 +88,7 @@ ${container_javascripts()} </%def> -${render_tool_shed_repository_actions( repository, repo=repo, metadata=metadata, changeset_revision=changeset_revision )} +${render_tool_shed_repository_actions( repository, metadata=metadata, changeset_revision=changeset_revision )} %if message: ${render_msg( message, status )} Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 14 May '13

14 May '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/88c1da16340b/ Changeset: 88c1da16340b User: Dave Bouvier Date: 2013-05-14 19:44:24 Summary: Expand the installation errors dict. Affected #: 1 file diff -r d3be8f8f8096ad723644019bd742904046df54ba -r 88c1da16340bf08dca453316fbcc7ff0868de57d test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -625,6 +625,17 @@ # repository, with tool and repository dependencies also selected for installation. result, _ = run_tests( test_config ) success = result.wasSuccessful() + repository_status = get_tool_test_results_from_api( galaxy_tool_shed_url, metadata_revision_id ) + if 'test_environment' not in repository_status: + repository_status[ 'test_environment' ] = {} + test_environment = get_test_environment( repository_status[ 'test_environment' ] ) + test_environment[ 'galaxy_database_version' ] = get_database_version( app ) + test_environment[ 'galaxy_revision'] = get_repository_current_revision( os.getcwd() ) + repository_status[ 'test_environment' ] = test_environment + repository_status[ 'passed_tests' ] = [] + repository_status[ 'failed_tests' ] = [] + repository_status[ 'installation_errors' ] = dict( current_repository=[], repository_dependencies=[], tool_dependencies=[] ) + repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) # If the installation succeeds, configure and run functional tests for this repository. This is equivalent to # sh run_functional_tests.sh -installed if success: @@ -684,18 +695,8 @@ # }, # ] # } - repository_status = get_tool_test_results_from_api( galaxy_tool_shed_url, metadata_revision_id ) - if 'test_environment' not in repository_status: - repository_status[ 'test_environment' ] = {} - test_environment = get_test_environment( repository_status[ 'test_environment' ] ) - test_environment[ 'galaxy_database_version' ] = get_database_version( app ) - test_environment[ 'galaxy_revision'] = get_repository_current_revision( os.getcwd() ) - repository_status[ 'test_environment' ] = test_environment - repository_status[ 'passed_tests' ] = [] - repository_status[ 'failed_tests' ] = [] - repository_status[ 'installation_errors' ] = [] - repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) - missing_tool_dependencies = repository.includes_tool_dependencies and repository.missing_tool_dependencies + failed_tool_dependencies = repository.includes_tool_dependencies and repository.tool_dependencies_with_installation_errors + failed_repository_dependencies = repository.repository_dependencies_with_installation_errors if 'missing_test_components' not in repository_status: repository_status[ 'missing_test_components' ] = [] if not has_test_data: @@ -708,9 +709,10 @@ tool_version = tool[ 'version' ] tool_guid = tool[ 'guid' ] # In keeping with the standard display layout, add the error message to the dict for each tool individually. - failed_tests = dict( tool_id=tool_id, tool_version=tool_version, tool_guid=tool_guid, - missing_components="Repository %s is missing a test-data directory." % name ) - repository_status[ 'missing_test_components' ].append( failed_tests ) + missing_components = dict( tool_id=tool_id, tool_version=tool_version, tool_guid=tool_guid, + missing_components="Repository %s is missing a test-data directory." % name ) + if missing_components not in repository_status[ 'missing_test_components' ]: + repository_status[ 'missing_test_components' ].append( missing_components ) # Record the status of this repository in the tool shed. register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=False ) # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the @@ -719,17 +721,26 @@ # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. test_toolbox.toolbox = app.toolbox repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) - elif missing_tool_dependencies: + elif failed_tool_dependencies or failed_repository_dependencies: # If a tool dependency fails to install correctly, this should be considered an installation error, # and functional tests should be skipped, since the tool dependency needs to be correctly installed # for the test to be considered reliable. log.error( 'One or more tool dependencies of this repository are marked as missing.' ) log.error( 'Updating repository and skipping functional tests.' ) # In keeping with the standard display layout, add the error message to the dict for each tool individually. - for dependency in repository.missing_tool_dependencies: - test_result = dict( name=name, owner=owner, changeset_revision=changeset_revision, + for dependency in repository.tool_dependencies_with_installation_errors: + test_result = dict( type=dependency.type, + name=dependency.name, + version=dependency.version, error_message=dependency.error_message ) - repository_status[ 'installation_errors' ].append( test_result ) + repository_status[ 'installation_errors' ][ 'tool_dependencies' ].append( test_result ) + for dependency in repository.repository_dependencies_with_installation_errors: + test_result = dict( tool_shed=dependency.tool_shed, + name=dependency.name, + owner=dependency.owner, + changeset_revision=dependency.changeset_revision, + error_message=dependency.error_message ) + repository_status[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) # Record the status of this repository in the tool shed. register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=False ) # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the @@ -838,6 +849,14 @@ else: # Even if the repository failed to install, execute the uninstall method, in case a dependency did succeed. log.debug( 'Uninstalling repository %s', repository_info_dict[ 'name' ] ) + repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) + test_result = dict( tool_shed=repository.tool_shed, + name=repository.name, + owner=repository.owner, + changeset_revision=repository.changeset_revision, + error_message=repository.error_message ) + repository_status[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) + register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=False ) success = execute_uninstall_method( app ) if not success: log.error( 'Repository %s failed to uninstall.', repository_info_dict[ 'name' ] ) https://bitbucket.org/galaxy/galaxy-central/commits/344658666b2e/ Changeset: 344658666b2e User: Dave Bouvier Date: 2013-05-14 19:45:17 Summary: Display the information in the installation errors dict on repository manage page under the tool test results folder. Affected #: 2 files diff -r 88c1da16340bf08dca453316fbcc7ff0868de57d -r 344658666b2ea9e0607b9532d8c9c0d1874f2c39 lib/galaxy/webapps/tool_shed/util/container_util.py --- a/lib/galaxy/webapps/tool_shed/util/container_util.py +++ b/lib/galaxy/webapps/tool_shed/util/container_util.py @@ -26,6 +26,10 @@ self.invalid_repository_dependencies = [] self.invalid_tool_dependencies = [] self.invalid_tools = [] + self.installation_errors = [] + self.current_repository_installation_errors = [] + self.repository_installation_errors = [] + self.tool_dependency_installation_errors = [] self.valid_tools = [] self.valid_data_managers = [] self.invalid_data_managers = [] @@ -195,6 +199,18 @@ return [ self.toolshed, self.repository_name, self.repository_owner, self.changeset_revision, asbool( str( self.prior_installation_required ) ) ] +class RepositoryInstallationError( object ): + """Repository installation error object""" + + def __init__( self, id=None, tool_shed=None, name=None, owner=None, changeset_revision=None, error_message=None ): + self.id = id + self.tool_shed = tool_shed + self.name = name + self.owner = owner + self.changeset_revision = changeset_revision + self.error_message = error_message + + class TestEnvironment( object ): """Tool test environment object""" @@ -250,6 +266,17 @@ return [ self.name, self.version, self.type ] +class ToolDependencyInstallationError( object ): + """Tool dependency installation error object""" + + def __init__( self, id=None, type=None, name=None, version=None, error_message=None ): + self.id = id + self.type = type + self.name = name + self.version = version + self.error_message = error_message + + class Workflow( object ): """Workflow object.""" @@ -1059,6 +1086,68 @@ tool_id=missing_test_components_dict.get( 'tool_id', '' ), tool_version=missing_test_components_dict.get( 'tool_version', '' ) ) folder.missing_test_components.append( missing_test_component ) + installation_error_dicts = tool_test_results_dict.get( 'installation_errors', {} ) + if installation_error_dicts: + current_repository_errors = installation_error_dicts.get( 'current_repository', [] ) + repository_dependency_errors = installation_error_dicts.get( 'repository_dependencies', [] ) + tool_dependency_errors = installation_error_dicts.get( 'tool_dependencies', [] ) + if current_repository_errors or repository_dependency_errors or tool_dependency_errors: + folder_id += 1 + installation_error_base_folder = Folder( id=folder_id, + key='installation_errors', + label='Installation errors', + parent=test_results_folder ) + if current_repository_errors: + folder_id += 1 + subfolder = Folder( id=folder_id, + key='current_repository_errors', + label='This repository', + parent=installation_error_base_folder ) + repository_error_id = 0 + for repository_error_dict in current_repository_errors: + repository_error_id += 1 + repository_installation_error = RepositoryInstallationError( id=repository_error_id, + tool_shed=repository_error_dict.get( 'tool_shed', '' ), + name=repository_error_dict.get( 'name', '' ), + owner=repository_error_dict.get( 'owner', '' ), + changeset_revision=repository_error_dict.get( 'changeset_revision', '' ), + error_message=repository_error_dict.get( 'error_message', '' ) ) + subfolder.current_repository_installation_errors.append( repository_installation_error ) + installation_error_base_folder.folders.append( subfolder ) + if repository_dependency_errors: + folder_id += 1 + subfolder = Folder( id=folder_id, + key='repository_dependency_errors', + label='Repository dependencies', + parent=installation_error_base_folder ) + repository_error_id = 0 + for repository_error_dict in repository_dependency_errors: + repository_error_id += 1 + repository_installation_error = RepositoryInstallationError( id=repository_error_id, + tool_shed=repository_error_dict.get( 'tool_shed', '' ), + name=repository_error_dict.get( 'name', '' ), + owner=repository_error_dict.get( 'owner', '' ), + changeset_revision=repository_error_dict.get( 'changeset_revision', '' ), + error_message=repository_error_dict.get( 'error_message', '' ) ) + subfolder.repository_installation_errors.append( repository_installation_error ) + installation_error_base_folder.folders.append( subfolder ) + if tool_dependency_errors: + folder_id += 1 + subfolder = Folder( id=folder_id, + key='tool_dependency_errors', + label='Tool dependencies', + parent=installation_error_base_folder ) + tool_dependency_error_id = 0 + for tool_dependency_error_dict in tool_dependency_errors: + tool_dependency_error_id += 1 + tool_dependency_installation_error = ToolDependencyInstallationError( id=tool_dependency_error_id, + type=tool_dependency_error_dict.get( 'type', '' ), + name=tool_dependency_error_dict.get( 'name', '' ), + version=tool_dependency_error_dict.get( 'version', '' ), + error_message=tool_dependency_error_dict.get( 'error_message', '' ) ) + subfolder.tool_dependency_installation_errors.append( tool_dependency_installation_error ) + installation_error_base_folder.folders.append( subfolder ) + test_results_folder.installation_errors.append( installation_error_base_folder ) else: tool_test_results_root_folder = None return folder_id, tool_test_results_root_folder diff -r 88c1da16340bf08dca453316fbcc7ff0868de57d -r 344658666b2ea9e0607b9532d8c9c0d1874f2c39 templates/webapps/tool_shed/repository/common.mako --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -220,10 +220,12 @@ elif folder.label in [ 'Installed repository dependencies', 'Repository dependencies', 'Missing repository dependencies' ]: if folder.description: folder_label = "%s<i> - %s</i>" % ( folder_label, folder.description ) - elif folder.label not in [ 'Installed repository dependencies' ]: + elif folder.label not in [ 'Installed repository dependencies' ] and folder.parent.label not in [ 'Installation errors' ]: folder_label = "%s<i> - installation of these additional repositories is required</i>" % folder_label if trans.webapp.name == 'galaxy': col_span_str = 'colspan="4"' + elif folder.label == 'Installation errors': + folder_label = "%s<i> - no functional tests were run for any tools in this changeset revision</i>" % folder_label elif folder.label == 'Invalid repository dependencies': folder_label = "%s<i> - click the repository dependency to see why it is invalid</i>" % folder_label elif folder.label == 'Invalid tool dependencies': @@ -345,6 +347,26 @@ ${render_missing_test_component( missing_test_component, pad, my_row, row_counter )} %endfor %endif + %if folder.installation_errors: + %for installation_error in folder.installation_errors: + ${render_folder( installation_error, pad, my_row, row_counter )} + %endfor + %endif + %if folder.tool_dependency_installation_errors: + %for tool_dependency_installation_error in folder.tool_dependency_installation_errors: + ${render_tool_dependency_installation_error( tool_dependency_installation_error, pad, my_row, row_counter )} + %endfor + %endif + %if folder.repository_installation_errors: + %for repository_installation_error in folder.repository_installation_errors: + ${render_repository_installation_error( repository_installation_error, pad, my_row, row_counter, is_current_repository=False )} + %endfor + %endif + %if folder.current_repository_installation_errors: + %for repository_installation_error in folder.current_repository_installation_errors: + ${render_repository_installation_error( repository_installation_error, pad, my_row, row_counter, is_current_repository=True )} + %endfor + %endif </%def><%def name="render_datatype( datatype, pad, parent, row_counter, row_is_header=False )"> @@ -608,6 +630,65 @@ %></%def> +<%def name="render_tool_dependency_installation_error( installation_error, pad, parent, row_counter, row_is_header=False )"> + <% encoded_id = trans.security.encode_id( installation_error.id ) %> + <tr class="datasetRow" + %if parent is not None: + parent="${parent}" + %endif + id="libraryItem-${encoded_id}"> + <td style="padding-left: ${pad+20}px;"> + <table class="grid" id="td_installation_errors"> + <tr bgcolor="#FFFFCC"> + <th>Type</th><th>Name</th><th>Version</th> + </tr> + <tr> + <td>${installation_error.name | h}</td> + <td>${installation_error.type | h}</td> + <td>${installation_error.version | h}</td> + </tr> + <tr><th>Error</th></tr> + <tr><td colspan="3">${installation_error.error_message | h}</td></tr> + </table> + </td> + </tr> + <% + my_row = row_counter.count + row_counter.increment() + %> +</%def> + +<%def name="render_repository_installation_error( installation_error, pad, parent, row_counter, row_is_header=False, is_current_repository=False )"> + <% encoded_id = trans.security.encode_id( installation_error.id ) %> + <tr class="datasetRow" + %if parent is not None: + parent="${parent}" + %endif + id="libraryItem-${encoded_id}"> + <td style="padding-left: ${pad+20}px;"> + <table class="grid" id="repository_installation_errors"> + %if not is_current_repository: + <tr bgcolor="#FFFFCC"> + <th>Tool shed</th><th>Name</th><th>Owner</th><th>Changeset revision</th> + </tr> + <tr> + <td>${installation_error.tool_shed | h}</td> + <td>${installation_error.name | h}</td> + <td>${installation_error.owner | h}</td> + <td>${installation_error.changeset_revision | h}</td> + </tr> + %endif + <tr><th>Error</th></tr> + <tr><td colspan="4">${installation_error.error_message | h}</td></tr> + </table> + </td> + </tr> + <% + my_row = row_counter.count + row_counter.increment() + %> +</%def> + <%def name="render_passed_test( passed_test, pad, parent, row_counter, row_is_header=False )"><% encoded_id = trans.security.encode_id( passed_test.id ) %><tr class="datasetRow" https://bitbucket.org/galaxy/galaxy-central/commits/1cabd9fd107c/ Changeset: 1cabd9fd107c User: Dave Bouvier Date: 2013-05-14 19:48:05 Summary: Merged. Affected #: 1 file diff -r 344658666b2ea9e0607b9532d8c9c0d1874f2c39 -r 1cabd9fd107c14caa379e591ff3fea66c3cc5ce4 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -9,6 +9,7 @@ from galaxy.tools.data_manager.manager import DataManager from galaxy.util import inflector from galaxy.util import json +from galaxy.web import url_for from galaxy.webapps.tool_shed.util import container_util import tool_shed.util.shed_util_common as suc from tool_shed.util import common_util @@ -1050,6 +1051,9 @@ is_valid = True error_message = '' toolshed = repository_elem.get( 'toolshed' ) + if not toolshed: + # Default to the current tool shed. + toolshed = str( url_for( '/', qualified=True ) ).rstrip( '/' ) name = repository_elem.get( 'name' ) owner = repository_elem.get( 'owner' ) changeset_revision = repository_elem.get( 'changeset_revision' ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Allow tool shed repository dependency definition tags to not include a toolshed attribute. If missing, the toolshed value will default to the current tool shed. The following example shows what was required before this changeset:
by commits-noreply＠bitbucket.org 14 May '13

14 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/842a78530fcd/ Changeset: 842a78530fcd User: greg Date: 2013-05-14 19:45:08 Summary: Allow tool shed repository dependency definition tags to not include a toolshed attribute. If missing, the toolshed value will default to the current tool shed. The following example shows what was required before this changeset: <repository toolshed="http://localhost:9009/" name="molecule_datatypes" owner="test" changeset_revision="1a070566e9c6" /> Now the following is also valid: <repository name="molecule_datatypes" owner="test" changeset_revision="1a070566e9c6" /> Affected #: 1 file diff -r d3be8f8f8096ad723644019bd742904046df54ba -r 842a78530fcd87670198d55d96d9adbdfc53483e lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -9,6 +9,7 @@ from galaxy.tools.data_manager.manager import DataManager from galaxy.util import inflector from galaxy.util import json +from galaxy.web import url_for from galaxy.webapps.tool_shed.util import container_util import tool_shed.util.shed_util_common as suc from tool_shed.util import common_util @@ -1050,6 +1051,9 @@ is_valid = True error_message = '' toolshed = repository_elem.get( 'toolshed' ) + if not toolshed: + # Default to the current tool shed. + toolshed = str( url_for( '/', qualified=True ) ).rstrip( '/' ) name = repository_elem.get( 'name' ) owner = repository_elem.get( 'owner' ) changeset_revision = repository_elem.get( 'changeset_revision' ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 14 May '13

14 May '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6fac39a6786e/ Changeset: 6fac39a6786e Branch: stable User: dan Date: 2013-05-14 19:13:21 Summary: Have datatypes.checkers.check_binary call util.is_binary. Fixes unicode file upload under certain conditions. Affected #: 1 file diff -r 7cf2f256b3d9c32f76567f49a0935b0ca9da9f7e -r 6fac39a6786e555c5bd57f98b4601c9949aae37d lib/galaxy/datatypes/checkers.py --- a/lib/galaxy/datatypes/checkers.py +++ b/lib/galaxy/datatypes/checkers.py @@ -58,7 +58,7 @@ for chars in temp: for char in chars: chars_read += 1 - if ord( char ) > 128: + if util.is_binary( char ): is_binary = True break if chars_read > 100: https://bitbucket.org/galaxy/galaxy-central/commits/686170571216/ Changeset: 686170571216 Branch: stable User: dan Date: 2013-05-14 19:13:21 Summary: Use util.unicodify when generating dataset peeks. Affected #: 1 file diff -r 6fac39a6786e555c5bd57f98b4601c9949aae37d -r 6861705712161c545398e6db1a7cdadda931fc7a lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -860,7 +860,7 @@ text = "%s file" % file_type else: try: - text = unicode( '\n'.join( lines ), 'utf-8' ) + text = util.unicodify( '\n'.join( lines ) ) except UnicodeDecodeError: text = "binary/unknown file" return text Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 14 May '13

14 May '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3667a65b86a9/ Changeset: 3667a65b86a9 User: dan Date: 2013-05-14 19:13:21 Summary: Have datatypes.checkers.check_binary call util.is_binary. Fixes unicode file upload under certain conditions. Affected #: 1 file diff -r 8882e45504a37ab9f87f2872df2a6749673bd741 -r 3667a65b86a9418bc67c89ce0fdfd0dfc55a0021 lib/galaxy/datatypes/checkers.py --- a/lib/galaxy/datatypes/checkers.py +++ b/lib/galaxy/datatypes/checkers.py @@ -58,7 +58,7 @@ for chars in temp: for char in chars: chars_read += 1 - if ord( char ) > 128: + if util.is_binary( char ): is_binary = True break if chars_read > 100: https://bitbucket.org/galaxy/galaxy-central/commits/d3be8f8f8096/ Changeset: d3be8f8f8096 User: dan Date: 2013-05-14 19:13:21 Summary: Use util.unicodify when generating dataset peeks. Affected #: 1 file diff -r 3667a65b86a9418bc67c89ce0fdfd0dfc55a0021 -r d3be8f8f8096ad723644019bd742904046df54ba lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -865,7 +865,7 @@ text = "%s file" % file_type else: try: - text = unicode( '\n'.join( lines ), 'utf-8' ) + text = util.unicodify( '\n'.join( lines ) ) except UnicodeDecodeError: text = "binary/unknown file" return text Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Add annotation when importing workflow.
by commits-noreply＠bitbucket.org 14 May '13

14 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8882e45504a3/ Changeset: 8882e45504a3 User: jgoecks Date: 2013-05-14 17:54:00 Summary: Add annotation when importing workflow. Affected #: 1 file diff -r c842439a245cf97137a819aaae9907240f47e6d9 -r 8882e45504a37ab9f87f2872df2a6749673bd741 lib/galaxy/webapps/galaxy/controllers/workflow.py --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -1855,14 +1855,19 @@ conn.output_name = conn_dict['output_name'] conn.output_step = steps_by_external_id[ conn_dict['id'] ] del step.temp_input_connections + # Order the steps if possible attach_ordered_steps( workflow, steps ) + # Connect up stored = model.StoredWorkflow() stored.name = workflow.name workflow.stored_workflow = stored stored.latest_workflow = workflow stored.user = trans.user + if data[ 'annotation' ]: + self.add_item_annotation( trans.sa_session, stored.user, stored, data[ 'annotation' ] ) + # Persist trans.sa_session.add( stored ) trans.sa_session.flush() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Fix for manage_data_manager.mako.
by commits-noreply＠bitbucket.org 13 May '13

13 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c842439a245c/ Changeset: c842439a245c User: dan Date: 2013-05-13 21:56:34 Summary: Fix for manage_data_manager.mako. Affected #: 1 file diff -r b8af8fa7ab2e61700875249c62308302a105bb38 -r c842439a245cf97137a819aaae9907240f47e6d9 templates/webapps/galaxy/data_manager/manage_data_manager.mako --- a/templates/webapps/galaxy/data_manager/manage_data_manager.mako +++ b/templates/webapps/galaxy/data_manager/manage_data_manager.mako @@ -15,7 +15,7 @@ <p>Access managed data by job</p> %if jobs: -<form name="jobs" action="${h.url_for()}" method="POST"> +<div><table class="manage-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"><tr class="header"><td>Job ID</td> @@ -42,7 +42,7 @@ %endfor </table><p/> -</form> +</div> %else: <div class="infomessage">There are no jobs for this data manager.</div> %endif Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0