1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a3e1fb22f5b5/ Changeset: a3e1fb22f5b5 User: greg Date: 2013-05-15 20:03:43 Summary: Add the ability for a tool shed repository owner to set a flag to skip testing a specified repository revision. Add a new test_install_error column to the repository_metadata table, and add additional filtered list of the latest revision of repositories using these new filters. Affected #: 9 files diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -54,11 +54,25 @@ includes_tools = kwd.get( 'includes_tools', None ) if includes_tools is not None: clause_list.append( trans.model.RepositoryMetadata.table.c.includes_tools == util.string_as_bool( includes_tools ) ) + # Filter by skip_tool_test if received. + skip_tool_test = kwd.get( 'skip_tool_test', None ) + if skip_tool_test is not None: + skip_tool_test = util.string_as_bool( skip_tool_test ) + # Generate and execute the query. try: - query = trans.sa_session.query( trans.app.model.RepositoryMetadata ) \ - .filter( and_( *clause_list ) ) \ - .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id ) \ - .all() + if skip_tool_test: + # The skip_tool_test filter was received as True. + query = trans.sa_session.query( trans.app.model.RepositoryMetadata ) \ + .join( trans.model.SkipToolTest ) \ + .filter( and_( *clause_list ) ) \ + .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id ) \ + .all() + else: + # The skip_tool_test filter was not received or it was received as False. + query = trans.sa_session.query( trans.app.model.RepositoryMetadata ) \ + .filter( and_( *clause_list ) ) \ + .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id ) \ + .all() for repository_metadata in query: repository_metadata_dict = repository_metadata.get_api_value( view='collection', value_mapper=default_value_mapper( trans, repository_metadata ) ) diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -64,6 +64,8 @@ my_writable_repositories_with_failing_tool_tests_grid = repository_grids.MyWritableRepositoriesWithFailingToolTestsGrid() my_writable_repositories_with_invalid_tools_grid = repository_grids.MyWritableRepositoriesWithInvalidToolsGrid() my_writable_repositories_with_no_failing_tool_tests_grid = repository_grids.MyWritableRepositoriesWithNoFailingToolTestsGrid() + my_writable_repositories_with_skip_tests_checked_grid = repository_grids.MyWritableRepositoriesWithSkipTestsCheckedGrid() + my_writable_repositories_with_test_install_errors_grid = repository_grids.MyWritableRepositoriesWithTestInstallErrorsGrid() repositories_by_user_grid = repository_grids.RepositoriesByUserGrid() repositories_i_own_grid = repository_grids.RepositoriesIOwnGrid() repositories_in_category_grid = repository_grids.RepositoriesInCategoryGrid() @@ -71,6 +73,8 @@ repositories_with_failing_tool_tests_grid = repository_grids.RepositoriesWithFailingToolTestsGrid() repositories_with_invalid_tools_grid = repository_grids.RepositoriesWithInvalidToolsGrid() repositories_with_no_failing_tool_tests_grid = repository_grids.RepositoriesWithNoFailingToolTestsGrid() + repositories_with_skip_tests_checked_grid = repository_grids.RepositoriesWithSkipTestsCheckedGrid() + repositories_with_test_install_errors_grid = repository_grids.RepositoriesWithTestInstallErrorsGrid() repository_dependencies_grid = repository_grids.RepositoryDependenciesGrid() repository_grid = repository_grids.RepositoryGrid() # The repository_metadata_grid is not currently displayed, but is sub-classed by several grids. @@ -222,6 +226,7 @@ message += '<li>you are authorized to update them</li>' message += '<li>the latest installable revision contains at least 1 tool</li>' message += '<li>the latest installable revision is not missing any tool test components</li>' + message += '<li>the latest installable revision has no installation errors</li>' message += '<li>the latest installable revision has at least 1 tool test that fails</li>' message += '</ul>' kwd[ 'message' ] = message @@ -286,6 +291,63 @@ return self.my_writable_repositories_with_no_failing_tool_tests_grid( trans, **kwd ) @web.expose + def browse_my_writable_repositories_with_install_errors( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd[ 'operation' ].lower() + if operation == "view_or_manage_repository": + return trans.response.send_redirect( web.url_for( controller='repository', + action='view_or_manage_repository', + **kwd ) ) + elif operation == "repositories_by_user": + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories_by_user', + **kwd ) ) + elif operation in [ 'mark as deprecated', 'mark as not deprecated' ]: + kwd[ 'mark_deprecated' ] = operation == 'mark as deprecated' + return trans.response.send_redirect( web.url_for( controller='repository', + action='deprecate', + **kwd ) ) + if 'message' not in kwd: + message = 'This list contains repositories that match the following criteria:<br>' + message += '<ul>' + message += '<li>you are authorized to update them</li>' + message += '<li>the latest installable revision contains at least 1 tool</li>' + message += '<li>the latest installable revision is not missing any tool test components</li>' + message += '<li>the latest installable revision has installation errors (the repository itself, repository dependencies or tool dependencies)</li>' + message += '</ul>' + kwd[ 'message' ] = message + kwd[ 'status' ] = 'warning' + return self.my_writable_repositories_with_test_install_errors_grid( trans, **kwd ) + + @web.expose + def browse_my_writable_repositories_with_skip_tool_test_checked( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd[ 'operation' ].lower() + if operation == "view_or_manage_repository": + return trans.response.send_redirect( web.url_for( controller='repository', + action='view_or_manage_repository', + **kwd ) ) + elif operation == "repositories_by_user": + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories_by_user', + **kwd ) ) + elif operation in [ 'mark as deprecated', 'mark as not deprecated' ]: + kwd[ 'mark_deprecated' ] = operation == 'mark as deprecated' + return trans.response.send_redirect( web.url_for( controller='repository', + action='deprecate', + **kwd ) ) + if 'message' not in kwd: + message = 'This list contains repositories that match the following criteria:<br>' + message += '<ul>' + message += '<li>you are authorized to update them</li>' + message += '<li>the latest installable revision contains at least 1 tool</li>' + message += '<li>the latest installable revision has <b>Skip tool tests</b> checked</li>' + message += '</ul>' + kwd[ 'message' ] = message + kwd[ 'status' ] = 'warning' + return self.my_writable_repositories_with_skip_tests_checked_grid( trans, **kwd ) + + @web.expose def browse_repositories( self, trans, **kwd ): # We add params to the keyword dict in this method in order to rename the param with an "f-" prefix, simulating filtering by clicking a search # link. We have to take this approach because the "-" character is illegal in HTTP requests. @@ -484,6 +546,34 @@ return self.repositories_with_failing_tool_tests_grid( trans, **kwd ) @web.expose + def browse_repositories_with_install_errors( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd[ 'operation' ].lower() + if operation == "view_or_manage_repository": + return trans.response.send_redirect( web.url_for( controller='repository', + action='view_or_manage_repository', + **kwd ) ) + elif operation == "repositories_by_user": + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories_by_user', + **kwd ) ) + elif operation in [ 'mark as deprecated', 'mark as not deprecated' ]: + kwd[ 'mark_deprecated' ] = operation == 'mark as deprecated' + return trans.response.send_redirect( web.url_for( controller='repository', + action='deprecate', + **kwd ) ) + if 'message' not in kwd: + message = 'This list contains repositories that match the following criteria:<br>' + message += '<ul>' + message += '<li>the latest installable revision contains at least 1 tool</li>' + message += '<li>the latest installable revision is not missing any tool test components</li>' + message += '<li>the latest installable revision has installation errors (the repository itself, repository dependencies or tool dependencies)</li>' + message += '</ul>' + kwd[ 'message' ] = message + kwd[ 'status' ] = 'warning' + return self.repositories_with_test_install_errors_grid( trans, **kwd ) + + @web.expose def browse_repositories_with_invalid_tools( self, trans, **kwd ): if 'operation' in kwd: operation = kwd[ 'operation' ].lower() @@ -539,6 +629,33 @@ return self.repositories_with_no_failing_tool_tests_grid( trans, **kwd ) @web.expose + def browse_repositories_with_skip_tool_test_checked( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd[ 'operation' ].lower() + if operation == "view_or_manage_repository": + return trans.response.send_redirect( web.url_for( controller='repository', + action='view_or_manage_repository', + **kwd ) ) + elif operation == "repositories_by_user": + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories_by_user', + **kwd ) ) + elif operation in [ 'mark as deprecated', 'mark as not deprecated' ]: + kwd[ 'mark_deprecated' ] = operation == 'mark as deprecated' + return trans.response.send_redirect( web.url_for( controller='repository', + action='deprecate', + **kwd ) ) + if 'message' not in kwd: + message = 'This list contains repositories that match the following criteria:<br>' + message += '<ul>' + message += '<li>the latest installable revision contains at least 1 tool</li>' + message += '<li>the latest installable revision has <b>Skip tool tests</b> checked</li>' + message += '</ul>' + kwd[ 'message' ] = message + kwd[ 'status' ] = 'warning' + return self.repositories_with_skip_tests_checked_grid( trans, **kwd ) + + @web.expose def browse_repository( self, trans, id, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/galaxy/webapps/tool_shed/model/__init__.py --- a/lib/galaxy/webapps/tool_shed/model/__init__.py +++ b/lib/galaxy/webapps/tool_shed/model/__init__.py @@ -219,13 +219,14 @@ class RepositoryMetadata( object, APIItem ): api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'has_repository_dependencies', 'includes_datatypes', 'includes_tools', 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) - api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'tools_functionally_correct', - 'do_not_test', 'time_last_tested', 'tool_test_results', 'has_repository_dependencies', 'includes_datatypes', 'includes_tools', - 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) + api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'tools_functionally_correct', 'do_not_test', + 'test_install_error', 'time_last_tested', 'tool_test_results', 'has_repository_dependencies', 'includes_datatypes', + 'includes_tools', 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) def __init__( self, id=None, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False, downloadable=False, - missing_test_components=None, tools_functionally_correct=False, do_not_test=False, time_last_tested=None, tool_test_results=None, - has_repository_dependencies=False, includes_datatypes=False, includes_tools=False, includes_tool_dependencies=False, includes_workflows=False ): + missing_test_components=None, tools_functionally_correct=False, do_not_test=False, test_install_error=False, time_last_tested=None, + tool_test_results=None, has_repository_dependencies=False, includes_datatypes=False, includes_tools=False, includes_tool_dependencies=False, + includes_workflows=False ): self.id = id self.repository_id = repository_id self.changeset_revision = changeset_revision @@ -236,6 +237,7 @@ self.missing_test_components = missing_test_components self.tools_functionally_correct = tools_functionally_correct self.do_not_test = do_not_test + self.test_install_error = test_install_error self.time_last_tested = time_last_tested self.tool_test_results = tool_test_results self.has_repository_dependencies = has_repository_dependencies @@ -274,6 +276,37 @@ return rval +class SkipToolTest( object, APIItem ): + api_collection_visible_keys = ( 'id', 'repository_metadata_id', 'initial_changeset_revision' ) + api_element_visible_keys = ( 'id', 'repository_metadata_id', 'initial_changeset_revision', 'comment' ) + + def __init__( self, id=None, repository_metadata_id=None, initial_changeset_revision=None, comment=None ): + self.id = id + self.repository_metadata_id = repository_metadata_id + self.initial_changeset_revision = initial_changeset_revision + self.comment = comment + + def as_dict( self, value_mapper=None ): + return self.get_api_value( view='element', value_mapper=value_mapper ) + + def get_api_value( self, view='collection', value_mapper=None ): + if value_mapper is None: + value_mapper = {} + rval = {} + try: + visible_keys = self.__getattribute__( 'api_' + view + '_visible_keys' ) + except AttributeError: + raise Exception( 'Unknown API view: %s' % view ) + for key in visible_keys: + try: + rval[ key ] = self.__getattribute__( key ) + if key in value_mapper: + rval[ key ] = value_mapper.get( key, rval[ key ] ) + except AttributeError: + rval[ key ] = None + return rval + + class RepositoryReview( object, APIItem ): api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating', 'deleted' ) api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating', 'deleted' ) diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/galaxy/webapps/tool_shed/model/mapping.py --- a/lib/galaxy/webapps/tool_shed/model/mapping.py +++ b/lib/galaxy/webapps/tool_shed/model/mapping.py @@ -132,6 +132,7 @@ Column( "downloadable", Boolean, default=True ), Column( "tools_functionally_correct", Boolean, default=False, index=True ), Column( "do_not_test", Boolean, default=False, index=True ), + Column( "test_install_error", Boolean, default=False, index=True ), Column( "time_last_tested", DateTime, default=None, nullable=True ), Column( "missing_test_components", Boolean, default=False, index=True ), Column( "tool_test_results", JSONType, nullable=True ), @@ -141,6 +142,14 @@ Column( "includes_tool_dependencies", Boolean, default=False, index=True ), Column( "includes_workflows", Boolean, default=False, index=True ) ) +SkipToolTest.table = Table( "skip_tool_test", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "repository_metadata_id", Integer, ForeignKey( "repository_metadata.id" ), index=True ), + Column( "initial_changeset_revision", TrimmedString( 255 ), index=True ), + Column( "comment" , TEXT ) ) + RepositoryReview.table = Table( "repository_review", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -267,10 +276,14 @@ foreign_keys=[ RepositoryMetadata.table.c.repository_id, RepositoryMetadata.table.c.changeset_revision ], primaryjoin=( ( RepositoryMetadata.table.c.repository_id == RepositoryReview.table.c.repository_id ) & ( RepositoryMetadata.table.c.changeset_revision == RepositoryReview.table.c.changeset_revision ) ) ) ) ) +mapper( SkipToolTest, SkipToolTest.table, + properties=dict( repository_revision=relation( RepositoryMetadata, + backref='skip_tool_tests' ) ) ) + mapper( RepositoryReview, RepositoryReview.table, properties=dict( repository=relation( Repository, primaryjoin=( RepositoryReview.table.c.repository_id == Repository.table.c.id ) ), - # Take case when using the mapper below! It should be used only when a new review is being created for a repository change set revision. + # Take care when using the mapper below! It should be used only when a new review is being created for a repository change set revision. # Keep in mind that repository_metadata records can be removed from the database for certain change set revisions when metadata is being # reset on a repository! repository_metadata=relation( RepositoryMetadata, diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/galaxy/webapps/tool_shed/model/migrate/versions/0019_add_skip_tool_test_table_and_test_install_error_column.py --- /dev/null +++ b/lib/galaxy/webapps/tool_shed/model/migrate/versions/0019_add_skip_tool_test_table_and_test_install_error_column.py @@ -0,0 +1,82 @@ +""" +Migration script to add the skip_tool_test table and add the test_install_error column to the repository_metadata table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * +import datetime + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +now = datetime.datetime.utcnow + +metadata = MetaData() + +SkipToolTest_table = Table( "skip_tool_test", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "repository_metadata_id", Integer, ForeignKey( "repository_metadata.id" ), index=True ), + Column( "initial_changeset_revision", TrimmedString( 255 ), index=True ), + Column( "comment" , TEXT ) ) + +def upgrade( migrate_engine ): + print __doc__ + metadata.bind = migrate_engine + metadata.reflect() + # Initialize. + if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + default_false = "0" + elif migrate_engine.name in [ 'postgresql', 'postgres' ]: + default_false = "false" + + try: + RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True ) + except NoSuchTableError: + RepositoryMetadata_table = None + log.debug( "Failed loading table repository_metadata." ) + + if RepositoryMetadata_table is not None: + # Create the test_install_error column. + c = Column( "test_install_error", Boolean, default=False, index=True ) + try: + c.create( RepositoryMetadata_table, index_name="ix_repository_metadata_ttie") + assert c is RepositoryMetadata_table.c.test_install_error + migrate_engine.execute( "UPDATE repository_metadata SET test_install_error=%s" % default_false ) + except Exception, e: + print "Adding test_install_error column to the repository_metadata table failed: %s" % str( e ) + + # Create skip_tool_test table. + try: + SkipToolTest_table.create() + except Exception, e: + print "Creating the skip_tool_test table failed: %s" % str( e ) + +def downgrade( migrate_engine ): + metadata.bind = migrate_engine + metadata.reflect() + + # Drop the skip_tool_test table. + try: + SkipToolTest_table.drop() + except Exception, e: + print "Dropping the skip_tool_test table failed: %s" % str( e ) + + # Drop test_install_error column from the repository_metadata table. + RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True ) + try: + RepositoryMetadata_table.c.test_install_error.drop() + except Exception, e: + print "Dropping column test_install_error from the repository_metadata table failed: %s" % str( e ) diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/tool_shed/grids/repository_grids.py --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -604,6 +604,180 @@ .filter( model.Repository.table.c.id < 0 ) +class RepositoriesWithTestInstallErrorsGrid( RepositoryGrid ): + # This grid displays only the latest installable revision of each repository. + title = "Repositories with tool test installation errors" + columns = [ + RepositoryGrid.NameColumn( "Name", + key="name", + link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), + attach_popup=False ), + RepositoryGrid.LatestInstallableRevisionColumn( "Latest Installable Revision" ), + RepositoryGrid.UserColumn( "Owner", + key="User.username", + model_class=model.User, + link=( lambda item: dict( operation="repositories_by_user", id=item.id ) ), + attach_popup=False ) + ] + columns.append( grids.MulticolFilterColumn( "Search repository name", + cols_to_filter=[ columns[0] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + operations = [] + use_paging = False + + def build_initial_query( self, trans, **kwd ): + # Filter by latest installable revisions that contain tools with missing tool test components. + revision_clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ): + changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ) + if changeset_revision: + revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) + if revision_clause_list: + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ) \ + .join( model.RepositoryMetadata ) \ + .filter( or_( *revision_clause_list ) ) \ + .join( model.User.table ) + # Return an empty query. + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.table.c.id < 0 ) + + +class MyWritableRepositoriesWithTestInstallErrorsGrid( RepositoriesWithTestInstallErrorsGrid ): + # This grid displays only the latest installable revision of each repository. + title = "Repositories I can change with tool test installation errors" + columns = [ col for col in RepositoriesWithTestInstallErrorsGrid.columns ] + operations = [] + use_paging = False + + def build_initial_query( self, trans, **kwd ): + # First get all repositories that the current user is authorized to update. + username = trans.user.username + user_clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ): + allow_push = repository.allow_push( trans.app ) + if allow_push: + allow_push_usernames = allow_push.split( ',' ) + if username in allow_push_usernames: + user_clause_list.append( model.Repository.table.c.id == repository.id ) + if user_clause_list: + # We have the list of repositories that the current user is authorized to update, so filter further by latest installable revisions that contain + # tools with missing tool test components. + revision_clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ) \ + .filter( or_( *user_clause_list ) ): + changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ) + if changeset_revision: + revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) + if revision_clause_list: + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ) \ + .join( model.User.table ) \ + .filter( or_( *user_clause_list ) ) \ + .join( model.RepositoryMetadata ) \ + .filter( or_( *revision_clause_list ) ) + # Return an empty query. + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.table.c.id < 0 ) + + +class RepositoriesWithSkipTestsCheckedGrid( RepositoryGrid ): + # This grid displays only the latest installable revision of each repository. + title = "Repositories with skip tool tests checked" + columns = [ + RepositoryGrid.NameColumn( "Name", + key="name", + link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), + attach_popup=False ), + RepositoryGrid.LatestInstallableRevisionColumn( "Latest Installable Revision" ), + RepositoryGrid.UserColumn( "Owner", + key="User.username", + model_class=model.User, + link=( lambda item: dict( operation="repositories_by_user", id=item.id ) ), + attach_popup=False ) + ] + columns.append( grids.MulticolFilterColumn( "Search repository name", + cols_to_filter=[ columns[0] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + operations = [] + use_paging = False + + def build_initial_query( self, trans, **kwd ): + # Filter by latest installable revisions that contain tools with missing tool test components. + revision_clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ): + changeset_revision = filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ) + if changeset_revision: + revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) + if revision_clause_list: + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ) \ + .join( model.RepositoryMetadata ) \ + .filter( or_( *revision_clause_list ) ) \ + .join( model.User.table ) + # Return an empty query. + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.table.c.id < 0 ) + + +class MyWritableRepositoriesWithSkipTestsCheckedGrid( RepositoriesWithSkipTestsCheckedGrid ): + # This grid displays only the latest installable revision of each repository. + title = "Repositories I can change with skip tool tests checked" + columns = [ col for col in RepositoriesWithSkipTestsCheckedGrid.columns ] + operations = [] + use_paging = False + + def build_initial_query( self, trans, **kwd ): + # First get all repositories that the current user is authorized to update. + username = trans.user.username + user_clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ): + allow_push = repository.allow_push( trans.app ) + if allow_push: + allow_push_usernames = allow_push.split( ',' ) + if username in allow_push_usernames: + user_clause_list.append( model.Repository.table.c.id == repository.id ) + if user_clause_list: + # We have the list of repositories that the current user is authorized to update, so filter further by latest installable revisions that contain + # tools with missing tool test components. + revision_clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ) \ + .filter( or_( *user_clause_list ) ): + changeset_revision = filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ) + if changeset_revision: + revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) + if revision_clause_list: + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ) \ + .join( model.User.table ) \ + .filter( or_( *user_clause_list ) ) \ + .join( model.RepositoryMetadata ) \ + .filter( or_( *revision_clause_list ) ) + # Return an empty query. + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.table.c.id < 0 ) + + class DeprecatedRepositoriesIOwnGrid( RepositoriesIOwnGrid ): title = "Deprecated repositories I own" columns = [ @@ -1475,7 +1649,10 @@ repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) repository_metadata = get_latest_installable_repository_metadata_if_it_includes_tools( trans, repository ) - if repository_metadata and not repository_metadata.missing_test_components and not repository_metadata.tools_functionally_correct: + if repository_metadata \ + and not repository_metadata.missing_test_components \ + and not repository_metadata.tools_functionally_correct \ + and not repository_metadata.test_install_error: return repository_metadata.changeset_revision return None @@ -1516,6 +1693,32 @@ return repository_metadata.changeset_revision return None +def filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ): + """ + Inspect the latest installable changeset revision for the received repository to see if it has tool test installation errors. + """ + encoded_repository_id = trans.security.encode_id( repository.id ) + repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) + tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) + repository_metadata = get_latest_installable_repository_metadata_if_it_includes_tools( trans, repository ) + if repository_metadata \ + and repository_metadata.test_install_error \ + and not repository_metadata.missing_test_components: + return repository_metadata.changeset_revision + return None + +def filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ): + """ + Inspect the latest installable changeset revision for the received repository to see if skip tests is checked. + """ + encoded_repository_id = trans.security.encode_id( repository.id ) + repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) + tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) + repository_metadata = get_latest_installable_repository_metadata_if_it_includes_tools( trans, repository ) + if repository_metadata and repository_metadata.skip_tool_tests: + return repository_metadata.changeset_revision + return None + def get_latest_repository_metadata_if_it_includes_invalid_tools( trans, repository ): """Return the latest repository_metadata record for the received repository that contains invalid tools if one exists.""" encoded_repository_id = trans.security.encode_id( repository.id ) diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -259,6 +259,8 @@ downloadable = has_repository_dependencies or includes_datatypes or includes_tools or includes_tool_dependencies or includes_workflows repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) if repository_metadata: + # A repository metadata record already exists with the received changeset_revision, so we don't need to check the skip_tool_test table. + check_skip_tool_test = False repository_metadata.metadata = metadata_dict repository_metadata.downloadable = downloadable repository_metadata.has_repository_dependencies = has_repository_dependencies @@ -267,6 +269,8 @@ repository_metadata.includes_tool_dependencies = includes_tool_dependencies repository_metadata.includes_workflows = includes_workflows else: + # No repository_metadata record exists for the received changeset_revision, so we may need to update the skip_tool_test table. + check_skip_tool_test = True repository_metadata = trans.model.RepositoryMetadata( repository_id=repository.id, changeset_revision=changeset_revision, metadata=metadata_dict, @@ -279,11 +283,38 @@ # Always set the default values for the following columns. When resetting all metadata on a repository, this will reset the values. repository_metadata.tools_functionally_correct = False repository_metadata.missing_test_components = False + repository_metadata.test_install_error = False repository_metadata.do_not_test = False repository_metadata.time_last_tested = None repository_metadata.tool_test_results = None trans.sa_session.add( repository_metadata ) trans.sa_session.flush() + if check_skip_tool_test: + # Since we created a new repository_metadata record, we may need to update the skip_tool_test table to point to it. Inspect each + # changeset revision in the received repository's changelog (up to the received changeset revision) to see if it is contained in the + # skip_tool_test table. If it is, but is not associated with a repository_metadata record, reset that skip_tool_test record to the + # newly created repository_metadata record. + repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) + for changeset in repo.changelog: + changeset_hash = str( repo.changectx( changeset ) ) + skip_tool_test = suc.get_skip_tool_test_by_changeset_revision( trans, changeset_hash ) + if skip_tool_test: + # We found a skip_tool_test record associated with the changeset_revision, so see if it has a valid repository_revision. + try: + repository_revision = skip_tool_test.repository_revision + # The skip_tool_test record is associated with a valid repository_metadata record, so proceed. + continue + except: + # We found a skip_tool_test record that is associated with an invalid repository_metadata record, so update it to point to + # the newly created repository_metadata record. In some special cases there may be multiple skip_tool_test records that + # require updating, so we won't break here, we'll continue to inspect the rest of the changelog up to the received + # changeset_revision. + skip_tool_test.repository_revision = repository_metadata + trans.sa_session.add( skip_tool_test ) + trans.sa_session.flush() + if changeset_hash == changeset_revision: + # Proceed no further than the received changeset_revision. + break return repository_metadata def generate_data_manager_metadata( app, repository, repo_dir, data_manager_config_filename, metadata_dict, shed_config_dict=None ): diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -793,6 +793,13 @@ if shed_tool_conf == file_name: return index, shed_tool_conf_dict +def get_skip_tool_test_by_changeset_revision( trans, changeset_revision ): + """Return a skip_tool_test record whose initial_changeset_revision is the received changeset_revision.""" + # There should only be one, but we'll use first() so callers won't have to handle exceptions. + return trans.sa_session.query( trans.model.SkipToolTest ) \ + .filter( trans.model.SkipToolTest.table.c.initial_changeset_revision == changeset_revision ) \ + .first() + def get_tool_panel_config_tool_path_install_dir( app, repository ): """ Return shed-related tool panel config, the tool_path configured in it, and the relative path to the directory where the repository is installed. diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 templates/webapps/tool_shed/index.mako --- a/templates/webapps/tool_shed/index.mako +++ b/templates/webapps/tool_shed/index.mako @@ -116,9 +116,15 @@ <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_missing_tool_test_components' )}">Latest revision: missing tool tests</a></div><div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_install_errors' )}">Latest revision: installation errors</a> + </div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_failing_tool_tests' )}">Latest revision: failing tool tests</a></div><div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_skip_tool_test_checked' )}">Latest revision: skip tool tests</a> + </div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_no_failing_tool_tests' )}">Latest revision: all tool tests pass</a></div><div class="toolTitle"> @@ -167,9 +173,15 @@ <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories_missing_tool_test_components' )}">Latest revision: missing tool tests</a></div><div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories_with_install_errors' )}">Latest revision: installation errors</a> + </div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories_with_failing_tool_tests' )}">Latest revision: failing tool tests</a></div><div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories_with_skip_tool_test_checked' )}">Latest revision: skip tool tests</a> + </div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories_with_no_failing_tool_tests' )}">Latest revision: all tool tests pass</a></div><div class="toolTitle"> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.