1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e7b2e3c62632/ Changeset: e7b2e3c62632 User: greg Date: 2014-04-02 21:18:40 Summary: Add a new RepositoryGridFilterManager to the Tool Shed framework to support filtering the repositories displayed in the many Tool Shed grids. The current filters are CERTIFIED_LEVEL_ONE, CERTIFIED_LEVEL_ONE_SUITE and CERTIFIED_LEVEL_TWO, CERTIFIED_LEVEL_TWO_SUITE. The "level one" filters display repositories or suites that have been certified by the Tool Shed's install and test framework. The "level two" filters are not yet supported (the query enhancements are trivial, but not yet done) since no repositories in the test or main Tool Shed have been certified at this level (this level requires reviews and approvals by IUC members). When set, filters are stored in session cookies. Filters are easy to set. The Javascript event handler (e.g. checking a filter checkbox) would make a call something like: trans.app.repository_grid_filter_manager.set_filter( trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE ). Affected #: 7 files diff -r 55c0a99ff035b9d11adef3a931312c64bb99a45b -r e7b2e3c626322294668c5aab56c830d5a2eb5139 lib/galaxy/webapps/tool_shed/app.py --- a/lib/galaxy/webapps/tool_shed/app.py +++ b/lib/galaxy/webapps/tool_shed/app.py @@ -1,4 +1,5 @@ -import sys, config +import config +import sys from galaxy import tools import galaxy.tools.data import galaxy.quota @@ -7,52 +8,61 @@ from galaxy.openid.providers import OpenIDProviders from galaxy.web import security from galaxy.tags.tag_handler import CommunityTagHandler +from tool_shed.grids.util import RepositoryGridFilterManager import tool_shed.repository_types.registry + class UniverseApplication( object ): """Encapsulates the state of a Universe application""" - def __init__( self, **kwargs ): + + def __init__( self, **kwd ): print >> sys.stderr, "python path is: " + ", ".join( sys.path ) self.name = "tool_shed" - # Read config file and check for errors - self.config = config.Configuration( **kwargs ) + # Read the tool_shed_wsgi.ini configuration file and check for errors. + self.config = config.Configuration( **kwd ) self.config.check() config.configure_logging( self.config ) - # Set up datatypes registry + # Initialize the Galaxy datatypes registry. self.datatypes_registry = galaxy.datatypes.registry.Registry() self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config ) - # Set up the repository_types registry. + # Initialize the Tool Shed repository_types registry. self.repository_types_registry = tool_shed.repository_types.registry.Registry() - # Determine the database url + # Initialize the RepositoryGridFilterManager. + self.repository_grid_filter_manager = RepositoryGridFilterManager() + # Determine the Tool Shed database connection string. if self.config.database_connection: db_url = self.config.database_connection else: db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database - # Initialize database / check for appropriate schema version + # Initialize the Tool Shed database and check for appropriate schema version. from galaxy.webapps.tool_shed.model.migrate.check import create_or_verify_database create_or_verify_database( db_url, self.config.database_engine_options ) - # Setup the database engine and ORM + # Set up the Tool Shed database engine and ORM. from galaxy.webapps.tool_shed.model import mapping self.model = mapping.init( self.config.file_path, db_url, self.config.database_engine_options ) - # Security helper + # Initialize the Tool SHed security helper. self.security = security.SecurityHelper( id_secret=self.config.id_secret ) - # Tag handler + # initialize the Tool Shed tag handler. self.tag_handler = CommunityTagHandler() - # Tool data tables - never pass a config file here because the tool shed should always have an empty dictionary! + # Initialize the Tool Shed tool data tables. Never pass a configuration file here + # because the Tool Shed should always have an empty dictionary! self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_path ) - # The tool shed has no toolbox, but this attribute is still required. + # The Tool Shed makes no use of a Galaxy toolbox, but this attribute is still required. self.toolbox = tools.ToolBox( [], self.config.tool_path, self ) - # Load security policy + # Initialize the Tool Shed security agent. self.security_agent = self.model.security_agent + # The Tool Shed makes no use of a quota, but this attribute is still required. self.quota_agent = galaxy.quota.NoQuotaAgent( self.model ) # TODO: Add OpenID support self.openid_providers = OpenIDProviders() + # Initialize the baseline Tool Shed statistics component. self.shed_counter = self.model.shed_counter - # Let the HgwebConfigManager know where the hgweb.config file is located. + # Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located. self.hgweb_config_manager = self.model.hgweb_config_manager self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir print >> sys.stderr, "Tool shed hgweb.config file is: ", self.hgweb_config_manager.hgweb_config + def shutdown( self ): pass diff -r 55c0a99ff035b9d11adef3a931312c64bb99a45b -r e7b2e3c626322294668c5aab56c830d5a2eb5139 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -115,6 +115,10 @@ return trans.response.send_redirect( web.url_for( controller='repository', action='browse_repositories', **kwd ) ) + title = trans.app.repository_grid_filter_manager.get_grid_title( trans, + trailing_string='by Category', + default='Categories' ) + self.category_grid.title = title return self.category_grid( trans, **kwd ) @web.expose @@ -400,12 +404,16 @@ operation='view_or_manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=selected_changeset_revision ) ) + title = trans.app.repository_grid_filter_manager.get_grid_title( trans, + trailing_string='', + default='Repositories' ) + self.repository_grid.title = title return self.repository_grid( trans, **kwd ) @web.expose def browse_repositories_by_user( self, trans, **kwd ): """Display the list of repositories owned by a specified user.""" - # Eliminate the current filters if any exist. + # Eliminate the current search filters if any exist. for k, v in kwd.items(): if k.startswith( 'f-' ): del kwd[ k ] @@ -433,7 +441,15 @@ changeset_revision=selected_changeset_revision ) ) if user_id: user = suc.get_user( trans, user_id ) - self.repositories_by_user_grid.title = "Repositories owned by %s" % user.username + trailing_string = 'Owned by %s' % str( user.username ) + default = 'Repositories Owned by %s' % str( user.username ) + else: + trailing_string = '' + default='Repositories' + title = trans.app.repository_grid_filter_manager.get_grid_title( trans, + trailing_string=trailing_string, + default=default ) + self.repositories_by_user_grid.title = title return self.repositories_by_user_grid( trans, **kwd ) @web.expose @@ -519,7 +535,15 @@ if category_id: category = suc.get_category( trans, category_id ) if category: - self.repositories_in_category_grid.title = 'Category %s' % str( category.name ) + trailing_string = 'in Category %s' % str( category.name ) + else: + trailing_string = 'in Category' + else: + trailing_string = 'in Category' + title = trans.app.repository_grid_filter_manager.get_grid_title( trans, + trailing_string=trailing_string, + default='Repositories' ) + self.repositories_in_category_grid.title = title return self.repositories_in_category_grid( trans, **kwd ) @web.expose @@ -767,6 +791,7 @@ @web.expose def browse_valid_categories( self, trans, **kwd ): + """Filter repositories per category by those that are valid for installing into Galaxy.""" # The request came from Galaxy, so restrict category links to display only valid repository changeset revisions. galaxy_url = suc.handle_galaxy_url( trans, **kwd ) if galaxy_url: @@ -797,10 +822,15 @@ return trans.response.send_redirect( web.url_for( controller='repository', action='browse_valid_repositories', **kwd ) ) + title = trans.app.repository_grid_filter_manager.get_grid_title( trans, + trailing_string='by Category', + default='Categories of Valid Repositories' ) + self.valid_category_grid.title = title return self.valid_category_grid( trans, **kwd ) @web.expose def browse_valid_repositories( self, trans, **kwd ): + """Filter repositories to those that are installable into Galaxy.""" galaxy_url = suc.handle_galaxy_url( trans, **kwd ) if galaxy_url: kwd[ 'galaxy_url' ] = galaxy_url @@ -843,6 +873,10 @@ url_args=url_args, allow_multiple=False, async_compatible=False ) ] + title = trans.app.repository_grid_filter_manager.get_grid_title( trans, + trailing_string='', + default='Valid Repositories' ) + self.valid_repository_grid.title = title return self.valid_repository_grid( trans, **kwd ) @web.expose @@ -1962,7 +1996,7 @@ repository_metadata = trans.sa_session.query( trans.model.RepositoryMetadata ).first() current_user = trans.user # TODO: move the following to some in-memory register so these queries can be done once - # at startup. The in-memory registe can then be managed during the current session. + # at startup. The in-memory register can then be managed during the current session. can_administer_repositories = False has_reviewed_repositories = False has_deprecated_repositories = False diff -r 55c0a99ff035b9d11adef3a931312c64bb99a45b -r e7b2e3c626322294668c5aab56c830d5a2eb5139 lib/tool_shed/grids/repository_grids.py --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -41,11 +41,28 @@ class RepositoriesColumn( grids.TextColumn ): def get_value( self, trans, grid, category ): + # TODO: we should probably keep an in-memory register to improve speed here. if category.repositories: viewable_repositories = 0 for rca in category.repositories: - if not rca.repository.deleted and not rca.repository.deprecated: - viewable_repositories += 1 + repository = rca.repository + filter = trans.app.repository_grid_filter_manager.get_filter( trans ) + if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: + if not repository.deprecated: + is_level_one_certified_tuple = metadata_util.is_level_one_certified( trans, repository ) + latest_installable_changeset_revision, is_level_one_certified = is_level_one_certified_tuple + if is_level_one_certified: + viewable_repositories += 1 + elif filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: + if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and not repository.deprecated: + is_level_one_certified_tuple = metadata_util.is_level_one_certified( trans, repository ) + latest_installable_changeset_revision, is_level_one_certified = is_level_one_certified_tuple + if is_level_one_certified: + viewable_repositories += 1 + else: + # The value filter is None. + if not repository.deleted and not repository.deprecated: + viewable_repositories += 1 return viewable_repositories return 0 @@ -305,82 +322,32 @@ use_paging = False def build_initial_query( self, trans, **kwd ): - return trans.sa_session.query( model.Repository ) \ - .filter( and_( model.Repository.table.c.deleted == False, - model.Repository.table.c.deprecated == False ) ) \ - .join( model.User.table ) \ - .outerjoin( model.RepositoryCategoryAssociation.table ) \ - .outerjoin( model.Category.table ) - - -class CertifiedLevelOneRepositoryGrid( RepositoryGrid ): - columns = [ c for c in RepositoryGrid.columns ] - - def build_initial_query( self, trans, **kwd ): - clause_list = [] - for repository in trans.sa_session.query( model.Repository ) \ - .filter( and_( model.Repository.table.c.deleted == False, - model.Repository.table.c.deprecated == False ) ): - repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) - # Get the list of latest installable changeset revision for each repository since that is all - # that is currently configured for testing. - if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION ]: - # We can use the changelog tip. - latest_changeset_revision = str( repo.changectx( repo.changelog.tip() ) ) - else: - latest_changeset_revision = suc.get_latest_downloadable_changeset_revision( trans, repository, repo ) - if latest_changeset_revision not in [ None, suc.INITIAL_CHANGELOG_HASH ]: - encoded_repository_id = trans.security.encode_id( repository.id ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, - encoded_repository_id, - latest_changeset_revision ) - # Filter out repository revisions that have not been tested. - if repository_metadata.time_last_tested is not None and repository_metadata.tool_test_results is not None: - if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION ]: - # Look in the tool_test_results dictionary for installation errors. - try: - tool_test_results_dict = repository_metadata.tool_test_results[ -1 ] - except Exception, e: - # The repository is not certified. - continue - if 'installation_errors' in tool_test_results_dict: - continue - # The revision is level 1 certified - clause_list.append( "%s=%d and %s='%s'" % ( model.RepositoryMetadata.table.c.repository_id, - repository.id, - model.RepositoryMetadata.table.c.changeset_revision, - changeset_revision ) ) - else: - # We have an unrestricted repository. - if latest_changeset_revision.includes_tools: - if latest_changeset_revision.tools_functionally_correct: - # The revision is level 1 certified - clause_list.append( "%s=%d and %s='%s'" % ( model.RepositoryMetadata.table.c.repository_id, - repository.id, - model.RepositoryMetadata.table.c.changeset_revision, - changeset_revision ) ) - else: - # Look in the tool_test_results dictionary for installation errors. - try: - tool_test_results_dict = repository_metadata.tool_test_results[ -1 ] - except Exception, e: - # The repository is not certified. - continue - if 'installation_errors' in tool_test_results_dict: - continue - # The revision is level 1 certified - clause_list.append( "%s=%d and %s='%s'" % ( model.RepositoryMetadata.table.c.repository_id, - repository.id, - model.RepositoryMetadata.table.c.changeset_revision, - changeset_revision ) ) - return trans.sa_session.query( model.Repository ) \ - .filter( and_( model.Repository.table.c.deleted == False, - model.Repository.table.c.deprecated == False ) ) \ - .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ - .join( model.User.table ) \ - .outerjoin( model.RepositoryCategoryAssociation.table ) \ - .outerjoin( model.Category.table ) + filter = trans.app.repository_grid_filter_manager.get_filter( trans ) + if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: + clause_list = get_certified_level_one_clause_list( trans ) + return trans.sa_session.query( model.Repository ) \ + .join( model.RepositoryMetadata.table ) \ + .filter( or_( *clause_list ) ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) + if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: + clause_list = get_certified_level_one_clause_list( trans ) + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.type == rt_util.REPOSITORY_SUITE_DEFINITION ) \ + .join( model.RepositoryMetadata.table ) \ + .filter( or_( *clause_list ) ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) + else: + # The filter is None. + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deleted == False, + model.Repository.table.c.deprecated == False ) ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) class EmailAlertsRepositoryGrid( RepositoryGrid ): @@ -566,13 +533,35 @@ def build_initial_query( self, trans, **kwd ): decoded_user_id = trans.security.decode_id( kwd[ 'user_id' ] ) - return trans.sa_session.query( model.Repository ) \ - .filter( and_( model.Repository.table.c.deleted == False, - model.Repository.table.c.deprecated == False, - model.Repository.table.c.user_id == decoded_user_id ) ) \ - .join( model.User.table ) \ - .outerjoin( model.RepositoryCategoryAssociation.table ) \ - .outerjoin( model.Category.table ) + filter = trans.app.repository_grid_filter_manager.get_filter( trans ) + if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: + clause_list = get_certified_level_one_clause_list( trans ) + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.table.c.user_id == decoded_user_id ) \ + .join( model.RepositoryMetadata.table ) \ + .filter( or_( *clause_list ) ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) + if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: + clause_list = get_certified_level_one_clause_list( trans ) + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.type == rt_util.REPOSITORY_SUITE_DEFINITION, + model.Repository.table.c.user_id == decoded_user_id ) ) \ + .join( model.RepositoryMetadata.table ) \ + .filter( or_( *clause_list ) ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) + else: + # The value of filter is None. + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deleted == False, + model.Repository.table.c.deprecated == False, + model.Repository.table.c.user_id == decoded_user_id ) ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) class RepositoriesInCategoryGrid( RepositoryGrid ): @@ -610,22 +599,63 @@ def build_initial_query( self, trans, **kwd ): category_id = kwd.get( 'id', None ) - if category_id: - category = suc.get_category( trans, category_id ) - if category: - return trans.sa_session.query( model.Repository ) \ - .filter( and_( model.Repository.table.c.deleted == False, - model.Repository.table.c.deprecated == False ) ) \ - .join( model.User.table ) \ - .outerjoin( model.RepositoryCategoryAssociation.table ) \ - .outerjoin( model.Category.table ) \ - .filter( model.Category.table.c.name == category.name ) - return trans.sa_session.query( model.Repository ) \ - .filter( and_( model.Repository.table.c.deleted == False, - model.Repository.table.c.deprecated == False ) ) \ - .join( model.User.table ) \ - .outerjoin( model.RepositoryCategoryAssociation.table ) \ - .outerjoin( model.Category.table ) + filter = trans.app.repository_grid_filter_manager.get_filter( trans ) + if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: + clause_list = get_certified_level_one_clause_list( trans ) + if category_id: + category = suc.get_category( trans, category_id ) + if category: + return trans.sa_session.query( model.Repository ) \ + .join( model.RepositoryMetadata.table ) \ + .filter( or_( *clause_list ) ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) \ + .filter( model.Category.table.c.name == category.name ) + return trans.sa_session.query( model.Repository ) \ + .join( model.RepositoryMetadata.table ) \ + .filter( or_( *clause_list ) ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) + if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: + clause_list = get_certified_level_one_clause_list( trans ) + if category_id: + category = suc.get_category( trans, category_id ) + if category: + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.type == rt_util.REPOSITORY_SUITE_DEFINITION ) \ + .join( model.RepositoryMetadata.table ) \ + .filter( or_( *clause_list ) ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) \ + .filter( model.Category.table.c.name == category.name ) + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.type == rt_util.REPOSITORY_SUITE_DEFINITION ) \ + .join( model.RepositoryMetadata.table ) \ + .filter( or_( *clause_list ) ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) + else: + # The value of filter is None. + if category_id: + category = suc.get_category( trans, category_id ) + if category: + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deleted == False, + model.Repository.table.c.deprecated == False ) ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) \ + .filter( model.Category.table.c.name == category.name ) + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deleted == False, + model.Repository.table.c.deprecated == False ) ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) class RepositoriesIOwnGrid( RepositoryGrid ): @@ -767,8 +797,8 @@ if username in allow_push_usernames: user_clause_list.append( model.Repository.table.c.id == repository.id ) if user_clause_list: - # We have the list of repositories that the current user is authorized to update, so filter further by latest installable revisions that contain - # tools with missing tool test components. + # We have the list of repositories that the current user is authorized to update, so filter + # further by latest installable revisions that contain tools with missing tool test components. revision_clause_list = [] for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, @@ -854,8 +884,8 @@ if username in allow_push_usernames: user_clause_list.append( model.Repository.table.c.id == repository.id ) if user_clause_list: - # We have the list of repositories that the current user is authorized to update, so filter further by latest installable revisions that contain - # tools with missing tool test components. + # We have the list of repositories that the current user is authorized to update, so filter + # further by latest installable revisions that contain tools with missing tool test components. revision_clause_list = [] for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, @@ -941,8 +971,8 @@ if username in allow_push_usernames: user_clause_list.append( model.Repository.table.c.id == repository.id ) if user_clause_list: - # We have the list of repositories that the current user is authorized to update, so filter further by latest installable revisions that contain - # tools with missing tool test components. + # We have the list of repositories that the current user is authorized to update, so filter + # further by latest installable revisions that contain tools with missing tool test components. revision_clause_list = [] for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, @@ -1060,8 +1090,8 @@ if username in allow_push_usernames: user_clause_list.append( model.Repository.table.c.id == repository.id ) if user_clause_list: - # We have the list of repositories that the current user is authorized to update, so filter further by latest installable revisions that contain - # tools with at least 1 failing tool test. + # We have the list of repositories that the current user is authorized to update, so filter + # further by latest installable revisions that contain tools with at least 1 failing tool test. revision_clause_list = [] for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, @@ -1107,8 +1137,8 @@ use_paging = False def build_initial_query( self, trans, **kwd ): - # We have the list of repositories that the current user is authorized to update, so filter further by latest installable revisions that contain - # tools with at least 1 failing tool test. + # We have the list of repositories that the current user is authorized to update, so filter + # further by latest installable revisions that contain tools with at least 1 failing tool test. revision_clause_list = [] for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, @@ -1148,8 +1178,9 @@ if username in allow_push_usernames: user_clause_list.append( model.Repository.table.c.id == repository.id ) if user_clause_list: - # We have the list of repositories that the current user is authorized to update, so filter further by latest installable revisions that contain - # at least 1 tool, no missing tool test components, and no failing tool tests. + # We have the list of repositories that the current user is authorized to update, so filter + # further by latest installable revisions that contain at least 1 tool, no missing tool test + # components, and no failing tool tests. revision_clause_list = [] for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, @@ -1254,8 +1285,8 @@ if username in allow_push_usernames: user_clause_list.append( model.Repository.table.c.id == repository.id ) if user_clause_list: - # We have the list of repositories that the current user is authorized to update, so filter further by latest metadata revisions that contain - # invalid tools. + # We have the list of repositories that the current user is authorized to update, so filter + # further by latest metadata revisions that contain invalid tools. revision_clause_list = [] for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, @@ -1709,16 +1740,33 @@ class RepositoriesColumn( grids.TextColumn ): def get_value( self, trans, grid, category ): + # TODO: We should probably keep in in-memory register for speed improvements. if category.repositories: viewable_repositories = 0 for rca in category.repositories: repository = rca.repository - if not repository.deleted and not repository.deprecated and repository.downloadable_revisions: - viewable_repositories += 1 + filter = trans.app.repository_grid_filter_manager.get_filter( trans ) + if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: + if not repository.deprecated and repository.downloadable_revisions: + is_level_one_certified_tuple = metadata_util.is_level_one_certified( trans, repository ) + latest_installable_changeset_revision, is_level_one_certified = is_level_one_certified_tuple + if is_level_one_certified: + viewable_repositories += 1 + if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: + if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and \ + not repository.deprecated and repository.downloadable_revisions: + is_level_one_certified_tuple = metadata_util.is_level_one_certified( trans, repository ) + latest_installable_changeset_revision, is_level_one_certified = is_level_one_certified_tuple + if is_level_one_certified: + viewable_repositories += 1 + else: + # The value of filter is None. + if not repository.deleted and not repository.deprecated and repository.downloadable_revisions: + viewable_repositories += 1 return viewable_repositories return 0 - title = "Categories of valid repositories" + title = "Categories of Valid Repositories" model_class = model.Category template='/webapps/tool_shed/category/valid_grid.mako' default_sort_key = "name" @@ -1786,7 +1834,7 @@ return select_field.options[ 0 ][ 0 ] return '' - title = "Valid repositories" + title = "Valid Repositories" columns = [ RepositoryGrid.NameColumn( "Name", key="name", @@ -1815,26 +1863,72 @@ use_paging = False def build_initial_query( self, trans, **kwd ): + filter = trans.app.repository_grid_filter_manager.get_filter( trans ) if 'id' in kwd: - # The user is browsing categories of valid repositories, so filter the request by the received id, which is a category id. + # The user is browsing categories of valid repositories, so filter the request by the received id, + # which is a category id. + if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: + clause_list = get_certified_level_one_clause_list( trans ) + return trans.sa_session.query( model.Repository ) \ + .join( model.RepositoryMetadata.table ) \ + .filter( or_( *clause_list ) ) \ + .join( model.User.table ) \ + .join( model.RepositoryCategoryAssociation.table ) \ + .join( model.Category.table ) \ + .filter( and_( model.Category.table.c.id == trans.security.decode_id( kwd[ 'id' ] ), + model.RepositoryMetadata.table.c.downloadable == True ) ) + if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: + clause_list = get_certified_level_one_clause_list( trans ) + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.type == rt_util.REPOSITORY_SUITE_DEFINITION ) \ + .join( model.RepositoryMetadata.table ) \ + .filter( or_( *clause_list ) ) \ + .join( model.User.table ) \ + .join( model.RepositoryCategoryAssociation.table ) \ + .join( model.Category.table ) \ + .filter( and_( model.Category.table.c.id == trans.security.decode_id( kwd[ 'id' ] ), + model.RepositoryMetadata.table.c.downloadable == True ) ) + else: + # The value of filter is None. + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deleted == False, + model.Repository.table.c.deprecated == False ) ) \ + .join( model.RepositoryMetadata.table ) \ + .join( model.User.table ) \ + .join( model.RepositoryCategoryAssociation.table ) \ + .join( model.Category.table ) \ + .filter( and_( model.Category.table.c.id == trans.security.decode_id( kwd[ 'id' ] ), + model.RepositoryMetadata.table.c.downloadable == True ) ) + # The user performed a free text search on the ValidCategoryGrid. + if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: + clause_list = get_certified_level_one_clause_list( trans ) + return trans.sa_session.query( model.Repository ) \ + .join( model.RepositoryMetadata.table ) \ + .filter( or_( *clause_list ) ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) \ + .filter( model.RepositoryMetadata.table.c.downloadable == True ) + if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: + clause_list = get_certified_level_one_clause_list( trans ) + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.type == rt_util.REPOSITORY_SUITE_DEFINITION ) \ + .join( model.RepositoryMetadata.table ) \ + .filter( or_( *clause_list ) ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) \ + .filter( model.RepositoryMetadata.table.c.downloadable == True ) + else: + # The value of filter is None. return trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deleted == False, model.Repository.table.c.deprecated == False ) ) \ .join( model.RepositoryMetadata.table ) \ .join( model.User.table ) \ - .join( model.RepositoryCategoryAssociation.table ) \ - .join( model.Category.table ) \ - .filter( and_( model.Category.table.c.id == trans.security.decode_id( kwd[ 'id' ] ), - model.RepositoryMetadata.table.c.downloadable == True ) ) - # The user performed a free text search on the ValidCategoryGrid. - return trans.sa_session.query( model.Repository ) \ - .filter( and_( model.Repository.table.c.deleted == False, - model.Repository.table.c.deprecated == False ) ) \ - .join( model.RepositoryMetadata.table ) \ - .join( model.User.table ) \ - .outerjoin( model.RepositoryCategoryAssociation.table ) \ - .outerjoin( model.Category.table ) \ - .filter( model.RepositoryMetadata.table.c.downloadable == True ) + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) \ + .filter( model.RepositoryMetadata.table.c.downloadable == True ) # ------ utility methods ------------------- @@ -1914,6 +2008,20 @@ return repository_metadata.changeset_revision return None +def get_certified_level_one_clause_list( trans ): + clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deleted == False, + model.Repository.table.c.deprecated == False ) ): + is_level_one_certified_tuple = metadata_util.is_level_one_certified( trans, repository ) + latest_installable_changeset_revision, is_level_one_certified = is_level_one_certified_tuple + if is_level_one_certified: + clause_list.append( "%s=%d and %s='%s'" % ( model.RepositoryMetadata.table.c.repository_id, + repository.id, + model.RepositoryMetadata.table.c.changeset_revision, + latest_installable_changeset_revision ) ) + return clause_list + def get_latest_downloadable_repository_metadata( trans, repository ): """ Return the latest downloadable repository_metadata record for the received repository. This will diff -r 55c0a99ff035b9d11adef3a931312c64bb99a45b -r e7b2e3c626322294668c5aab56c830d5a2eb5139 lib/tool_shed/grids/util.py --- a/lib/tool_shed/grids/util.py +++ b/lib/tool_shed/grids/util.py @@ -1,7 +1,52 @@ -import os, logging +import logging +import os import tool_shed.util.shed_util_common as suc import tool_shed.util.metadata_util as metadata_util from galaxy.web.form_builder import SelectField +from galaxy.util.bunch import Bunch + +log = logging.getLogger( __name__ ) + + +class RepositoryGridFilterManager( object ): + """Provides filtered views of the many Tool SHed repository grids.""" + + filters = Bunch( CERTIFIED_LEVEL_ONE = 'certified_level_one', + CERTIFIED_LEVEL_TWO = 'certified_level_two', + CERTIFIED_LEVEL_ONE_SUITES = 'certified_level_one_suites', + CERTIFIED_LEVEL_TWO_SUITES = 'certified_level_two_suites' ) + + def get_grid_title( self, trans, trailing_string='', default='' ): + filter = self.get_filter( trans ) + if filter == self.filters.CERTIFIED_LEVEL_ONE: + return "Certified 1 Repositories %s" % trailing_string + if filter == self.filters.CERTIFIED_LEVEL_TWO: + return "Certified 2 Repositories %s" % trailing_string + if filter == self.filters.CERTIFIED_LEVEL_ONE_SUITES: + return "Certified 1 Repository Suites %s" % trailing_string + if filter == self.filters.CERTIFIED_LEVEL_TWO_SUITES: + return "Certified 2 Repository Suites %s" % trailing_string + return "%s" % default + + def get_filter( self, trans ): + filter = trans.get_cookie( name='toolshedrepogridfilter' ) + return filter or None + + def is_valid_filter( self, filter ): + if filter is None: + return True + for valid_key, valid_filter in self.filters.items(): + if filter == valid_filter: + return True + return False + + def set_filter( self, trans, **kwd ): + # Set a session cookie value with the selected filter. + filter = kwd.get( 'filter', None ) + if filter is not None and self.is_valid_filter( filter ): + trans.set_cookie( value=filter, name='toolshedrepogridfilter' ) + # if the filter is not valid, expire the cookie. + trans.set_cookie( value=filter,name='toolshedrepogridfilter', age=-1 ) def build_approved_select_field( trans, name, selected_value=None, for_component=True ): options = [ ( 'No', trans.model.ComponentReview.approved_states.NO ), @@ -18,7 +63,10 @@ def build_changeset_revision_select_field( trans, repository, selected_value=None, add_id_to_name=True, downloadable=False, reviewed=False, not_reviewed=False ): - """Build a SelectField whose options are the changeset_rev strings of certain revisions of the received repository.""" + """ + Build a SelectField whose options are the changeset_rev strings of certain revisions of the + received repository. + """ options = [] changeset_tups = [] refresh_on_change_values = [] diff -r 55c0a99ff035b9d11adef3a931312c64bb99a45b -r e7b2e3c626322294668c5aab56c830d5a2eb5139 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -1349,6 +1349,63 @@ return True return False +def is_level_one_certified( trans, repository ): + """ + Return True if the latest installable changeset_revision of the received repository is level one certified. + """ + if repository.deleted: + return ( None, False ) + repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) + # Get the latest installable changeset revision since that is all that is currently configured for testing. + latest_installable_changeset_revision = suc.get_latest_downloadable_changeset_revision( trans, repository, repo ) + if latest_installable_changeset_revision not in [ None, suc.INITIAL_CHANGELOG_HASH ]: + encoded_repository_id = trans.security.encode_id( repository.id ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + encoded_repository_id, + latest_installable_changeset_revision ) + if repository_metadata: + # Filter out repository revisions that have not been tested. + if repository_metadata.time_last_tested is not None and repository_metadata.tool_test_results is not None: + if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION ]: + # Look in the tool_test_results dictionary for installation errors. + try: + tool_test_results_dict = repository_metadata.tool_test_results[ 0 ] + except Exception, e: + message = 'Error attempting to retrieve install and test results for repository %s:\n' % str( repository.name ) + message += '%s' % str( e ) + log.exception( message ) + return ( latest_installable_changeset_revision, False ) + if 'installation_errors' in tool_test_results_dict: + return ( latest_installable_changeset_revision, False ) + return ( latest_installable_changeset_revision, True ) + else: + # We have a repository with type Unrestricted. + if repository_metadata.includes_tools: + if repository_metadata.tools_functionally_correct: + return ( latest_installable_changeset_revision, True ) + return ( latest_installable_changeset_revision, False ) + else: + # Look in the tool_test_results dictionary for installation errors. + try: + tool_test_results_dict = repository_metadata.tool_test_results[ 0 ] + except Exception, e: + message = 'Error attempting to retrieve install and test results for repository %s:\n' % str( repository.name ) + message += '%s' % str( e ) + log.exception( message ) + return ( latest_installable_changeset_revision, False ) + if 'installation_errors' in tool_test_results_dict: + return ( latest_installable_changeset_revision, False ) + return ( latest_installable_changeset_revision, True ) + else: + # No test results. + return ( latest_installable_changeset_revision, False ) + else: + # No repository_metadata. + return ( latest_installable_changeset_revision, False ) + else: + # No installable changeset_revision. + return ( None, False ) + def new_datatypes_metadata_required( trans, repository_metadata, metadata_dict ): """ Compare the last saved metadata for each datatype in the repository with the new metadata in metadata_dict to determine if a new diff -r 55c0a99ff035b9d11adef3a931312c64bb99a45b -r e7b2e3c626322294668c5aab56c830d5a2eb5139 templates/webapps/tool_shed/category/grid.mako --- a/templates/webapps/tool_shed/category/grid.mako +++ b/templates/webapps/tool_shed/category/grid.mako @@ -3,11 +3,11 @@ <%namespace name="grid_common" file="../common/grid_common.mako" import="*" /><%def name="insert()"> -<% - from tool_shed.grids.repository_grids import RepositoryGrid - repo_grid = RepositoryGrid() - grid_common.render_grid_filters(repo_grid) -%> + <% + from tool_shed.grids.repository_grids import RepositoryGrid + repo_grid = RepositoryGrid() + grid_common.render_grid_filters(repo_grid) + %></%def> -${grid_base.load(False, capture(self.insert))} \ No newline at end of file +${grid_base.load(False, capture(self.insert))} diff -r 55c0a99ff035b9d11adef3a931312c64bb99a45b -r e7b2e3c626322294668c5aab56c830d5a2eb5139 test/tool_shed/functional/test_0420_citable_urls_for_repositories.py --- a/test/tool_shed/functional/test_0420_citable_urls_for_repositories.py +++ b/test/tool_shed/functional/test_0420_citable_urls_for_repositories.py @@ -207,7 +207,7 @@ strings_displayed = [ '/repository', 'browse_repositories', 'user1' ] strings_displayed.extend( [ 'list+of+repositories+owned', 'does+not+include+one+named', '%21%21invalid%21%21', 'status=error' ] ) strings_displayed_in_iframe = [ 'user1', 'filtering_0420' ] - strings_displayed_in_iframe.append( 'Repositories owned by user1' ) + strings_displayed_in_iframe.append( 'Repositories Owned by user1' ) strings_displayed_in_iframe.append( tip_revision ) self.load_citable_url( username='user1', repository_name='!!invalid!!', Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.