[galaxy-commits] commit/galaxy-central: greg: Add a repository registry to the Tool Shed to keep in-memory information about repositories. This will significantly improve some page rendering in the tool shed, specifically the current main Categories splash page. This registry will provide filtered certification information for the planned Tool Shed splash page which will be rendered with Javascript. The Tool Shed's install and test framework still needs to be enhanced to update this registry during install

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b8e434a78496/ Changeset: b8e434a78496 User: greg Date: 2014-05-20 23:18:14 Summary: Add a repository registry to the Tool Shed to keep in-memory information about repositories. This will significantly improve some page rendering in the tool shed, specifically the current main Categories splash page. This registry will provide filtered certification information for the planned Tool Shed splash page which will be rendered with Javascript. The Tool Shed's install and test framework still needs to be enhanced to update this registry during install and test runs. Affected #: 20 files diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -127,7 +127,7 @@ action='show', id=encoded_repository_id ) # Get the repository_metadata information. - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, changeset_revision ) if repository_metadata is None: @@ -136,7 +136,7 @@ repo_dir = repository.repo_path( trans.app ) repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) new_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, new_changeset_revision ) changeset_revision = new_changeset_revision diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -164,7 +164,7 @@ continue repository_dependency_id = trans.security.encode_id( repository_dependency.id ) repository_dependency_repository_metadata = \ - suc.get_repository_metadata_by_changeset_revision( trans, repository_dependency_id, changeset_revision ) + suc.get_repository_metadata_by_changeset_revision( trans.app, repository_dependency_id, changeset_revision ) if repository_dependency_repository_metadata is None: # The changeset_revision column in the repository_metadata table has been updated with a new # value value, so find the changeset_revision to which we need to update. @@ -174,7 +174,7 @@ repo, changeset_revision ) repository_dependency_repository_metadata = \ - suc.get_repository_metadata_by_changeset_revision( trans, + suc.get_repository_metadata_by_changeset_revision( trans.app, repository_dependency_id, new_changeset_revision ) if repository_dependency_repository_metadata is None: diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/galaxy/webapps/tool_shed/app.py --- a/lib/galaxy/webapps/tool_shed/app.py +++ b/lib/galaxy/webapps/tool_shed/app.py @@ -9,7 +9,8 @@ from galaxy.util.dbkeys import GenomeBuilds from galaxy.web import security from galaxy.tags.tag_handler import CommunityTagHandler -from tool_shed.grids.util import RepositoryGridFilterManager +from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager +import tool_shed.repository_registry import tool_shed.repository_types.registry @@ -64,6 +65,8 @@ # Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located. self.hgweb_config_manager = self.model.hgweb_config_manager self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir + # Initialize the repository registry. + self.repository_registry = tool_shed.repository_registry.Registry( self ) print >> sys.stderr, "Tool shed hgweb.config file is: ", self.hgweb_config_manager.hgweb_config def shutdown( self ): diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -920,7 +920,7 @@ # Return the same value for changeset_revision and latest_changeset_revision. url += latest_changeset_revision else: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_revision ) if repository_metadata: @@ -945,7 +945,7 @@ latest_changeset_revision = changeset_hash break else: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_hash ) if repository_metadata: @@ -1189,7 +1189,7 @@ repositories_archive_path, file_name = os.path.split( repositories_archive.name ) suc.remove_dir( repositories_archive_path ) return opened_archive - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) metadata = repository_metadata.metadata # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. repository_dependencies = \ @@ -1454,7 +1454,7 @@ owner = kwd.get( 'owner', None ) changeset_revision = kwd.get( 'changeset_revision', None ) repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_revision ) has_galaxy_utilities_dict = has_galaxy_utilities( repository_metadata ) @@ -1500,7 +1500,7 @@ changeset_hash = str( repo.changectx( changeset ) ) ctx = hg_util.get_changectx_for_changeset( repo, changeset_hash ) if update_to_changeset_hash: - update_to_repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + update_to_repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_hash ) if update_to_repository_metadata: @@ -1606,7 +1606,7 @@ repository = repository_metadata.repository repo_dir = repository.repo_path( trans.app ) repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) - latest_downloadable_changeset_revsion = suc.get_latest_downloadable_changeset_revision( trans, repository, repo ) + latest_downloadable_changeset_revsion = suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) if repository_metadata.changeset_revision == latest_downloadable_changeset_revsion: # We'll display only the test run for the latest installable revision in the rss feed. tool_test_results = repository_metadata.tool_test_results @@ -1674,7 +1674,7 @@ repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) if repository: repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) - return suc.get_latest_downloadable_changeset_revision( trans, repository, repo ) + return suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) return suc.INITIAL_CHANGELOG_HASH @web.json @@ -1690,7 +1690,7 @@ repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) if repository: repository_metadata = \ - suc.get_repository_metadata_by_changeset_revision( trans, + suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_revision ) if repository_metadata: @@ -1862,7 +1862,7 @@ tool_version_dicts = [] for changeset in repo.changelog: current_changeset_revision = str( repo.changectx( changeset ) ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), current_changeset_revision ) if repository_metadata and repository_metadata.tool_versions: @@ -1884,14 +1884,14 @@ repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans, repository ) repo_dir = repository.repo_path( trans.app ) repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if not repository_metadata: # The received changeset_revision is no longer associated with metadata, so get the next changeset_revision in the repository # changelog that is associated with metadata. changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, after_changeset_revision=changeset_revision ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) repo_info_dict = repository_util.create_repo_info_dict( trans=trans, repository_clone_url=repository_clone_url, @@ -2233,7 +2233,7 @@ trans.sa_session.flush() message += "The repository information has been updated." elif kwd.get( 'skip_tool_tests_button', False ): - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) skip_tool_test = repository_metadata.skip_tool_tests if skip_tool_test: # Handle the mapper behavior. @@ -2327,7 +2327,7 @@ skip_tool_test = None repository_dependencies = None if changeset_revision != suc.INITIAL_CHANGELOG_HASH: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata: revision_label = hg_util.get_revision_label( trans, repository, changeset_revision, include_date=False ) metadata = repository_metadata.metadata @@ -2338,7 +2338,7 @@ previous_changeset_revision = \ suc.get_previous_metadata_changeset_revision( repository, repo, changeset_revision, downloadable=False ) if previous_changeset_revision != suc.INITIAL_CHANGELOG_HASH: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, previous_changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, previous_changeset_revision ) if repository_metadata: revision_label = hg_util.get_revision_label( trans, repository, previous_changeset_revision, include_date=False ) metadata = repository_metadata.metadata @@ -2439,7 +2439,7 @@ changeset_revision = kwd.get( 'changeset_revision', repository.tip( trans.app ) ) metadata = None if changeset_revision != suc.INITIAL_CHANGELOG_HASH: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata else: @@ -2453,7 +2453,7 @@ changeset_revision, downloadable=False ) if previous_changeset_revision != suc.INITIAL_CHANGELOG_HASH: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, previous_changeset_revision ) if repository_metadata: @@ -2545,7 +2545,7 @@ repo_dir = repository.repo_path( trans.app ) repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) changeset_revision = kwd.get( 'changeset_revision', repository.tip( trans.app ) ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if repository_metadata: repository_metadata_id = trans.security.encode_id( repository_metadata.id ), metadata = repository_metadata.metadata @@ -2857,7 +2857,7 @@ def set_malicious( self, trans, id, ctx_str, **kwd ): malicious = kwd.get( 'malicious', '' ) if kwd.get( 'malicious_button', False ): - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, ctx_str ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, ctx_str ) malicious_checked = CheckboxField.is_checked( malicious ) repository_metadata.malicious = malicious_checked trans.sa_session.add( repository_metadata ) @@ -2970,7 +2970,7 @@ changeset_revision = kwd.get( 'changeset_revision', None ) repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) if repository: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_revision ) repo_dir = repository.repo_path( trans.app ) @@ -3070,7 +3070,7 @@ changesets = [] for changeset in repo.changelog: ctx = repo.changectx( changeset ) - if suc.get_repository_metadata_by_changeset_revision( trans, id, str( ctx ) ): + if suc.get_repository_metadata_by_changeset_revision( trans.app, id, str( ctx ) ): has_metadata = True else: has_metadata = False @@ -3225,7 +3225,7 @@ add_id_to_name=False, downloadable=False ) revision_label = hg_util.get_revision_label( trans, repository, changeset_revision, include_date=False ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. @@ -3291,7 +3291,7 @@ guid = None original_tool_data_path = trans.app.config.tool_data_path revision_label = hg_util.get_revision_label( trans, repository, changeset_revision, include_date=False ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if repository_metadata: repository_metadata_id = trans.security.encode_id( repository_metadata.id ) metadata = repository_metadata.metadata diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/galaxy/webapps/tool_shed/controllers/repository_review.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py @@ -155,7 +155,7 @@ **kwd ) ) # A review can be initially performed only on an installable revision of a repository, so make sure we have metadata associated # with the received changeset_revision. - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata if metadata: @@ -485,7 +485,7 @@ repository_reviews = review_util.get_reviews_by_repository_id_changeset_revision( trans, repository_id, changeset_revision ) # Determine if the current user can add a review to this revision. can_add_review = trans.user not in [ repository_review.user for repository_review in repository_reviews ] - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if repository_metadata: repository_metadata_reviews = util.listify( repository_metadata.reviews ) else: diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -59,7 +59,7 @@ repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) if trans.webapp.name == 'tool_shed': # We're in the tool shed. - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_revision ) if repository_metadata: @@ -189,7 +189,7 @@ repo_dir = repository.repo_path( trans.app ) repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans, repository ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if not repository_metadata: @@ -200,7 +200,7 @@ suc.get_next_downloadable_changeset_revision( repository,repo, changeset_revision ) if next_downloadable_changeset_revision: repository_metadata = \ - suc.get_repository_metadata_by_changeset_revision( trans, repository_id, next_downloadable_changeset_revision ) + suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, next_downloadable_changeset_revision ) if repository_metadata: # For now, we'll always assume that we'll get repository_metadata, but if we discover our assumption # is not valid we'll have to enhance the callers to handle repository_metadata values of None in the diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py @@ -552,4 +552,3 @@ remove_from_disk=False ) tool_dependencies.append( tool_dependency ) return tool_dependencies - diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/grids/repository_grid_filter_manager.py --- /dev/null +++ b/lib/tool_shed/grids/repository_grid_filter_manager.py @@ -0,0 +1,50 @@ +import logging +import os + +from galaxy.util.bunch import Bunch + +log = logging.getLogger( __name__ ) + + +class RepositoryGridFilterManager( object ): + """Provides filtered views of the many Tool SHed repository grids.""" + + filters = Bunch( CERTIFIED_LEVEL_ONE = 'certified_level_one', + CERTIFIED_LEVEL_TWO = 'certified_level_two', + CERTIFIED_LEVEL_ONE_SUITES = 'certified_level_one_suites', + CERTIFIED_LEVEL_TWO_SUITES = 'certified_level_two_suites', + SUITES = 'suites' ) + + def get_grid_title( self, trans, trailing_string='', default='' ): + filter = self.get_filter( trans ) + if filter == self.filters.CERTIFIED_LEVEL_ONE: + return "Certified 1 Repositories %s" % trailing_string + if filter == self.filters.CERTIFIED_LEVEL_TWO: + return "Certified 2 Repositories %s" % trailing_string + if filter == self.filters.CERTIFIED_LEVEL_ONE_SUITES: + return "Certified 1 Repository Suites %s" % trailing_string + if filter == self.filters.CERTIFIED_LEVEL_TWO_SUITES: + return "Certified 2 Repository Suites %s" % trailing_string + if filter == self.filters.SUITES: + return "Repository Suites %s" % trailing_string + return "%s" % default + + def get_filter( self, trans ): + filter = trans.get_cookie( name='toolshedrepogridfilter' ) + return filter or None + + def is_valid_filter( self, filter ): + if filter is None: + return True + for valid_key, valid_filter in self.filters.items(): + if filter == valid_filter: + return True + return False + + def set_filter( self, trans, **kwd ): + # Set a session cookie value with the selected filter. + filter = kwd.get( 'filter', None ) + if filter is not None and self.is_valid_filter( filter ): + trans.set_cookie( value=filter, name='toolshedrepogridfilter' ) + # if the filter is not valid, expire the cookie. + trans.set_cookie( value=filter,name='toolshedrepogridfilter', age=-1 ) diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/grids/repository_grids.py --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -42,30 +42,17 @@ class RepositoriesColumn( grids.TextColumn ): def get_value( self, trans, grid, category ): - # TODO: we should probably keep an in-memory register to improve speed here. - if category.repositories: - viewable_repositories = 0 - for rca in category.repositories: - repository = rca.repository - filter = trans.app.repository_grid_filter_manager.get_filter( trans ) - if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: - if not repository.deprecated: - is_level_one_certified_tuple = metadata_util.is_level_one_certified( trans, repository ) - latest_installable_changeset_revision, is_level_one_certified = is_level_one_certified_tuple - if is_level_one_certified: - viewable_repositories += 1 - elif filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: - if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and not repository.deprecated: - is_level_one_certified_tuple = metadata_util.is_level_one_certified( trans, repository ) - latest_installable_changeset_revision, is_level_one_certified = is_level_one_certified_tuple - if is_level_one_certified: - viewable_repositories += 1 - else: - # The value filter is None. - if not repository.deleted and not repository.deprecated: - viewable_repositories += 1 - return viewable_repositories - return 0 + category_name = str( category.name ) + filter = trans.app.repository_grid_filter_manager.get_filter( trans ) + if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: + return trans.app.repository_registry.certified_level_one_viewable_repositories_and_suites_by_category.get( category_name, 0 ) + elif filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: + return trans.app.repository_registry.certified_level_one_viewable_suites_by_category.get( category_name, 0 ) + elif filter == trans.app.repository_grid_filter_manager.filters.SUITES: + return trans.app.repository_registry.viewable_suites_by_category.get( category_name, 0 ) + else: + # The value filter is None. + return trans.app.repository_registry.viewable_repositories_and_suites_by_category.get( category_name, 0 ) title = "Categories" model_class = model.Category @@ -325,19 +312,17 @@ def build_initial_query( self, trans, **kwd ): filter = trans.app.repository_grid_filter_manager.get_filter( trans ) if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: - clause_list = get_certified_level_one_clause_list( trans ) return trans.sa_session.query( model.Repository ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: - clause_list = get_certified_level_one_clause_list( trans ) return trans.sa_session.query( model.Repository ) \ .filter( model.Repository.type == rt_util.REPOSITORY_SUITE_DEFINITION ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) @@ -536,21 +521,19 @@ decoded_user_id = trans.security.decode_id( kwd[ 'user_id' ] ) filter = trans.app.repository_grid_filter_manager.get_filter( trans ) if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: - clause_list = get_certified_level_one_clause_list( trans ) return trans.sa_session.query( model.Repository ) \ .filter( model.Repository.table.c.user_id == decoded_user_id ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: - clause_list = get_certified_level_one_clause_list( trans ) return trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.type == rt_util.REPOSITORY_SUITE_DEFINITION, model.Repository.table.c.user_id == decoded_user_id ) ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) @@ -602,32 +585,30 @@ category_id = kwd.get( 'id', None ) filter = trans.app.repository_grid_filter_manager.get_filter( trans ) if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: - clause_list = get_certified_level_one_clause_list( trans ) if category_id: category = suc.get_category( trans, category_id ) if category: return trans.sa_session.query( model.Repository ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) \ .filter( model.Category.table.c.name == category.name ) return trans.sa_session.query( model.Repository ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: - clause_list = get_certified_level_one_clause_list( trans ) if category_id: category = suc.get_category( trans, category_id ) if category: return trans.sa_session.query( model.Repository ) \ .filter( model.Repository.type == rt_util.REPOSITORY_SUITE_DEFINITION ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) \ @@ -635,7 +616,7 @@ return trans.sa_session.query( model.Repository ) \ .filter( model.Repository.type == rt_util.REPOSITORY_SUITE_DEFINITION ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) @@ -763,7 +744,8 @@ for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_missing_tool_test_components( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_that_has_missing_tool_test_components( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -805,7 +787,8 @@ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ) \ .filter( or_( *user_clause_list ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_missing_tool_test_components( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_that_has_missing_tool_test_components( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -850,7 +833,8 @@ for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -892,7 +876,8 @@ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ) \ .filter( or_( *user_clause_list ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -937,7 +922,8 @@ for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -979,7 +965,8 @@ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ) \ .filter( or_( *user_clause_list ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -1056,7 +1043,8 @@ for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_failing_tool_tests( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_that_has_failing_tool_tests( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -1098,7 +1086,8 @@ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ) \ .filter( or_( *user_clause_list ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_failing_tool_tests( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_that_has_failing_tool_tests( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -1144,7 +1133,8 @@ for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_no_failing_tool_tests( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_that_has_no_failing_tool_tests( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -1187,7 +1177,8 @@ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ) \ .filter( or_( *user_clause_list ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_no_failing_tool_tests( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_that_has_no_failing_tool_tests( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -1216,7 +1207,8 @@ # At the time this grid is displayed we know that the received repository will have invalid tools in its latest changeset revision # that has associated metadata. val = '' - repository_metadata = get_latest_repository_metadata_if_it_includes_invalid_tools( trans, repository ) + repository_metadata = \ + grids_util.get_latest_repository_metadata_if_it_includes_invalid_tools( trans, repository ) metadata = repository_metadata.metadata invalid_tools = metadata.get( 'invalid_tools', [] ) if invalid_tools: @@ -1251,7 +1243,8 @@ for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ): - changeset_revision = filter_by_latest_metadata_changeset_revision_that_has_invalid_tools( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_metadata_changeset_revision_that_has_invalid_tools( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -1293,7 +1286,8 @@ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ) \ .filter( or_( *user_clause_list ) ): - changeset_revision = filter_by_latest_metadata_changeset_revision_that_has_invalid_tools( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_metadata_changeset_revision_that_has_invalid_tools( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -1478,16 +1472,21 @@ required_repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) if required_repository and not required_repository.deleted: required_repository_id = trans.security.encode_id( required_repository.id ) - required_repository_metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, - required_repository_id, - changeset_revision ) + required_repository_metadata = \ + metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, + required_repository_id, + changeset_revision ) if not required_repository_metadata: repo_dir = required_repository.repo_path( trans.app ) repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) - updated_changeset_revision = suc.get_next_downloadable_changeset_revision( required_repository, repo, changeset_revision ) - required_repository_metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, - required_repository_id, - updated_changeset_revision ) + updated_changeset_revision = \ + suc.get_next_downloadable_changeset_revision( required_repository, + repo, + changeset_revision ) + required_repository_metadata = \ + metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, + required_repository_id, + updated_changeset_revision ) required_repository_metadata_id = trans.security.encode_id( required_repository_metadata.id ) rd_str += '<a href="browse_repository_dependencies?operation=view_or_manage_repository&id=%s">' % ( required_repository_metadata_id ) rd_str += 'Repository <b>%s</b> revision <b>%s</b> owned by <b>%s</b>' % ( escape_html( rd_tup[ 1 ] ), escape_html( rd_tup[ 3 ] ), escape_html( rd_tup[ 2 ] ) ) @@ -1741,31 +1740,17 @@ class RepositoriesColumn( grids.TextColumn ): def get_value( self, trans, grid, category ): - # TODO: We should probably keep in in-memory register for speed improvements. - if category.repositories: - viewable_repositories = 0 - for rca in category.repositories: - repository = rca.repository - filter = trans.app.repository_grid_filter_manager.get_filter( trans ) - if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: - if not repository.deprecated and repository.downloadable_revisions: - is_level_one_certified_tuple = metadata_util.is_level_one_certified( trans, repository ) - latest_installable_changeset_revision, is_level_one_certified = is_level_one_certified_tuple - if is_level_one_certified: - viewable_repositories += 1 - if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: - if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and \ - not repository.deprecated and repository.downloadable_revisions: - is_level_one_certified_tuple = metadata_util.is_level_one_certified( trans, repository ) - latest_installable_changeset_revision, is_level_one_certified = is_level_one_certified_tuple - if is_level_one_certified: - viewable_repositories += 1 - else: - # The value of filter is None. - if not repository.deleted and not repository.deprecated and repository.downloadable_revisions: - viewable_repositories += 1 - return viewable_repositories - return 0 + category_name = str( category.name ) + filter = trans.app.repository_grid_filter_manager.get_filter( trans ) + if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: + return trans.app.repository_registry.certified_level_one_viewable_repositories_and_suites_by_category.get( category_name, 0 ) + elif filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: + return trans.app.repository_registry.certified_level_one_viewable_suites_by_category.get( category_name, 0 ) + elif filter == trans.app.repository_grid_filter_manager.filters.SUITES: + return trans.app.repository_registry.viewable_valid_suites_by_category.get( category_name, 0 ) + else: + # The value filter is None. + return trans.app.repository_registry.viewable_valid_repositories_and_suites_by_category.get( category_name, 0 ) title = "Categories of Valid Repositories" model_class = model.Category @@ -1869,21 +1854,19 @@ # The user is browsing categories of valid repositories, so filter the request by the received id, # which is a category id. if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: - clause_list = get_certified_level_one_clause_list( trans ) return trans.sa_session.query( model.Repository ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .join( model.RepositoryCategoryAssociation.table ) \ .join( model.Category.table ) \ .filter( and_( model.Category.table.c.id == trans.security.decode_id( kwd[ 'id' ] ), model.RepositoryMetadata.table.c.downloadable == True ) ) if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: - clause_list = get_certified_level_one_clause_list( trans ) return trans.sa_session.query( model.Repository ) \ .filter( model.Repository.type == rt_util.REPOSITORY_SUITE_DEFINITION ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .join( model.RepositoryCategoryAssociation.table ) \ .join( model.Category.table ) \ @@ -1902,20 +1885,18 @@ model.RepositoryMetadata.table.c.downloadable == True ) ) # The user performed a free text search on the ValidCategoryGrid. if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: - clause_list = get_certified_level_one_clause_list( trans ) return trans.sa_session.query( model.Repository ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) \ .filter( model.RepositoryMetadata.table.c.downloadable == True ) if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: - clause_list = get_certified_level_one_clause_list( trans ) return trans.sa_session.query( model.Repository ) \ .filter( model.Repository.type == rt_util.REPOSITORY_SUITE_DEFINITION ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) \ @@ -1930,206 +1911,3 @@ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) \ .filter( model.RepositoryMetadata.table.c.downloadable == True ) - -# ------ utility methods ------------------- - -def filter_by_latest_downloadable_changeset_revision_that_has_failing_tool_tests( trans, repository ): - """ - Inspect the latest downloadable changeset revision for the received repository to see if it - includes at least 1 tool that has at least 1 failing test. This will filter out repositories - of type repository_suite_definition and tool_dependency_definition. - """ - repository_metadata = get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ) - if repository_metadata is not None \ - and has_been_tested( repository_metadata ) \ - and not repository_metadata.missing_test_components \ - and not repository_metadata.tools_functionally_correct \ - and not repository_metadata.test_install_error: - return repository_metadata.changeset_revision - return None - -def filter_by_latest_downloadable_changeset_revision_that_has_missing_tool_test_components( trans, repository ): - """ - Inspect the latest downloadable changeset revision for the received repository to see if it - includes tools that are either missing functional tests or functional test data. If the - changset revision includes tools but is missing tool test components, return the changeset - revision hash. This will filter out repositories of type repository_suite_definition and - tool_dependency_definition. - """ - repository_metadata = get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ) - if repository_metadata is not None \ - and has_been_tested( repository_metadata ) \ - and repository_metadata.missing_test_components: - return repository_metadata.changeset_revision - return None - -def filter_by_latest_downloadable_changeset_revision_that_has_no_failing_tool_tests( trans, repository ): - """ - Inspect the latest downloadable changeset revision for the received repository to see if it - includes tools with no failing tests. This will filter out repositories of type repository_suite_definition - and tool_dependency_definition. - """ - repository_metadata = get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ) - if repository_metadata is not None \ - and has_been_tested( repository_metadata ) \ - and not repository_metadata.missing_test_components \ - and repository_metadata.tools_functionally_correct: - return repository_metadata.changeset_revision - return None - -def filter_by_latest_metadata_changeset_revision_that_has_invalid_tools( trans, repository ): - """ - Inspect the latest changeset revision with associated metadata for the received repository - to see if it has invalid tools. This will filter out repositories of type repository_suite_definition - and tool_dependency_definition. - """ - repository_metadata = get_latest_repository_metadata_if_it_includes_invalid_tools( trans, repository ) - if repository_metadata is not None: - return repository_metadata.changeset_revision - return None - -def filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ): - """ - Inspect the latest downloadable changeset revision for the received repository to see if - it has tool test installation errors. This will return repositories of type unrestricted - as well as types repository_suite_definition and tool_dependency_definition. - """ - repository_metadata = get_latest_downloadable_repository_metadata_if_it_has_test_install_errors( trans, repository ) - # Filter further by eliminating repositories that are missing test components. - if repository_metadata is not None \ - and has_been_tested( repository_metadata ) \ - and not repository_metadata.missing_test_components: - return repository_metadata.changeset_revision - return None - -def filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ): - """ - Inspect the latest downloadable changeset revision for the received repository to see if skip tests - is checked. This will return repositories of type unrestricted as well as types repository_suite_definition - and tool_dependency_definition. - """ - repository_metadata = get_latest_downloadable_repository_metadata( trans, repository ) - # The skip_tool_tests attribute is a SkipToolTest table mapping backref to the RepositoryMetadata table. - if repository_metadata is not None and repository_metadata.skip_tool_tests: - return repository_metadata.changeset_revision - return None - -def get_certified_level_one_clause_list( trans ): - clause_list = [] - for repository in trans.sa_session.query( model.Repository ) \ - .filter( and_( model.Repository.table.c.deleted == False, - model.Repository.table.c.deprecated == False ) ): - is_level_one_certified_tuple = metadata_util.is_level_one_certified( trans, repository ) - latest_installable_changeset_revision, is_level_one_certified = is_level_one_certified_tuple - if is_level_one_certified: - clause_list.append( "%s=%d and %s='%s'" % ( model.RepositoryMetadata.table.c.repository_id, - repository.id, - model.RepositoryMetadata.table.c.changeset_revision, - latest_installable_changeset_revision ) ) - return clause_list - -def get_latest_downloadable_repository_metadata( trans, repository ): - """ - Return the latest downloadable repository_metadata record for the received repository. This will - return repositories of type unrestricted as well as types repository_suite_definition and - tool_dependency_definition. - """ - encoded_repository_id = trans.security.encode_id( repository.id ) - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) - tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) - repository_metadata = None - try: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, tip_ctx ) - if repository_metadata is not None and repository_metadata.downloadable: - return repository_metadata - return None - except: - latest_downloadable_revision = suc.get_previous_metadata_changeset_revision( repository, repo, tip_ctx, downloadable=True ) - if latest_downloadable_revision == suc.INITIAL_CHANGELOG_HASH: - return None - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, - encoded_repository_id, - latest_downloadable_revision ) - if repository_metadata is not None and repository_metadata.downloadable: - return repository_metadata - return None - -def get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ): - """ - Return the latest downloadable repository_metadata record for the received repository if its - includes_tools attribute is True. This will filter out repositories of type repository_suite_definition - and tool_dependency_definition. - """ - repository_metadata = get_latest_downloadable_repository_metadata( trans, repository ) - if repository_metadata is not None and repository_metadata.includes_tools: - return repository_metadata - return None - -def get_latest_downloadable_repository_metadata_if_it_has_test_install_errors( trans, repository ): - """ - Return the latest downloadable repository_metadata record for the received repository if its - test_install_error attribute is True. This will return repositories of type unrestricted as - well as types repository_suite_definition and tool_dependency_definition. - """ - repository_metadata = get_latest_downloadable_repository_metadata( trans, repository ) - if repository_metadata is not None \ - and has_been_tested( repository_metadata ) \ - and repository_metadata.test_install_error: - return repository_metadata - return None - -def get_latest_repository_metadata( trans, repository ): - """ - Return the latest repository_metadata record for the received repository if it exists. This will - return repositories of type unrestricted as well as types repository_suite_definition and - tool_dependency_definition. - """ - encoded_repository_id = trans.security.encode_id( repository.id ) - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) - tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) - try: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, tip_ctx ) - return repository_metadata - except: - latest_downloadable_revision = suc.get_previous_metadata_changeset_revision( repository, repo, tip_ctx, downloadable=False ) - if latest_downloadable_revision == suc.INITIAL_CHANGELOG_HASH: - return None - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, - encoded_repository_id, - latest_downloadable_revision ) - return repository_metadata - -def get_latest_repository_metadata_if_it_includes_invalid_tools( trans, repository ): - """ - Return the latest repository_metadata record for the received repository that contains invalid - tools if one exists. This will filter out repositories of type repository_suite_definition and - tool_dependency_definition. - """ - repository_metadata = get_latest_repository_metadata( trans, repository ) - if repository_metadata is not None: - metadata = repository_metadata.metadata - if metadata is not None and 'invalid_tools' in metadata: - return repository_metadata - return None - -def has_been_tested( repository_metadata ): - """ - Return True if the received repository_metadata record'd tool_test_results column was populated by - the Tool Shed's install and test framework. - """ - tool_test_results = repository_metadata.tool_test_results - if tool_test_results is None: - return False - # The install and test framework's preparation scripts will populate the tool_test_results column - # with something like this: - # [{"test_environment": - # {"time_tested": "2014-05-15 16:15:18", - # "tool_shed_database_version": 22, - # "tool_shed_mercurial_version": "2.2.3", - # "tool_shed_revision": "13459:9a1415f8108f"} - # }] - tool_test_results = listify( tool_test_results ) - for test_results_dict in tool_test_results: - if len( test_results_dict ) > 1: - return True - return False diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/grids/util.py --- a/lib/tool_shed/grids/util.py +++ b/lib/tool_shed/grids/util.py @@ -1,53 +1,13 @@ import logging import os -import tool_shed.util.shed_util_common as suc + +from galaxy.util import listify +from galaxy.web.form_builder import SelectField + from tool_shed.util import hg_util -from galaxy.web.form_builder import SelectField -from galaxy.util.bunch import Bunch log = logging.getLogger( __name__ ) - -class RepositoryGridFilterManager( object ): - """Provides filtered views of the many Tool SHed repository grids.""" - - filters = Bunch( CERTIFIED_LEVEL_ONE = 'certified_level_one', - CERTIFIED_LEVEL_TWO = 'certified_level_two', - CERTIFIED_LEVEL_ONE_SUITES = 'certified_level_one_suites', - CERTIFIED_LEVEL_TWO_SUITES = 'certified_level_two_suites' ) - - def get_grid_title( self, trans, trailing_string='', default='' ): - filter = self.get_filter( trans ) - if filter == self.filters.CERTIFIED_LEVEL_ONE: - return "Certified 1 Repositories %s" % trailing_string - if filter == self.filters.CERTIFIED_LEVEL_TWO: - return "Certified 2 Repositories %s" % trailing_string - if filter == self.filters.CERTIFIED_LEVEL_ONE_SUITES: - return "Certified 1 Repository Suites %s" % trailing_string - if filter == self.filters.CERTIFIED_LEVEL_TWO_SUITES: - return "Certified 2 Repository Suites %s" % trailing_string - return "%s" % default - - def get_filter( self, trans ): - filter = trans.get_cookie( name='toolshedrepogridfilter' ) - return filter or None - - def is_valid_filter( self, filter ): - if filter is None: - return True - for valid_key, valid_filter in self.filters.items(): - if filter == valid_filter: - return True - return False - - def set_filter( self, trans, **kwd ): - # Set a session cookie value with the selected filter. - filter = kwd.get( 'filter', None ) - if filter is not None and self.is_valid_filter( filter ): - trans.set_cookie( value=filter, name='toolshedrepogridfilter' ) - # if the filter is not valid, expire the cookie. - trans.set_cookie( value=filter,name='toolshedrepogridfilter', age=-1 ) - def build_approved_select_field( trans, name, selected_value=None, for_component=True ): options = [ ( 'No', trans.model.ComponentReview.approved_states.NO ), ( 'Yes', trans.model.ComponentReview.approved_states.YES ) ] @@ -119,3 +79,190 @@ selected = selected_value and option_tup[ 1 ] == selected_value select_field.add_option( option_tup[ 0 ], option_tup[ 1 ], selected=selected ) return select_field + +def filter_by_latest_downloadable_changeset_revision_that_has_failing_tool_tests( trans, repository ): + """ + Inspect the latest downloadable changeset revision for the received repository to see if it + includes at least 1 tool that has at least 1 failing test. This will filter out repositories + of type repository_suite_definition and tool_dependency_definition. + """ + repository_metadata = get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ) + if repository_metadata is not None \ + and has_been_tested( repository_metadata ) \ + and not repository_metadata.missing_test_components \ + and not repository_metadata.tools_functionally_correct \ + and not repository_metadata.test_install_error: + return repository_metadata.changeset_revision + return None + +def filter_by_latest_downloadable_changeset_revision_that_has_missing_tool_test_components( trans, repository ): + """ + Inspect the latest downloadable changeset revision for the received repository to see if it + includes tools that are either missing functional tests or functional test data. If the + changset revision includes tools but is missing tool test components, return the changeset + revision hash. This will filter out repositories of type repository_suite_definition and + tool_dependency_definition. + """ + repository_metadata = get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ) + if repository_metadata is not None \ + and has_been_tested( repository_metadata ) \ + and repository_metadata.missing_test_components: + return repository_metadata.changeset_revision + return None + +def filter_by_latest_downloadable_changeset_revision_that_has_no_failing_tool_tests( trans, repository ): + """ + Inspect the latest downloadable changeset revision for the received repository to see if it + includes tools with no failing tests. This will filter out repositories of type repository_suite_definition + and tool_dependency_definition. + """ + repository_metadata = get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ) + if repository_metadata is not None \ + and has_been_tested( repository_metadata ) \ + and not repository_metadata.missing_test_components \ + and repository_metadata.tools_functionally_correct: + return repository_metadata.changeset_revision + return None + +def filter_by_latest_metadata_changeset_revision_that_has_invalid_tools( trans, repository ): + """ + Inspect the latest changeset revision with associated metadata for the received repository + to see if it has invalid tools. This will filter out repositories of type repository_suite_definition + and tool_dependency_definition. + """ + repository_metadata = get_latest_repository_metadata_if_it_includes_invalid_tools( trans, repository ) + if repository_metadata is not None: + return repository_metadata.changeset_revision + return None + +def filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ): + """ + Inspect the latest downloadable changeset revision for the received repository to see if + it has tool test installation errors. This will return repositories of type unrestricted + as well as types repository_suite_definition and tool_dependency_definition. + """ + repository_metadata = get_latest_downloadable_repository_metadata_if_it_has_test_install_errors( trans, repository ) + # Filter further by eliminating repositories that are missing test components. + if repository_metadata is not None \ + and has_been_tested( repository_metadata ) \ + and not repository_metadata.missing_test_components: + return repository_metadata.changeset_revision + return None + +def filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ): + """ + Inspect the latest downloadable changeset revision for the received repository to see if skip tests + is checked. This will return repositories of type unrestricted as well as types repository_suite_definition + and tool_dependency_definition. + """ + repository_metadata = get_latest_downloadable_repository_metadata( trans, repository ) + # The skip_tool_tests attribute is a SkipToolTest table mapping backref to the RepositoryMetadata table. + if repository_metadata is not None and repository_metadata.skip_tool_tests: + return repository_metadata.changeset_revision + return None + +def get_latest_downloadable_repository_metadata( trans, repository ): + """ + Return the latest downloadable repository_metadata record for the received repository. This will + return repositories of type unrestricted as well as types repository_suite_definition and + tool_dependency_definition. + """ + encoded_repository_id = trans.security.encode_id( repository.id ) + repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) + repository_metadata = None + try: + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, tip_ctx ) + if repository_metadata is not None and repository_metadata.downloadable: + return repository_metadata + return None + except: + latest_downloadable_revision = suc.get_previous_metadata_changeset_revision( repository, repo, tip_ctx, downloadable=True ) + if latest_downloadable_revision == suc.INITIAL_CHANGELOG_HASH: + return None + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, + encoded_repository_id, + latest_downloadable_revision ) + if repository_metadata is not None and repository_metadata.downloadable: + return repository_metadata + return None + +def get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ): + """ + Return the latest downloadable repository_metadata record for the received repository if its + includes_tools attribute is True. This will filter out repositories of type repository_suite_definition + and tool_dependency_definition. + """ + repository_metadata = get_latest_downloadable_repository_metadata( trans, repository ) + if repository_metadata is not None and repository_metadata.includes_tools: + return repository_metadata + return None + +def get_latest_downloadable_repository_metadata_if_it_has_test_install_errors( trans, repository ): + """ + Return the latest downloadable repository_metadata record for the received repository if its + test_install_error attribute is True. This will return repositories of type unrestricted as + well as types repository_suite_definition and tool_dependency_definition. + """ + repository_metadata = get_latest_downloadable_repository_metadata( trans, repository ) + if repository_metadata is not None \ + and has_been_tested( repository_metadata ) \ + and repository_metadata.test_install_error: + return repository_metadata + return None + +def get_latest_repository_metadata( trans, repository ): + """ + Return the latest repository_metadata record for the received repository if it exists. This will + return repositories of type unrestricted as well as types repository_suite_definition and + tool_dependency_definition. + """ + encoded_repository_id = trans.security.encode_id( repository.id ) + repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) + try: + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, tip_ctx ) + return repository_metadata + except: + latest_downloadable_revision = suc.get_previous_metadata_changeset_revision( repository, repo, tip_ctx, downloadable=False ) + if latest_downloadable_revision == suc.INITIAL_CHANGELOG_HASH: + return None + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, + encoded_repository_id, + latest_downloadable_revision ) + return repository_metadata + +def get_latest_repository_metadata_if_it_includes_invalid_tools( trans, repository ): + """ + Return the latest repository_metadata record for the received repository that contains invalid + tools if one exists. This will filter out repositories of type repository_suite_definition and + tool_dependency_definition. + """ + repository_metadata = get_latest_repository_metadata( trans, repository ) + if repository_metadata is not None: + metadata = repository_metadata.metadata + if metadata is not None and 'invalid_tools' in metadata: + return repository_metadata + return None + +def has_been_tested( repository_metadata ): + """ + Return True if the received repository_metadata record'd tool_test_results column was populated by + the Tool Shed's install and test framework. + """ + tool_test_results = repository_metadata.tool_test_results + if tool_test_results is None: + return False + # The install and test framework's preparation scripts will populate the tool_test_results column + # with something like this: + # [{"test_environment": + # {"time_tested": "2014-05-15 16:15:18", + # "tool_shed_database_version": 22, + # "tool_shed_mercurial_version": "2.2.3", + # "tool_shed_revision": "13459:9a1415f8108f"} + # }] + tool_test_results = listify( tool_test_results ) + for test_results_dict in tool_test_results: + if len( test_results_dict ) > 1: + return True + return False diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/repository_registry.py --- /dev/null +++ b/lib/tool_shed/repository_registry.py @@ -0,0 +1,202 @@ +import os +import logging + +import tool_shed.repository_types.util as rt_util + +from galaxy.model.orm import and_ +from galaxy.model.orm import or_ +from galaxy.webapps.tool_shed import model + +from tool_shed.util import hg_util +from tool_shed.util import metadata_util +from tool_shed.util import shed_util_common as suc + +log = logging.getLogger( __name__ ) + + +class Registry( object ): + + def __init__( self, app ): + self.app = app + self.certified_level_one_clause_list = self.get_certified_level_one_clause_list() + # The following lists contain tuples like ( repository.name, repository.user.username, changeset_revision ) + # where the changeset_revision entry is always the latest installable changeset_revision.. + self.certified_level_one_repository_and_suite_tuples = [] + self.certified_level_one_suite_tuples = [] + # These category dictionaries contain entries where the key is the category and the value is the integer count + # of viewable repositories within that category. + self.certified_level_one_viewable_repositories_and_suites_by_category = {} + self.certified_level_one_viewable_suites_by_category = {} + self.certified_level_two_repository_and_suite_tuples = [] + self.certified_level_two_suite_tuples = [] + self.certified_level_two_viewable_repositories_and_suites_by_category = {} + self.certified_level_two_viewable_suites_by_category = {} + self.repository_and_suite_tuples = [] + self.suite_tuples = [] + self.viewable_repositories_and_suites_by_category = {} + self.viewable_suites_by_category = {} + self.viewable_valid_repositories_and_suites_by_category = {} + self.viewable_valid_suites_by_category = {} + self.load_viewable_repositories_and_suites_by_category() + self.load_repository_and_suite_tuples() + + def get_certified_level_one_clause_list( self ): + certified_level_one_tuples = [] + clause_list = [] + for repository in self.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deleted == False, + model.Repository.table.c.deprecated == False ) ): + certified_level_one_tuple = self.get_certified_level_one_tuple( repository ) + latest_installable_changeset_revision, is_level_one_certified = certified_level_one_tuple + if is_level_one_certified: + certified_level_one_tuples.append( certified_level_one_tuple ) + clause_list.append( "%s=%d and %s='%s'" % ( model.RepositoryMetadata.table.c.repository_id, + repository.id, + model.RepositoryMetadata.table.c.changeset_revision, + latest_installable_changeset_revision ) ) + return clause_list + + def get_certified_level_one_tuple( self, repository ): + """ + Return True if the latest installable changeset_revision of the received repository is level one certified. + """ + if repository.deleted: + return ( None, False ) + repo = hg_util.get_repo_for_repository( self.app, repository ) + # Get the latest installable changeset revision since that is all that is currently configured for testing. + latest_installable_changeset_revision = suc.get_latest_downloadable_changeset_revision( self.app, repository, repo ) + if latest_installable_changeset_revision not in [ None, suc.INITIAL_CHANGELOG_HASH ]: + encoded_repository_id = self.app.security.encode_id( repository.id ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, + encoded_repository_id, + latest_installable_changeset_revision ) + if repository_metadata: + # Filter out repository revisions that have not been tested. + if repository_metadata.time_last_tested is not None and repository_metadata.tool_test_results is not None: + if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION ]: + # Look in the tool_test_results dictionary for installation errors. + try: + tool_test_results_dict = repository_metadata.tool_test_results[ 0 ] + except Exception, e: + message = 'Error attempting to retrieve install and test results for repository %s:\n' % str( repository.name ) + message += '%s' % str( e ) + log.exception( message ) + return ( latest_installable_changeset_revision, False ) + if 'installation_errors' in tool_test_results_dict: + return ( latest_installable_changeset_revision, False ) + return ( latest_installable_changeset_revision, True ) + else: + # We have a repository with type Unrestricted. + if repository_metadata.includes_tools: + if repository_metadata.tools_functionally_correct: + return ( latest_installable_changeset_revision, True ) + return ( latest_installable_changeset_revision, False ) + else: + # Look in the tool_test_results dictionary for installation errors. + try: + tool_test_results_dict = repository_metadata.tool_test_results[ 0 ] + except Exception, e: + message = 'Error attempting to retrieve install and test results for repository %s:\n' % str( repository.name ) + message += '%s' % str( e ) + log.exception( message ) + return ( latest_installable_changeset_revision, False ) + if 'installation_errors' in tool_test_results_dict: + return ( latest_installable_changeset_revision, False ) + return ( latest_installable_changeset_revision, True ) + else: + # No test results. + return ( latest_installable_changeset_revision, False ) + else: + # No repository_metadata. + return ( latest_installable_changeset_revision, False ) + else: + # No installable changeset_revision. + return ( None, False ) + + def is_level_one_certified( self, repository_metadata ): + repository = repository_metadata.repository + if repository.deprecated or repository.deleted: + return False + tuple = ( str( repository.name ), str( repository.user.username ), str( repository_metadata.changeset_revision ) ) + if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION ]: + return tuple in self.certified_level_one_suite_tuples + else: + return tuple in self.certified_level_one_repository_and_suite_tuples + + def load_repository_and_suite_tuples( self ): + # Load self.certified_level_one_repository_and_suite_tuples and self.certified_level_one_suite_tuples. + for repository in self.sa_session.query( model.Repository ) \ + .join( model.RepositoryMetadata.table ) \ + .filter( or_( *self.certified_level_one_clause_list ) ) \ + .join( model.User.table ): + name = str( repository.name ) + owner = str( repository.user.username ) + tip_changeset_hash = repository.tip( self.app ) + certified_level_one_tuple = ( name, owner, tip_changeset_hash ) + if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: + self.certified_level_one_suite_tuples.append( certified_level_one_tuple ) + else: + self.certified_level_one_repository_and_suite_tuples.append( certified_level_one_tuple ) + # Load self.repository_and_suite_tuples and self.suite_tuples + for repository in self.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deleted == False, + model.Repository.table.c.deprecated == False ) ) \ + .join( model.User.table ): + name = str( repository.name ) + owner = str( repository.user.username ) + for repository_metadata in repository.metadata_revisions: + changeset_revision = str( repository_metadata.changeset_revision ) + tuple = ( name, owner, changeset_revision ) + self.repository_and_suite_tuples.append( tuple ) + if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: + self.suite_tuples.append( tuple ) + + def load_viewable_repositories_and_suites_by_category( self ): + # Clear all dictionaries just in case they were previously loaded. + self.certified_level_one_viewable_repositories_and_suites_by_category = {} + self.certified_level_one_viewable_suites_by_category = {} + self.certified_level_two_viewable_repositories_and_suites_by_category = {} + self.certified_level_two_viewable_suites_by_category = {} + self.viewable_repositories_and_suites_by_category = {} + self.viewable_suites_by_category = {} + self.viewable_valid_repositories_and_suites_by_category = {} + self.viewable_valid_suites_by_category = {} + for category in self.sa_session.query( model.Category ): + if category not in self.certified_level_one_viewable_repositories_and_suites_by_category: + self.certified_level_one_viewable_repositories_and_suites_by_category[ str( category.name ) ] = 0 + if category not in self.certified_level_one_viewable_suites_by_category: + self.certified_level_one_viewable_suites_by_category[ str( category.name ) ] = 0 + if category not in self.viewable_repositories_and_suites_by_category: + self.viewable_repositories_and_suites_by_category[ str( category.name ) ] = 0 + if category not in self.viewable_suites_by_category: + self.viewable_suites_by_category[ str( category.name ) ] = 0 + if category not in self.viewable_valid_repositories_and_suites_by_category: + self.viewable_valid_repositories_and_suites_by_category[ str( category.name ) ] = 0 + if category not in self.viewable_valid_suites_by_category: + self.viewable_valid_suites_by_category[ str( category.name ) ] = 0 + for rca in category.repositories: + repository = rca.repository + if repository.downloadable_revisions: + is_valid = True + else: + is_valid = False + encoded_repository_id = self.app.security.encode_id( repository.id ) + tip_changeset_hash = repository.tip( self.app ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, + encoded_repository_id, + tip_changeset_hash ) + self.viewable_repositories_and_suites_by_category[ str( category.name ) ] += 1 + if is_valid: + self.viewable_valid_repositories_and_suites_by_category[ str( category.name ) ] += 1 + if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION ]: + self.viewable_suites_by_category[ str( category.name ) ] += 1 + if is_valid: + self.viewable_valid_suites_by_category[ str( category.name ) ] += 1 + if self.is_level_one_certified( repository_metadata ): + self.certified_level_one_viewable_repositories_and_suites_by_category[ str( category.name ) ] += 1 + if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION ]: + self.certified_level_one_viewable_suites_by_category[ str( category.name ) ] += 1 + + @property + def sa_session( self ): + return self.app.model.context.current diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/repository_types/util.py --- a/lib/tool_shed/repository_types/util.py +++ b/lib/tool_shed/repository_types/util.py @@ -7,7 +7,7 @@ REPOSITORY_SUITE_DEFINITION = 'repository_suite_definition' TOOL_DEPENDENCY_DEFINITION = 'tool_dependency_definition' -types = [ UNRESTRICTED, TOOL_DEPENDENCY_DEFINITION ] +types = [ UNRESTRICTED, TOOL_DEPENDENCY_DEFINITION, REPOSITORY_SUITE_DEFINITION ] def build_repository_type_select_field( trans, repository=None, name='repository_type' ): """Called from the Tool Shed to generate the current list of supported repository types.""" diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/util/commit_util.py --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -344,7 +344,7 @@ if repository: repo_dir = repository.repo_path( trans.app ) repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) - lastest_installable_changeset_revision = suc.get_latest_downloadable_changeset_revision( trans, repository, repo ) + lastest_installable_changeset_revision = suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) if lastest_installable_changeset_revision != suc.INITIAL_CHANGELOG_HASH: elem.attrib[ 'changeset_revision' ] = lastest_installable_changeset_revision revised = True diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/util/export_util.py --- a/lib/tool_shed/util/export_util.py +++ b/lib/tool_shed/util/export_util.py @@ -223,7 +223,7 @@ received repository_id. """ repository = suc.get_repository_in_tool_shed( trans, repository_id ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) repository_dependencies = \ diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/util/hg_util.py --- a/lib/tool_shed/util/hg_util.py +++ b/lib/tool_shed/util/hg_util.py @@ -69,12 +69,15 @@ ctx_date = date.strftime( "%Y-%m-%d" ) return ctx_date +def get_repo_for_repository( app, repository ): + return hg.repository( get_configured_ui(), repository.repo_path( app ) ) + def get_revision_label( trans, repository, changeset_revision, include_date=True, include_hash=True ): """ Return a string consisting of the human read-able changeset rev and the changeset revision string which includes the revision date if the receive include_date is True. """ - repo = hg.repository( get_configured_ui(), repository.repo_path( trans.app ) ) + repo = get_repo_for_repository( trans.app, repository ) ctx = get_changectx_for_changeset( repo, changeset_revision ) if ctx: return get_revision_label_from_ctx( ctx, include_date=include_date, include_hash=include_hash ) diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/util/import_util.py --- a/lib/tool_shed/util/import_util.py +++ b/lib/tool_shed/util/import_util.py @@ -32,7 +32,7 @@ repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) # Do not allow the repository to be automatically installed if population resulted in errors. tip_changeset_revision = repository.tip( trans.app ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), tip_changeset_revision ) if repository_metadata: diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -307,7 +307,7 @@ downloadable = True else: downloadable = False - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata: # A repository metadata record already exists with the received changeset_revision, so we don't need to # check the skip_tool_test table. @@ -1048,17 +1048,17 @@ repository = trans.sa_session.query( trans.model.Repository ).get( decoded_repository_id ) repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) if downloadable: - changeset_revision = suc.get_latest_downloadable_changeset_revision( trans, repository, repo ) + changeset_revision = suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) else: changeset_revision = suc.get_latest_changeset_revision( trans, repository, repo ) - return suc.get_repository_metadata_by_changeset_revision( trans, trans.security.encode_id( repository.id ), changeset_revision ) + return suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_revision ) def get_parent_id( trans, id, old_id, version, guid, changeset_revisions ): parent_id = None # Compare from most recent to oldest. changeset_revisions.reverse() for changeset_revision in changeset_revisions: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) metadata = repository_metadata.metadata tools_dicts = metadata.get( 'tools', [] ) for tool_dict in tools_dicts: @@ -1093,11 +1093,11 @@ def get_repository_metadata_by_repository_id_changeset_revision( trans, id, changeset_revision, metadata_only=False ): """Get a specified metadata record for a specified repository in the tool shed.""" if metadata_only: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata and repository_metadata.metadata: return repository_metadata.metadata return None - return suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + return suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) def get_repository_metadata_revisions_for_review( repository, reviewed=True ): repository_metadata_revisions = [] @@ -1337,63 +1337,6 @@ return True return False -def is_level_one_certified( trans, repository ): - """ - Return True if the latest installable changeset_revision of the received repository is level one certified. - """ - if repository.deleted: - return ( None, False ) - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) - # Get the latest installable changeset revision since that is all that is currently configured for testing. - latest_installable_changeset_revision = suc.get_latest_downloadable_changeset_revision( trans, repository, repo ) - if latest_installable_changeset_revision not in [ None, suc.INITIAL_CHANGELOG_HASH ]: - encoded_repository_id = trans.security.encode_id( repository.id ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, - encoded_repository_id, - latest_installable_changeset_revision ) - if repository_metadata: - # Filter out repository revisions that have not been tested. - if repository_metadata.time_last_tested is not None and repository_metadata.tool_test_results is not None: - if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION ]: - # Look in the tool_test_results dictionary for installation errors. - try: - tool_test_results_dict = repository_metadata.tool_test_results[ 0 ] - except Exception, e: - message = 'Error attempting to retrieve install and test results for repository %s:\n' % str( repository.name ) - message += '%s' % str( e ) - log.exception( message ) - return ( latest_installable_changeset_revision, False ) - if 'installation_errors' in tool_test_results_dict: - return ( latest_installable_changeset_revision, False ) - return ( latest_installable_changeset_revision, True ) - else: - # We have a repository with type Unrestricted. - if repository_metadata.includes_tools: - if repository_metadata.tools_functionally_correct: - return ( latest_installable_changeset_revision, True ) - return ( latest_installable_changeset_revision, False ) - else: - # Look in the tool_test_results dictionary for installation errors. - try: - tool_test_results_dict = repository_metadata.tool_test_results[ 0 ] - except Exception, e: - message = 'Error attempting to retrieve install and test results for repository %s:\n' % str( repository.name ) - message += '%s' % str( e ) - log.exception( message ) - return ( latest_installable_changeset_revision, False ) - if 'installation_errors' in tool_test_results_dict: - return ( latest_installable_changeset_revision, False ) - return ( latest_installable_changeset_revision, True ) - else: - # No test results. - return ( latest_installable_changeset_revision, False ) - else: - # No repository_metadata. - return ( latest_installable_changeset_revision, False ) - else: - # No installable changeset_revision. - return ( None, False ) - def new_datatypes_metadata_required( trans, repository_metadata, metadata_dict ): """ Compare the last saved metadata for each datatype in the repository with the new metadata in metadata_dict to determine if a new @@ -1746,7 +1689,7 @@ changeset_revisions_that_contain_tools = [] for changeset in repo.changelog: changeset_revision = str( repo.changectx( changeset ) ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata if metadata: @@ -1756,7 +1699,7 @@ # If a repository includes tools, build a dictionary of { 'tool id' : 'parent tool id' } pairs for each tool in each changeset revision. for index, changeset_revision in enumerate( changeset_revisions_that_contain_tools ): tool_versions_dict = {} - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) metadata = repository_metadata.metadata tool_dicts = metadata[ 'tools' ] if index == 0: @@ -2038,7 +1981,7 @@ changeset_revisions = [] for changeset in repo.changelog: changeset_revision = str( repo.changectx( changeset ) ) - if suc.get_repository_metadata_by_changeset_revision( trans, encoded_id, changeset_revision ): + if suc.get_repository_metadata_by_changeset_revision( trans.app, encoded_id, changeset_revision ): changeset_revisions.append( changeset_revision ) add_tool_versions( trans, encoded_id, repository_metadata, changeset_revisions ) elif len( repo ) == 1 and not invalid_file_tups: diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/util/readme_util.py --- a/lib/tool_shed/util/readme_util.py +++ b/lib/tool_shed/util/readme_util.py @@ -27,7 +27,7 @@ can_use_disk_files = True else: repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) - latest_downloadable_changeset_revision = suc.get_latest_downloadable_changeset_revision( trans, repository, repo ) + latest_downloadable_changeset_revision = suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) can_use_disk_files = changeset_revision == latest_downloadable_changeset_revision readme_files_dict = {} if metadata: diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -131,7 +131,7 @@ def changeset_is_malicious( trans, id, changeset_revision, **kwd ): """Check the malicious flag in repository metadata for a specified change set""" - repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata: return repository_metadata.malicious return False @@ -488,14 +488,14 @@ def get_current_repository_metadata_for_changeset_revision( trans, repository, changeset_revision ): encoded_repository_id = trans.security.encode_id( repository.id ) - repository_metadata = get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, changeset_revision ) + repository_metadata = get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, changeset_revision ) if repository_metadata: return repository_metadata # The installable changeset_revision may have been changed because it was "moved ahead" in the repository changelog. repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) updated_changeset_revision = get_next_downloadable_changeset_revision( repository, repo, after_changeset_revision=changeset_revision ) if updated_changeset_revision: - repository_metadata = get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, updated_changeset_revision ) + repository_metadata = get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, updated_changeset_revision ) if repository_metadata: return repository_metadata return None @@ -610,7 +610,7 @@ def get_latest_changeset_revision( trans, repository, repo ): repository_tip = repository.tip( trans.app ) - repository_metadata = get_repository_metadata_by_changeset_revision( trans, trans.security.encode_id( repository.id ), repository_tip ) + repository_metadata = get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), repository_tip ) if repository_metadata and repository_metadata.downloadable: return repository_tip changeset_revisions = get_ordered_metadata_changeset_revisions( repository, repo, downloadable=False ) @@ -618,9 +618,9 @@ return changeset_revisions[ -1 ] return INITIAL_CHANGELOG_HASH -def get_latest_downloadable_changeset_revision( trans, repository, repo ): - repository_tip = repository.tip( trans.app ) - repository_metadata = get_repository_metadata_by_changeset_revision( trans, trans.security.encode_id( repository.id ), repository_tip ) +def get_latest_downloadable_changeset_revision( app, repository, repo ): + repository_tip = repository.tip( app ) + repository_metadata = get_repository_metadata_by_changeset_revision( app, app.security.encode_id( repository.id ), repository_tip ) if repository_metadata and repository_metadata.downloadable: return repository_tip changeset_revisions = get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ) @@ -1076,21 +1076,22 @@ """Get a repository on the tool shed side from the database via id.""" return trans.sa_session.query( trans.model.Repository ).get( trans.security.decode_id( id ) ) -def get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ): +def get_repository_metadata_by_changeset_revision( app, id, changeset_revision ): """Get metadata for a specified repository change set from the database.""" # Make sure there are no duplicate records, and return the single unique record for the changeset_revision. # Duplicate records were somehow created in the past. The cause of this issue has been resolved, but we'll # leave this method as is for a while longer to ensure all duplicate records are removed. - all_metadata_records = trans.sa_session.query( trans.model.RepositoryMetadata ) \ - .filter( and_( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ), - trans.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) ) \ - .order_by( trans.model.RepositoryMetadata.table.c.update_time.desc() ) \ - .all() + sa_session = app.model.context.current + all_metadata_records = sa_session.query( app.model.RepositoryMetadata ) \ + .filter( and_( app.model.RepositoryMetadata.table.c.repository_id == app.security.decode_id( id ), + app.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) ) \ + .order_by( app.model.RepositoryMetadata.table.c.update_time.desc() ) \ + .all() if len( all_metadata_records ) > 1: # Delete all records older than the last one updated. for repository_metadata in all_metadata_records[ 1: ]: - trans.sa_session.delete( repository_metadata ) - trans.sa_session.flush() + sa_session.delete( repository_metadata ) + sa_session.flush() return all_metadata_records[ 0 ] elif all_metadata_records: return all_metadata_records[ 0 ] diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -546,7 +546,7 @@ current_child_guid = guid for changeset in suc.reversed_upper_bounded_changelog( repo, repository_metadata.changeset_revision ): ctx = repo.changectx( changeset ) - rm = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, str( ctx ) ) + rm = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, str( ctx ) ) if rm: parent_guid = rm.tool_versions.get( current_child_guid, None ) if parent_guid: @@ -558,7 +558,7 @@ repository_metadata.changeset_revision, repository.tip( trans.app ) ): ctx = repo.changectx( changeset ) - rm = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, str( ctx ) ) + rm = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, str( ctx ) ) if rm: tool_versions = rm.tool_versions for child_guid, parent_guid in tool_versions.items(): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.