[galaxy-commits] commit/galaxy-central: greg: Tool shed database migration script to alter the repository_metadata table by dropping the tool_test_errors column and adding columns tool_test_results, missing_test_components.

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f11a2c7a7d32/ Changeset: f11a2c7a7d32 User: greg Date: 2013-04-25 15:36:52 Summary: Tool shed database migration script to alter the repository_metadata table by dropping the tool_test_errors column and adding columns tool_test_results, missing_test_components. Affected #: 17 files diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -42,6 +42,10 @@ tools_functionally_correct = kwd.get( 'tools_functionally_correct', None ) if tools_functionally_correct is not None: clause_list.append( trans.model.RepositoryMetadata.table.c.tools_functionally_correct == util.string_as_bool( tools_functionally_correct ) ) + # Filter by missing_test_components if received. + missing_test_components = kwd.get( 'missing_test_components', None ) + if missing_test_components is not None: + clause_list.append( trans.model.RepositoryMetadata.table.c.missing_test_components == util.string_as_bool( missing_test_components ) ) # Filter by do_not_test if received. do_not_test = kwd.get( 'do_not_test', None ) if do_not_test is not None: diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -738,12 +738,7 @@ """ The test framework in ~/test/install_and_test_tool_shed_repositories can be executed on a regularly defined schedule (e.g., via cron) to install appropriate repositories from a tool shed into a Galaxy instance and run defined functional tests for the tools included in the repository. This process affects the values - if these columns in the repository_metadata table: tools_functionally_correct, do_not_test, time_last_tested and tool_test_errors. The tool_test_errors is - slightly mis-named (it should have been named tool_test_results) it will contain a dictionary that includes information about the test environment even if all - tests passed and the tools_functionally_correct column is set to True. - The value of the tool_test_errors column will be a dictionary with the key / value pairs: - "test_environment", {"architecture": "i386", "python_version": "2.5.4", "system": "Darwin 10.8.0"} - "test_errors" [ { "test_id":<some test id>, "stdout":<stdout of running the test>, "stderr":<stderr of running the test>, "traceback":<traceback of running the test>] + if these columns in the repository_metadata table: do_not_test, missing_test_components, time_last_tested, tools_functionally_correct and tool_test_results. """ params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) @@ -1162,7 +1157,7 @@ trans.model.Repository.table.c.private == False, trans.model.Repository.table.c.deprecated == False, trans.model.Repository.table.c.user_id == user.id ) ): - if not metadata_row.tool_test_errors: + if not metadata_row.tool_test_results: continue # Per the RSS 2.0 specification, all dates in RSS feeds must be formatted as specified in RFC 822 # section 5.1, e.g. Sat, 07 Sep 2002 00:00:01 UT @@ -1171,9 +1166,9 @@ # Generate a citable URL for this repository with owner and changeset revision. repository_citable_url = suc.url_join( tool_shed_url, 'view', user.username, repository.name, metadata_row.changeset_revision ) title = 'Functional test results for changeset revision %s of %s' % ( metadata_row.changeset_revision, repository.name ) - tests_passed = len( metadata_row.tool_test_errors.get( 'tests_passed', [] ) ) - tests_failed = len( metadata_row.tool_test_errors.get( 'test_errors', [] ) ) - invalid_tests = len( metadata_row.tool_test_errors.get( 'invalid_tests', [] ) ) + tests_passed = len( metadata_row.tool_test_results.get( 'tests_passed', [] ) ) + tests_failed = len( metadata_row.tool_test_results.get( 'test_errors', [] ) ) + invalid_tests = len( metadata_row.tool_test_results.get( 'invalid_tests', [] ) ) description = '%d tests passed, %d tests failed, %d tests determined to be invalid.' % ( tests_passed, tests_failed, invalid_tests ) # The guid attribute in an RSS feed's list of items allows a feed reader to choose not to show an item as updated # if the guid is unchanged. For functional test results, the citable URL is sufficiently unique to enable @@ -2674,10 +2669,10 @@ if repository_metadata: repository_metadata_id = trans.security.encode_id( repository_metadata.id ) # TODO: Fix this when the install and test framework is completed. - # if repository_metadata.tool_test_errors: - # tool_test_errors = json.from_json_string( repository_metadata.tool_test_errors ) + # if repository_metadata.tool_test_results: + # tool_test_results = json.from_json_string( repository_metadata.tool_test_results ) # else: - # tool_test_errors = None + # tool_test_results = None metadata = repository_metadata.metadata if metadata: if 'tools' in metadata: @@ -2713,7 +2708,7 @@ else: repository_metadata_id = None metadata = None - #tool_test_errors = None + #tool_test_results = None is_malicious = suc.changeset_is_malicious( trans, repository_id, repository.tip( trans.app ) ) changeset_revision_select_field = grids_util.build_changeset_revision_select_field( trans, repository, @@ -2737,7 +2732,7 @@ tool=tool, tool_metadata_dict=tool_metadata_dict, tool_lineage=tool_lineage, - #tool_test_errors=tool_test_errors, + #tool_test_results=tool_test_results, changeset_revision=changeset_revision, revision_label=revision_label, changeset_revision_select_field=changeset_revision_select_field, diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/galaxy/webapps/tool_shed/controllers/repository_review.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py @@ -28,7 +28,7 @@ repositories_without_reviews_grid = repository_review_grids.RepositoriesWithoutReviewsGrid() repository_reviews_by_user_grid = repository_review_grids.RepositoryReviewsByUserGrid() reviewed_repositories_i_own_grid = repository_review_grids.ReviewedRepositoriesIOwnGrid() - repositories_with_invalid_tests_grid = repository_review_grids.RepositoriesWithInvalidTestsGrid() + repositories_with_no_tool_tests_grid = repository_review_grids.RepositoriesWithNoToolTestsGrid() @web.expose @web.require_login( "approve repository review" ) @@ -417,7 +417,10 @@ @web.expose @web.require_login( "manage repositories with invalid tests" ) def manage_repositories_with_invalid_tests( self, trans, **kwd ): - """Display a list of repositories that contain tools, have not yet been reviewed, and have invalid functional tests.""" + """ + Display a list of repositories that contain tools, have not yet been reviewed, and have invalid functional tests. Tests are defined as + invalid if they are missing from the tool config or if defined test data is not included in the repository. + """ if 'operation' in kwd: operation = kwd['operation'].lower() if operation == "inspect repository revisions": @@ -428,10 +431,10 @@ return trans.response.send_redirect( web.url_for( controller='repository_review', action='view_or_manage_repository', **kwd ) ) - message = 'These repositories contain tools with invalid functional tests (they have not yet been reviewed). ' + message = 'These repositories contain tools with missing functional tests or test data. ' kwd[ 'message' ] = message kwd[ 'status' ] = 'warning' - return self.repositories_with_invalid_tests_grid( trans, **kwd ) + return self.repositories_with_no_tool_tests_grid( trans, **kwd ) @web.expose @web.require_login( "manage repositories with reviews" ) diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/galaxy/webapps/tool_shed/model/__init__.py --- a/lib/galaxy/webapps/tool_shed/model/__init__.py +++ b/lib/galaxy/webapps/tool_shed/model/__init__.py @@ -1,28 +1,27 @@ -""" -Galaxy Tool Shed data model classes - -Naming: try to use class names that have a distinct plural form so that -the relationship cardinalities are obvious (e.g. prefer Dataset to Data) -""" -import os.path, os, errno, sys, codecs, operator, logging, tarfile, mimetypes, ConfigParser +import logging +import operator +import os from galaxy import util from galaxy.util.bunch import Bunch from galaxy.util.hash_util import new_secure_hash -from galaxy.web.form_builder import * from galaxy.model.item_attrs import APIItem from galaxy import eggs -eggs.require('mercurial') -from mercurial import hg, ui +eggs.require( 'mercurial' ) +from mercurial import hg +from mercurial import ui log = logging.getLogger( __name__ ) + class APIKeys( object ): pass + class User( object, APIItem ): api_collection_visible_keys = ( 'id', 'email' ) api_element_visible_keys = ( 'id', 'email', 'username' ) + def __init__( self, email=None, password=None ): self.email = email self.password = password @@ -31,6 +30,7 @@ self.purged = False self.username = None self.new_repo_alert = False + def all_roles( self ): roles = [ ura.role for ura in self.roles ] for group in [ uga.group for uga in self.groups ]: @@ -38,28 +38,37 @@ if role not in roles: roles.append( role ) return roles + def set_password_cleartext( self, cleartext ): """Set 'self.password' to the digest of 'cleartext'.""" self.password = new_secure_hash( text_type=cleartext ) + def check_password( self, cleartext ): """Check if 'cleartext' matches 'self.password' when hashed.""" return self.password == new_secure_hash( text_type=cleartext ) + def get_disk_usage( self, nice_size=False ): return 0 + def set_disk_usage( self, bytes ): pass + total_disk_usage = property( get_disk_usage, set_disk_usage ) + @property def nice_total_disk_usage( self ): return 0 + class Group( object, APIItem ): api_collection_visible_keys = ( 'id', 'name' ) api_element_visible_keys = ( 'id', 'name' ) + def __init__( self, name = None ): self.name = name self.deleted = False + class Role( object, APIItem ): api_collection_visible_keys = ( 'id', 'name' ) api_element_visible_keys = ( 'id', 'name', 'description', 'type' ) @@ -71,28 +80,34 @@ ADMIN = 'admin', SHARING = 'sharing' ) + def __init__( self, name="", description="", type="system", deleted=False ): self.name = name self.description = description self.type = type self.deleted = deleted + class UserGroupAssociation( object ): def __init__( self, user, group ): self.user = user self.group = group + class UserRoleAssociation( object ): def __init__( self, user, role ): self.user = user self.role = role + class GroupRoleAssociation( object ): def __init__( self, group, role ): self.group = group self.role = role + class GalaxySession( object ): + def __init__( self, id=None, user=None, @@ -113,6 +128,7 @@ self.is_valid = is_valid self.prev_session_id = prev_session_id + class Repository( object, APIItem ): api_collection_visible_keys = ( 'id', 'name', 'description', 'user_id', 'private', 'deleted', 'times_downloaded', 'deprecated' ) api_element_visible_keys = ( 'id', 'name', 'description', 'long_description', 'user_id', 'private', 'deleted', 'times_downloaded', 'deprecated' ) @@ -121,6 +137,7 @@ MARKED_FOR_REMOVAL = 'r', MARKED_FOR_ADDITION = 'a', NOT_TRACKED = '?' ) + def __init__( self, id=None, name=None, description=None, long_description=None, user_id=None, private=False, deleted=None, email_alerts=None, times_downloaded=0, deprecated=False ): self.id = id @@ -133,8 +150,10 @@ self.email_alerts = email_alerts self.times_downloaded = times_downloaded self.deprecated = deprecated + def as_dict( self, value_mapper=None ): return self.get_api_value( view='element', value_mapper=value_mapper ) + def get_api_value( self, view='collection', value_mapper=None ): if value_mapper is None: value_mapper = {} @@ -153,22 +172,28 @@ if 'user_id' in rval: rval[ 'owner' ] = self.user.username return rval + def repo_path( self, app ): return app.hgweb_config_manager.get_entry( os.path.join( "repos", self.user.username, self.name ) ) + def revision( self, app ): repo = hg.repository( ui.ui(), self.repo_path( app ) ) tip_ctx = repo.changectx( repo.changelog.tip() ) return "%s:%s" % ( str( tip_ctx.rev() ), str( repo.changectx( repo.changelog.tip() ) ) ) + def tip( self, app ): repo = hg.repository( ui.ui(), self.repo_path( app ) ) return str( repo.changectx( repo.changelog.tip() ) ) + def is_new( self, app ): repo = hg.repository( ui.ui(), self.repo_path( app ) ) tip_ctx = repo.changectx( repo.changelog.tip() ) return tip_ctx.rev() < 0 + def allow_push( self, app ): repo = hg.repository( ui.ui(), self.repo_path( app ) ) return repo.ui.config( 'web', 'allow_push' ) + def set_allow_push( self, app, usernames, remove_auth='' ): allow_push = util.listify( self.allow_push( app ) ) if remove_auth: @@ -190,15 +215,17 @@ fp.write( line ) fp.close() + class RepositoryMetadata( object, APIItem ): api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'has_repository_dependencies', 'includes_datatypes', 'includes_tools', 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'tools_functionally_correct', - 'do_not_test', 'time_last_tested', 'tool_test_errors', 'has_repository_dependencies', 'includes_datatypes', 'includes_tools', + 'do_not_test', 'time_last_tested', 'tool_test_results', 'has_repository_dependencies', 'includes_datatypes', 'includes_tools', 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) + def __init__( self, id=None, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False, downloadable=False, - tools_functionally_correct=False, do_not_test=False, time_last_tested=None, tool_test_errors=None, has_repository_dependencies=False, - includes_datatypes=False, includes_tools=False, includes_tool_dependencies=False, includes_workflows=False ): + missing_test_components=None, tools_functionally_correct=False, do_not_test=False, time_last_tested=None, tool_test_results=None, + has_repository_dependencies=False, includes_datatypes=False, includes_tools=False, includes_tool_dependencies=False, includes_workflows=False ): self.id = id self.repository_id = repository_id self.changeset_revision = changeset_revision @@ -206,15 +233,17 @@ self.tool_versions = tool_versions or dict() self.malicious = malicious self.downloadable = downloadable + self.missing_test_components = missing_test_components self.tools_functionally_correct = tools_functionally_correct self.do_not_test = do_not_test self.time_last_tested = time_last_tested - self.tool_test_errors = tool_test_errors + self.tool_test_results = tool_test_results self.has_repository_dependencies = has_repository_dependencies self.includes_datatypes = includes_datatypes self.includes_tools = includes_tools self.includes_tool_dependencies = includes_tool_dependencies self.includes_workflows = includes_workflows + @property def includes_tools_for_display_in_tool_panel( self ): if self.metadata: @@ -223,8 +252,10 @@ if tool_dict.get( 'add_to_tool_panel', True ): return True return False + def as_dict( self, value_mapper=None ): return self.get_api_value( view='element', value_mapper=value_mapper ) + def get_api_value( self, view='collection', value_mapper=None ): if value_mapper is None: value_mapper = {} @@ -242,10 +273,12 @@ rval[ key ] = None return rval + class RepositoryReview( object, APIItem ): api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating', 'deleted' ) api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating', 'deleted' ) approved_states = Bunch( NO='no', YES='yes' ) + def __init__( self, repository_id=None, changeset_revision=None, user_id=None, rating=None, deleted=False ): self.repository_id = repository_id self.changeset_revision = changeset_revision @@ -257,6 +290,7 @@ api_collection_visible_keys = ( 'id', 'repository_review_id', 'component_id', 'private', 'approved', 'rating', 'deleted' ) api_element_visible_keys = ( 'id', 'repository_review_id', 'component_id', 'private', 'approved', 'rating', 'deleted' ) approved_states = Bunch( NO='no', YES='yes', NA='not_applicable' ) + def __init__( self, repository_review_id=None, component_id=None, comment=None, private=False, approved=False, rating=None, deleted=False ): self.repository_review_id = repository_review_id self.component_id = component_id @@ -266,49 +300,65 @@ self.rating = rating self.deleted = deleted + class Component( object ): + def __init__( self, name=None, description=None ): self.name = name self.description = description + class ItemRatingAssociation( object ): + def __init__( self, id=None, user=None, item=None, rating=0, comment='' ): self.id = id self.user = user self.item = item self.rating = rating self.comment = comment + def set_item( self, item ): """ Set association's item. """ pass + class RepositoryRatingAssociation( ItemRatingAssociation ): + def set_item( self, repository ): self.repository = repository + class Category( object, APIItem ): api_collection_visible_keys = ( 'id', 'name', 'description', 'deleted' ) api_element_visible_keys = ( 'id', 'name', 'description', 'deleted' ) + def __init__( self, name=None, description=None, deleted=False ): self.name = name self.description = description self.deleted = deleted + class RepositoryCategoryAssociation( object ): + def __init__( self, repository=None, category=None ): self.repository = repository self.category = category + class Tag( object ): + def __init__( self, id=None, type=None, parent_id=None, name=None ): self.id = id self.type = type self.parent_id = parent_id self.name = name + def __str__ ( self ): return "Tag(id=%s, type=%i, parent_id=%s, name=%s)" % ( self.id, self.type, self.parent_id, self.name ) + class ItemTagAssociation( object ): + def __init__( self, id=None, user=None, item_id=None, tag_id=None, user_tname=None, value=None ): self.id = id self.user = user @@ -318,7 +368,9 @@ self.value = None self.user_value = None + class Workflow( object ): + def __init__( self ): self.user = None self.name = None @@ -326,7 +378,9 @@ self.has_errors = None self.steps = [] + class WorkflowStep( object ): + def __init__( self ): self.id = None self.type = None @@ -336,17 +390,20 @@ self.tool_errors = None self.position = None self.input_connections = [] - #self.output_connections = [] self.config = None + class WorkflowStepConnection( object ): + def __init__( self ): self.output_step = None self.output_name = None self.input_step = None self.input_name = None + ## ---- Utility methods ------------------------------------------------------- + def sort_by_attr( seq, attr ): """ Sort the sequence of objects by object's attribute @@ -362,6 +419,7 @@ intermed = map( None, map( getattr, seq, ( attr, ) * len( seq ) ), xrange( len( seq ) ), seq ) intermed.sort() return map( operator.getitem, intermed, ( -1, ) * len( intermed ) ) + def directory_hash_id( id ): s = str( id ) l = len( s ) diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/galaxy/webapps/tool_shed/model/mapping.py --- a/lib/galaxy/webapps/tool_shed/model/mapping.py +++ b/lib/galaxy/webapps/tool_shed/model/mapping.py @@ -134,7 +134,8 @@ Column( "tools_functionally_correct", Boolean, default=False, index=True ), Column( "do_not_test", Boolean, default=False, index=True ), Column( "time_last_tested", DateTime, default=None, nullable=True ), - Column( "tool_test_errors", JSONType, nullable=True ), + Column( "missing_test_components", Boolean, default=False, index=True ), + Column( "tool_test_results", JSONType, nullable=True ), Column( "has_repository_dependencies", Boolean, default=False, index=True ), Column( "includes_datatypes", Boolean, default=False, index=True ), Column( "includes_tools", Boolean, default=False, index=True ), diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/galaxy/webapps/tool_shed/model/migrate/versions/0018_add_repository_metadata_flag_columns.py --- /dev/null +++ b/lib/galaxy/webapps/tool_shed/model/migrate/versions/0018_add_repository_metadata_flag_columns.py @@ -0,0 +1,90 @@ +""" +Migration script to alter the repository_metadata table by dropping the tool_test_errors column and adding columns +tool_test_results, missing_test_components. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + # Initialize. + if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + default_false = "0" + elif migrate_engine.name == 'postgres': + default_false = "false" + + try: + RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True ) + except NoSuchTableError: + RepositoryMetadata_table = None + log.debug( "Failed loading table repository_metadata." ) + + if RepositoryMetadata_table: + # Drop the tool_test_errors column from the repository_metadata table as it is poorly named. It will be replaced with the new + # tool_test_results column. + try: + col = RepositoryMetadata_table.c.tool_test_errors + col.drop() + except Exception, e: + log.debug( "Dropping column 'tool_test_errors' from repository_metadata table failed: %s" % ( str( e ) ) ) + + # Create the tool_test_results column to replace the ill-named tool_test_errors column just dropped above. + c = Column( "tool_test_results", JSONType, nullable=True ) + try: + c.create( RepositoryMetadata_table ) + assert c is RepositoryMetadata_table.c.tool_test_results + except Exception, e: + print "Adding tool_test_results column to the repository_metadata table failed: %s" % str( e ) + + # Create the missing_test_components column. + c = Column( "missing_test_components", Boolean, default=False, index=True ) + try: + c.create( RepositoryMetadata_table ) + assert c is RepositoryMetadata_table.c.missing_test_components + db_session.execute( "UPDATE repository_metadata SET missing_test_components=%s" % default_false ) + except Exception, e: + print "Adding missing_test_components column to the repository_metadata table failed: %s" % str( e ) + +def downgrade(): + metadata.reflect() + # Drop missing_test_components and tool_test_results from the repository_metadata table and add tool_test_errors to the repository_metadata table. + RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True ) + + # Drop the missing_test_components column. + try: + RepositoryMetadata_table.c.missing_test_components.drop() + except Exception, e: + print "Dropping column missing_test_components from the repository_metadata table failed: %s" % str( e ) + + # Drop the tool_test_results column. + try: + RepositoryMetadata_table.c.tool_test_results.drop() + except Exception, e: + print "Dropping column tool_test_results from the repository_metadata table failed: %s" % str( e ) + + # Create the tool_test_errors column. + c = Column( "tool_test_errors", JSONType, nullable=True ) + try: + c.create( RepositoryMetadata_table ) + assert c is RepositoryMetadata_table.c.tool_test_errors + except Exception, e: + print "Adding tool_test_errors column to the repository_metadata table failed: %s" % str( e ) diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/tool_shed/grids/repository_review_grids.py --- a/lib/tool_shed/grids/repository_review_grids.py +++ b/lib/tool_shed/grids/repository_review_grids.py @@ -395,12 +395,12 @@ .outerjoin( ( model.Component.table, model.Component.table.c.id == model.ComponentReview.table.c.component_id ) ) -class RepositoriesWithInvalidTestsGrid( RepositoriesWithoutReviewsGrid ): +class RepositoriesWithNoToolTestsGrid( RepositoriesWithoutReviewsGrid ): # Repositories that are ready for human review are those that either: # 1) Have no tools # 2) Have tools that have been proven to be functionally correct within Galaxy. # This grid filters out repositories that have been marked as either deprecated or deleted. - title = "Repositories that contain tools with invalid functional tests" + title = "Repositories that contain tools with no tests or test data" columns = [ RepositoriesWithoutReviewsGrid.NameColumn( "Repository name", key="name", @@ -428,8 +428,7 @@ def build_initial_query( self, trans, **kwd ): return trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deleted == False, - model.Repository.table.c.deprecated == False, - model.Repository.reviews == None ) ) \ + model.Repository.table.c.deprecated == False ) ) \ .join( model.RepositoryMetadata.table ) \ .filter( and_( model.RepositoryMetadata.table.c.downloadable == True, model.RepositoryMetadata.table.c.includes_tools == True, diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/tool_shed/scripts/api/tool_shed_repository_revision_update.py --- a/lib/tool_shed/scripts/api/tool_shed_repository_revision_update.py +++ b/lib/tool_shed/scripts/api/tool_shed_repository_revision_update.py @@ -21,14 +21,14 @@ for key, value in [ kwarg.split( '=', 1 ) for kwarg in sys.argv[ 3: ] ]: """ This example script will properly handle updating the value of one or more of the following RepositoryMetadata attributes: - tools_functionally_correct, do_not_test, tool_test_errors + tools_functionally_correct, do_not_test, tool_test_results """ if key in [ 'tools_functionally_correct', 'do_not_test' ]: if str( value ).lower() in [ 'true', 'yes', 'on' ]: new_value = True else: new_value = False - elif key in [ 'tool_test_errors' ]: + elif key in [ 'tool_test_results' ]: new_value = from_json_string( value ) else: new_value = str( value ) diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -117,7 +117,7 @@ and test repositories script to process. If the tested changeset revision does not have a test-data directory, this script will also mark the revision not to be tested. - If any error is encountered, the script will update the repository_metadata.tool_test_errors attribute following this structure: + If any error is encountered, the script will update the repository_metadata.tool_test_results attribute following this structure: { "test_environment": { @@ -178,8 +178,8 @@ for metadata_record in metadata_records_to_check: # Initialize the repository_status dict with the test environment, but leave the test_errors empty. repository_status = {} - if metadata_record.tool_test_errors: - repository_status = metadata_record.tool_test_errors + if metadata_record.tool_test_results: + repository_status = metadata_record.tool_test_results # Clear any old invalid tests for this metadata revision, since this could lead to duplication of invalid test rows, # or tests incorrectly labeled as invalid. repository_status[ 'invalid_tests' ] = [] @@ -271,7 +271,7 @@ problem_found = True test_errors = dict( tool_id=tool_id, tool_version=tool_version, tool_guid=tool_guid, reason_test_is_invalid=failure_reason ) - # The repository_metadata.tool_test_errors attribute should always have the following structure: + # The repository_metadata.tool_test_results attribute should always have the following structure: # { # "test_environment": # { @@ -348,7 +348,7 @@ if should_set_do_not_test_flag( app, metadata_record.repository, changeset_revision ): metadata_record.do_not_test = True metadata_record.tools_functionally_correct = False - metadata_record.tool_test_errors = repository_status + metadata_record.tool_test_results = repository_status metadata_record.time_last_tested = datetime.utcnow() app.sa_session.add( metadata_record ) app.sa_session.flush() diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -277,9 +277,10 @@ includes_workflows=includes_workflows ) # Always set the default values for the following columns. When resetting all metadata on a repository, this will reset the values. repository_metadata.tools_functionally_correct = False + repository_metadata.missing_test_components = False repository_metadata.do_not_test = False repository_metadata.time_last_tested = None - repository_metadata.tool_test_errors = None + repository_metadata.tool_test_results = None trans.sa_session.add( repository_metadata ) trans.sa_session.flush() return repository_metadata @@ -1722,7 +1723,8 @@ repository_metadata.do_not_test = False repository_metadata.time_last_tested = None repository_metadata.tools_functionally_correct = False - repository_metadata.tool_test_errors = None + repository_metadata.missing_test_components = False + repository_metadata.tool_test_results = None trans.sa_session.add( repository_metadata ) trans.sa_session.flush() else: diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 templates/webapps/tool_shed/admin/index.mako --- a/templates/webapps/tool_shed/admin/index.mako +++ b/templates/webapps/tool_shed/admin/index.mako @@ -76,9 +76,6 @@ <a target="galaxy_main" href="${h.url_for( controller='repository_review', action='manage_repositories_ready_for_review' )}">Repositories ready for review</a></div><div class="toolTitle"> - <a target="galaxy_main" href="${h.url_for( controller='repository_review', action='manage_repositories_with_invalid_tests' )}">Repositories with invalid tests</a> - </div> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository_review', action='manage_repositories_without_reviews' )}">All repositories with no reviews</a></div> %if trans.user.repository_reviews: @@ -94,6 +91,17 @@ </div></div></div> + <div class="toolSectionPad"></div> + <div class="toolSectionTitle"> + Reviewing Repositories With Tools + </div> + <div class="toolSectionBody"> + <div class="toolSectionBg"> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository_review', action='manage_repositories_with_invalid_tests' )}">Repositories missing tests or data</a> + </div> + </div> + </div> %endif <div class="toolSectionPad"></div><div class="toolSectionTitle"> diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 templates/webapps/tool_shed/index.mako --- a/templates/webapps/tool_shed/index.mako +++ b/templates/webapps/tool_shed/index.mako @@ -133,9 +133,6 @@ <a target="galaxy_main" href="${h.url_for( controller='repository_review', action='manage_repositories_ready_for_review' )}">Repositories ready for review</a></div><div class="toolTitle"> - <a target="galaxy_main" href="${h.url_for( controller='repository_review', action='manage_repositories_with_invalid_tests' )}">Repositories with invalid tests</a> - </div> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository_review', action='manage_repositories_without_reviews' )}">All repositories with no reviews</a></div> %if trans.user.repository_reviews: @@ -151,6 +148,17 @@ </div></div></div> + <div class="toolSectionPad"></div> + <div class="toolSectionTitle"> + Reviewing Repositories With Tools + </div> + <div class="toolSectionBody"> + <div class="toolSectionBg"> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository_review', action='manage_repositories_with_invalid_tests' )}">Repositories missing tests or data</a> + </div> + </div> + </div> %endif %else: <div class="toolSectionPad"></div> diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 templates/webapps/tool_shed/repository/display_tool_functional_test_results.mako --- a/templates/webapps/tool_shed/repository/display_tool_functional_test_results.mako +++ b/templates/webapps/tool_shed/repository/display_tool_functional_test_results.mako @@ -51,16 +51,16 @@ can_review_repository = has_metadata and not is_deprecated and trans.app.security_agent.user_can_review_repositories( trans.user ) can_upload = can_push can_view_change_log = trans.webapp.name == 'tool_shed' and not is_new - if repository_metadata.tool_test_errors: - # The tool_test_errors is mis-named (it should have been named tool_test_results) it will contain a dictionary that includes information - # about the test environment even if all tests passed and the repository_metadata.tools_functionally_correct column is set to True. - tool_test_errors = repository_metadata.tool_test_errors - test_environment_dict = tool_test_errors.get( 'test_environment', None ) - invalid_tests = tool_test_errors.get( 'invalid_tests', [] ) - test_errors = tool_test_errors.get( 'test_errors', [] ) - tests_passed = tool_test_errors.get( 'tests_passed', [] ) + if repository_metadata.tool_test_results: + # The tool_test_results will contain a dictionary that includes information about the test environment even if all tests passed and the + # repository_metadata.tools_functionally_correct column is set to True. + tool_test_results = repository_metadata.tool_test_results + test_environment_dict = tool_test_results.get( 'test_environment', None ) + invalid_tests = tool_test_results.get( 'invalid_tests', [] ) + test_errors = tool_test_results.get( 'test_errors', [] ) + tests_passed = tool_test_results.get( 'tests_passed', [] ) else: - tool_test_errors = None + tool_test_results = None test_environment_dict = {} invalid_tests = [] test_errors = [] @@ -144,7 +144,7 @@ <b>Repository name:</b><br/> ${repository.name} %endif -%if invalid_tests or tool_test_errors or tests_passed: +%if invalid_tests or tool_test_results or tests_passed: <p/><div class="toolForm"><div class="toolFormTitle">Tool functional test results</div> diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 templates/webapps/tool_shed/repository/manage_repository.mako --- a/templates/webapps/tool_shed/repository/manage_repository.mako +++ b/templates/webapps/tool_shed/repository/manage_repository.mako @@ -27,7 +27,7 @@ can_upload = can_push can_view_change_log = not is_new if repository_metadata: - if repository_metadata.includes_tools and repository_metadata.tool_test_errors is not None: + if repository_metadata.includes_tools and repository_metadata.tool_test_results is not None: can_display_tool_functional_test_results = True else: can_display_tool_functional_test_results = False diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 templates/webapps/tool_shed/repository/view_repository.mako --- a/templates/webapps/tool_shed/repository/view_repository.mako +++ b/templates/webapps/tool_shed/repository/view_repository.mako @@ -26,7 +26,7 @@ changeset_revision_is_repository_tip = changeset_revision == repository.tip( trans.app ) if repository_metadata: - if repository_metadata.includes_tools and repository_metadata.tool_test_errors is not None: + if repository_metadata.includes_tools and repository_metadata.tool_test_results is not None: can_display_tool_functional_test_results = True else: can_display_tool_functional_test_results = False diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 templates/webapps/tool_shed/repository/view_tool_metadata.mako --- a/templates/webapps/tool_shed/repository/view_tool_metadata.mako +++ b/templates/webapps/tool_shed/repository/view_tool_metadata.mako @@ -23,7 +23,7 @@ can_upload = can_push can_view_change_log = trans.webapp.name == 'tool_shed' and not is_new # TODO: fix the following when the install and test buildbot is functional. - #can_view_tool_test_errors = tool_test_errors is not None + #can_view_tool_test_results = tool_test_results is not None if can_push: browse_label = 'Browse or delete repository tip files' diff -r a599aba4d18c634c51d40c68c64148e45f4e083b -r f11a2c7a7d325deaf5cf2c2f05a513b1e1b4a2a6 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -265,13 +265,13 @@ tool_id = parts[ -2 ] return tool_id, tool_version -def get_tool_test_errors_from_api( tool_shed_url, metadata_revision_id ): +def get_tool_test_results_from_api( tool_shed_url, metadata_revision_id ): api_path = [ 'api', 'repository_revisions', metadata_revision_id ] api_url = get_api_url( base=tool_shed_url, parts=api_path ) repository_metadata = json_from_url( api_url ) - if repository_metadata[ 'tool_test_errors' ] is None: + if repository_metadata[ 'tool_test_results' ] is None: return {} - return repository_metadata[ 'tool_test_errors' ] + return repository_metadata[ 'tool_test_results' ] def json_from_url( url ): url_handle = urllib.urlopen( url ) @@ -290,7 +290,7 @@ else: params[ 'tools_functionally_correct' ] = 'false' params[ 'do_not_test' ] = 'false' - params[ 'tool_test_errors' ] = test_results_dict + params[ 'tool_test_results' ] = test_results_dict if '-info_only' in sys.argv: return {} else: @@ -583,7 +583,7 @@ log.debug( 'Installation of %s succeeded, running all defined functional tests.' % name ) # Generate the shed_tools_dict that specifies the location of test data contained within this repository. If the repository # does not have a test-data directory, this will return has_test_data = False, and we will set the do_not_test flag to True, - # and the tools_functionally_correct flag to False, as well as updating tool_test_errors. + # and the tools_functionally_correct flag to False, as well as updating tool_test_results. file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( dict() ) ) has_test_data, shed_tools_dict = parse_tool_panel_config( galaxy_shed_tool_conf_file, from_json_string( file( galaxy_shed_tools_dict, 'r' ).read() ) ) # The repository_status dict should always have the following structure: @@ -628,7 +628,7 @@ # }, # ] # } - repository_status = get_tool_test_errors_from_api( galaxy_tool_shed_url, metadata_revision_id ) + repository_status = get_tool_test_results_from_api( galaxy_tool_shed_url, metadata_revision_id ) if 'test_environment' not in repository_status: repository_status[ 'test_environment' ] = {} test_environment = get_test_environment( repository_status[ 'test_environment' ] ) @@ -731,7 +731,7 @@ test_status[ output_type ] = '\n'.join( tmp_output[ output_type ] ) repository_status[ 'test_errors' ].append( test_status ) # Call the register_test_result method, which executes a PUT request to the repository_revisions API controller with the outcome - # of the tests, and updates tool_test_errors with the relevant log data. + # of the tests, and updates tool_test_results with the relevant log data. # This also sets the do_not_test and tools_functionally correct flags to the appropriate values, and updates the time_last_tested # field to today's date. repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.