1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/153f7f450f4b/ changeset: 153f7f450f4b user: greg date: 2013-03-01 17:13:15 summary: Add RepositoryMetadata attributes that enable improved query abilities for tool shed repository revisions that contain Galaxy utilities. affected #: 6 files diff -r 21c69586390abeefde4a52560321b826473ad041 -r 153f7f450f4b7537bc8866f09b5cc3ca57ad25ee lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -33,6 +33,7 @@ class NameColumn( grids.TextColumn ): + def get_value( self, trans, grid, category ): return category.name @@ -43,6 +44,7 @@ class RepositoriesColumn( grids.TextColumn ): + def get_value( self, trans, grid, category ): if category.repositories: viewable_repositories = 0 @@ -82,6 +84,7 @@ class RepositoriesColumn( grids.TextColumn ): + def get_value( self, trans, grid, category ): if category.repositories: viewable_repositories = 0 @@ -123,6 +126,7 @@ class NameColumn( grids.TextColumn ): + def get_value( self, trans, grid, repository ): return escape_html( repository.name ) @@ -468,6 +472,7 @@ class CategoryColumn( grids.TextColumn ): + def get_value( self, trans, grid, repository ): rval = '<ul>' if repository.categories: @@ -481,6 +486,7 @@ class RepositoryCategoryColumn( grids.GridColumn ): + def filter( self, trans, user, query, column_filter ): """Modify query to filter by category.""" if column_filter == "All": @@ -501,6 +507,7 @@ elif len( select_field.options ) == 1: return select_field.options[ 0 ][ 0 ] return '' + title = "Valid repositories" columns = [ RepositoryGrid.NameColumn( "Name", @@ -633,19 +640,171 @@ attach_popup=False ) +class RepositoryMetadataGrid( grids.Grid ): + + + class RepositoryNameColumn( grids.TextColumn ): + + def get_value( self, trans, grid, repository_metadata ): + return escape_html( repository_metadata.repository.name ) + + + class RepositoryOwnerColumn( grids.TextColumn ): + + def get_value( self, trans, grid, repository_metadata ): + return escape_html( repository_metadata.user.username ) + + + class ChangesetRevisionColumn( grids.TextColumn ): + + def get_value( self, trans, grid, repository_metadata ): + return escape_html( repository_metadata.changeset_revision ) + + + class MaliciousColumn( grids.BooleanColumn ): + def get_value( self, trans, grid, repository_metadata ): + if repository_metadata.malicious: + return 'yes' + return '' + + + class DownloadableColumn( grids.BooleanColumn ): + def get_value( self, trans, grid, repository_metadata ): + if repository_metadata.downloadable: + return 'yes' + return '' + + + class ToolsFunctionallyCorrectColumn( grids.BooleanColumn ): + def get_value( self, trans, grid, repository_metadata ): + if repository_metadata.tools_functionally_correct: + return 'yes' + return '' + + + class DoNotTestColumn( grids.BooleanColumn ): + def get_value( self, trans, grid, repository_metadata ): + if repository_metadata.do_not_test: + return 'yes' + return '' + + + class TimeLastTestedColumn( grids.DateTimeColumn ): + def get_value( self, trans, grid, repository_metadata ): + return repository_metadata.time_last_tested + + + class HasRepositoryDependenciesColumn( grids.BooleanColumn ): + def get_value( self, trans, grid, repository_metadata ): + if repository_metadata.has_repository_dependencies: + return 'yes' + return '' + + + class IncludesDatatypesColumn( grids.BooleanColumn ): + def get_value( self, trans, grid, repository_metadata ): + if repository_metadata.includes_datatypes: + return 'yes' + return '' + + + class IncludesToolsColumn( grids.BooleanColumn ): + def get_value( self, trans, grid, repository_metadata ): + if repository_metadata.includes_tools: + return 'yes' + return '' + + + class IncludesToolDependenciesColumn( grids.BooleanColumn ): + def get_value( self, trans, grid, repository_metadata ): + if repository_metadata.includes_tool_dependencies: + return 'yes' + return '' + + + class IncludesWorkflowsColumn( grids.BooleanColumn ): + def get_value( self, trans, grid, repository_metadata ): + if repository_metadata.includes_workflows: + return 'yes' + return '' + + title = "Repository metadata" + model_class = model.RepositoryMetadata + template='/webapps/tool_shed/repository/grid.mako' + default_sort_key = "Repository.name" + columns = [ + RepositoryNameColumn( "Repository name", + link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), + attach_popup=False ), + RepositoryOwnerColumn( "Owner", + model_class=model.User, + attach_popup=False, + key="User.username" ) + ] + columns.append( grids.MulticolFilterColumn( "Search repository name, description", + cols_to_filter=[ columns[0], columns[1] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + operations = [] + standard_filters = [] + default_filter = dict( malicious="False" ) + num_rows_per_page = 50 + preserve_state = False + use_paging = True + + def build_initial_query( self, trans, **kwd ): + return trans.sa_session.query( model.RepositoryMetadata ) \ + .join( model.Repository ) \ + .filter( and_( model.Repository.table.c.deleted == False, + model.Repository.table.c.deprecated == False ) ) \ + .join( model.User.table ) \ + .order_by( model.Repository.name ) + + +class RepositoryDependenciesGrid( RepositoryMetadataGrid ): + + title = "Repository dependencies available in this Tool Shed" + columns = [ + RepositoryMetadataGrid.RepositoryNameColumn( "Repository name", + link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), + attach_popup=False ), + RepositoryMetadataGrid.RepositoryOwnerColumn( "Owner", + model_class=model.User, + attach_popup=False, + key="User.username" ) + ] + columns.append( grids.MulticolFilterColumn( "Search repository name, description", + cols_to_filter=[ columns[0], columns[1] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + + def build_initial_query( self, trans, **kwd ): + return trans.sa_session.query( model.RepositoryMetadata ) \ + .join( model.Repository ) \ + .filter( and_( model.RepositoryMetadata.table.c.has_repository_dependencies == True, + model.Repository.table.c.deleted == False, + model.Repository.table.c.deprecated == False ) ) \ + .join( model.User.table ) \ + .order_by( model.Repository.name ) + + class RepositoryController( BaseUIController, common_util.ItemRatings ): + category_grid = CategoryGrid() + deprecated_repositories_i_own_grid = DeprecatedRepositoriesIOwnGrid() + email_alerts_repository_grid = EmailAlertsRepositoryGrid() install_matched_repository_grid = InstallMatchedRepositoryGrid() matched_repository_grid = MatchedRepositoryGrid() + my_writable_repositories_grid = MyWritableRepositoriesGrid() + repositories_by_user_grid = RepositoriesByUserGrid() + repositories_i_own_grid = RepositoriesIOwnGrid() + repository_grid = RepositoryGrid() + # The repository_metadata_grid is not currently displayed, but is sub-classed by several grids. + repository_metadata_grid = RepositoryMetadataGrid() + valid_category_grid = ValidCategoryGrid() valid_repository_grid = ValidRepositoryGrid() - repository_grid = RepositoryGrid() - email_alerts_repository_grid = EmailAlertsRepositoryGrid() - category_grid = CategoryGrid() - valid_category_grid = ValidCategoryGrid() - my_writable_repositories_grid = MyWritableRepositoriesGrid() - repositories_i_own_grid = RepositoriesIOwnGrid() - deprecated_repositories_i_own_grid = DeprecatedRepositoriesIOwnGrid() - repositories_by_user_grid = RepositoriesByUserGrid() @web.expose def browse_categories( self, trans, **kwd ): @@ -3036,7 +3195,11 @@ repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) if repository_metadata: repository_metadata_id = trans.security.encode_id( repository_metadata.id ) - tool_test_errors = json.from_json_string( repository_metadata.tool_test_errors ) + # TODO: Fix this when the install and test framework is completed. + # if repository_metadata.tool_test_errors: + # tool_test_errors = json.from_json_string( repository_metadata.tool_test_errors ) + # else: + # tool_test_errors = None metadata = repository_metadata.metadata if metadata: if 'tools' in metadata: @@ -3070,7 +3233,7 @@ else: repository_metadata_id = None metadata = None - tool_test_errors = None + #tool_test_errors = None is_malicious = suc.changeset_is_malicious( trans, repository_id, repository.tip( trans.app ) ) changeset_revision_select_field = build_changeset_revision_select_field( trans, repository, @@ -3094,7 +3257,7 @@ tool=tool, tool_metadata_dict=tool_metadata_dict, tool_lineage=tool_lineage, - tool_test_errors=tool_test_errors, + #tool_test_errors=tool_test_errors, changeset_revision=changeset_revision, revision_label=revision_label, changeset_revision_select_field=changeset_revision_select_field, diff -r 21c69586390abeefde4a52560321b826473ad041 -r 153f7f450f4b7537bc8866f09b5cc3ca57ad25ee lib/galaxy/webapps/tool_shed/model/__init__.py --- a/lib/galaxy/webapps/tool_shed/model/__init__.py +++ b/lib/galaxy/webapps/tool_shed/model/__init__.py @@ -191,11 +191,14 @@ fp.close() class RepositoryMetadata( object, APIItem ): - api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable' ) + api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'has_repository_dependencies', 'includes_datatypes', + 'includes_tools', 'includes_tool_dependencies', 'includes_workflows' ) api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'tools_functionally_correct', - 'do_not_test', 'time_last_tested', 'tool_test_errors' ) + 'do_not_test', 'time_last_tested', 'tool_test_errors', 'has_repository_dependencies', 'includes_datatypes', 'includes_tools', + 'includes_tool_dependencies', 'includes_workflows' ) def __init__( self, id=None, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False, downloadable=False, - tools_functionally_correct=False, do_not_test=False, time_last_tested=None, tool_test_errors=None ): + tools_functionally_correct=False, do_not_test=False, time_last_tested=None, tool_test_errors=None, has_repository_dependencies=False, + includes_datatypes=False, includes_tools=False, includes_tool_dependencies=False, includes_workflows=False ): self.id = id self.repository_id = repository_id self.changeset_revision = changeset_revision @@ -207,6 +210,11 @@ self.do_not_test = do_not_test self.time_last_tested = time_last_tested self.tool_test_errors = tool_test_errors + self.has_repository_dependencies = has_repository_dependencies + self.includes_datatypes = includes_datatypes + self.includes_tools = includes_tools + self.includes_tool_dependencies = includes_tool_dependencies + self.includes_workflows = includes_workflows def as_dict( self, value_mapper=None ): return self.get_api_value( view='element', value_mapper=value_mapper ) def get_api_value( self, view='collection', value_mapper=None ): @@ -227,8 +235,8 @@ return rval class RepositoryReview( object, APIItem ): - api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating' ) - api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating' ) + api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating', 'deleted' ) + api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating', 'deleted' ) approved_states = Bunch( NO='no', YES='yes' ) def __init__( self, repository_id=None, changeset_revision=None, user_id=None, rating=None, deleted=False ): self.repository_id = repository_id diff -r 21c69586390abeefde4a52560321b826473ad041 -r 153f7f450f4b7537bc8866f09b5cc3ca57ad25ee lib/galaxy/webapps/tool_shed/model/mapping.py --- a/lib/galaxy/webapps/tool_shed/model/mapping.py +++ b/lib/galaxy/webapps/tool_shed/model/mapping.py @@ -134,7 +134,12 @@ Column( "tools_functionally_correct", Boolean, default=False, index=True ), Column( "do_not_test", Boolean, default=False, index=True ), Column( "time_last_tested", DateTime, default=None, nullable=True ), - Column( "tool_test_errors", JSONType, nullable=True ) ) + Column( "tool_test_errors", JSONType, nullable=True ), + Column( "has_repository_dependencies", Boolean, default=False, index=True ), + Column( "includes_datatypes", Boolean, default=False, index=True ), + Column( "includes_tools", Boolean, default=False, index=True ), + Column( "includes_tool_dependencies", Boolean, default=False, index=True ), + Column( "includes_workflows", Boolean, default=False, index=True ) ) RepositoryReview.table = Table( "repository_review", metadata, Column( "id", Integer, primary_key=True ), diff -r 21c69586390abeefde4a52560321b826473ad041 -r 153f7f450f4b7537bc8866f09b5cc3ca57ad25ee lib/galaxy/webapps/tool_shed/model/migrate/versions/0017_add_galaxy_utility_columns_to_repository_metadata_table.py --- /dev/null +++ b/lib/galaxy/webapps/tool_shed/model/migrate/versions/0017_add_galaxy_utility_columns_to_repository_metadata_table.py @@ -0,0 +1,115 @@ +""" +Migration script to add the includes_datatypes, has_repository_dependencies, includes_tools, includes_tool_dependencies and includes_workflows +columns to the repository_metadata table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + # Initialize. + if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + default_false = "0" + elif migrate_engine.name == 'postgres': + default_false = "false" + # Create and initialize tools_functionally_correct, do_not_test, time_last_tested, and tool_test_errors columns in repository_metadata table. + RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True ) + + # Create tools_functionally_correct column + c = Column( "includes_datatypes", Boolean, default=False, index=True ) + try: + c.create( RepositoryMetadata_table ) + assert c is RepositoryMetadata_table.c.includes_datatypes + db_session.execute( "UPDATE repository_metadata SET includes_datatypes=%s" % default_false ) + except Exception, e: + print "Adding includes_datatypes column to the repository_metadata table failed: %s" % str( e ) + + # Create includes_datatypes column + c = Column( "has_repository_dependencies", Boolean, default=False, index=True ) + try: + c.create( RepositoryMetadata_table ) + assert c is RepositoryMetadata_table.c.has_repository_dependencies + db_session.execute( "UPDATE repository_metadata SET has_repository_dependencies=%s" % default_false ) + except Exception, e: + print "Adding has_repository_dependencies column to the repository_metadata table failed: %s" % str( e ) + + # Create includes_tools column + c = Column( "includes_tools", Boolean, default=False, index=True ) + try: + c.create( RepositoryMetadata_table ) + assert c is RepositoryMetadata_table.c.includes_tools + db_session.execute( "UPDATE repository_metadata SET includes_tools=%s" % default_false ) + except Exception, e: + print "Adding includes_tools column to the repository_metadata table failed: %s" % str( e ) + + # Create includes_tool_dependencies column + c = Column( "includes_tool_dependencies", Boolean, default=False, index=True ) + try: + c.create( RepositoryMetadata_table ) + assert c is RepositoryMetadata_table.c.includes_tool_dependencies + db_session.execute( "UPDATE repository_metadata SET includes_tool_dependencies=%s" % default_false ) + except Exception, e: + print "Adding includes_tool_dependencies column to the repository_metadata table failed: %s" % str( e ) + + # Create includes_workflows column + c = Column( "includes_workflows", Boolean, default=False, index=True ) + try: + c.create( RepositoryMetadata_table ) + assert c is RepositoryMetadata_table.c.includes_workflows + db_session.execute( "UPDATE repository_metadata SET includes_workflows=%s" % default_false ) + except Exception, e: + print "Adding includes_workflows column to the repository_metadata table failed: %s" % str( e ) + +def downgrade(): + metadata.reflect() + # Drop tool_test_errors, time_last_tested, do_not_test, and tools_functionally_correct columns from repository_metadata table. + RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True ) + + # Drop the includes_workflows column. + try: + RepositoryMetadata_table.c.includes_workflows.drop() + except Exception, e: + print "Dropping column includes_workflows from the repository_metadata table failed: %s" % str( e ) + + # Drop the includes_tool_dependencies column. + try: + RepositoryMetadata_table.c.includes_tool_dependencies.drop() + except Exception, e: + print "Dropping column includes_tool_dependencies from the repository_metadata table failed: %s" % str( e ) + + # Drop the includes_tools column. + try: + RepositoryMetadata_table.c.includes_tools.drop() + except Exception, e: + print "Dropping column includes_tools from the repository_metadata table failed: %s" % str( e ) + + # Drop the has_repository_dependencies column. + try: + RepositoryMetadata_table.c.has_repository_dependencies.drop() + except Exception, e: + print "Dropping column has_repository_dependencies from the repository_metadata table failed: %s" % str( e ) + + # Drop the includes_datatypes column. + try: + RepositoryMetadata_table.c.includes_datatypes.drop() + except Exception, e: + print "Dropping column includes_datatypes from the repository_metadata table failed: %s" % str( e ) diff -r 21c69586390abeefde4a52560321b826473ad041 -r 153f7f450f4b7537bc8866f09b5cc3ca57ad25ee lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -900,16 +900,42 @@ shutil.copy( full_source_path, os.path.join( dest_path, copied_file ) ) def create_or_update_repository_metadata( trans, id, repository, changeset_revision, metadata_dict ): """Create or update a repository_metadatqa record in the tool shed.""" - downloadable = is_downloadable( metadata_dict ) + has_repository_dependencies = False + includes_datatypes = False + includes_tools = False + includes_tool_dependencies = False + includes_workflows = False + if metadata_dict: + if 'repository_dependencies' in metadata_dict: + has_repository_dependencies = True + if 'datatypes' in metadata_dict: + includes_datatypes = True + if 'tools' in metadata_dict: + includes_tools = True + if 'tool_dependencies' in metadata_dict: + includes_tool_dependencies = True + if 'workflows' in metadata_dict: + includes_workflows = True + downloadable = has_repository_dependencies or includes_datatypes or includes_tools or includes_tool_dependencies or includes_workflows repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) if repository_metadata: repository_metadata.metadata = metadata_dict repository_metadata.downloadable = downloadable + repository_metadata.has_repository_dependencies = has_repository_dependencies + repository_metadata.includes_datatypes = includes_datatypes + repository_metadata.includes_tools = includes_tools + repository_metadata.includes_tool_dependencies = includes_tool_dependencies + repository_metadata.includes_workflows = includes_workflows else: repository_metadata = trans.model.RepositoryMetadata( repository_id=repository.id, changeset_revision=changeset_revision, metadata=metadata_dict, - downloadable=downloadable ) + downloadable=downloadable, + has_repository_dependencies=has_repository_dependencies, + includes_datatypes=includes_datatypes, + includes_tools=includes_tools, + includes_tool_dependencies=includes_tool_dependencies, + includes_workflows=includes_workflows ) # Always set the default values for the following columns. When resetting all metadata on a repository, this will reset the values. repository_metadata.tools_functionally_correct = False repository_metadata.do_not_test = False diff -r 21c69586390abeefde4a52560321b826473ad041 -r 153f7f450f4b7537bc8866f09b5cc3ca57ad25ee templates/webapps/tool_shed/repository/view_tool_metadata.mako --- a/templates/webapps/tool_shed/repository/view_tool_metadata.mako +++ b/templates/webapps/tool_shed/repository/view_tool_metadata.mako @@ -22,6 +22,8 @@ can_review_repository = has_metadata and not is_deprecated and trans.app.security_agent.user_can_review_repositories( trans.user ) can_upload = can_push can_view_change_log = trans.webapp.name == 'tool_shed' and not is_new + # TODO: fix the following when the install and test buildbot is functional. + #can_view_tool_test_errors = tool_test_errors is not None if can_push: browse_label = 'Browse or delete repository tip files' @@ -265,11 +267,26 @@ ${tool.force_history_refresh | h} <div style="clear: both"></div></div> - <div class="form-row"> - <label>Parallelism:</label> - ${tool.parallelism | h} - <div style="clear: both"></div> - </div> + <% parallelism_info = tool.parallelism %> + %if parallelism_info: + <div class="form-row"> + <table width="100%"> + <tr bgcolor="#D8D8D8" width="100%"><td><b>Parallelism</td></tr> + </table> + </div> + <div class="form-row"> + <label>Method:</label> + ${parallelism_info.method | h} + <div style="clear: both"></div> + </div> + %for key, val in parallelism_info.attributes.items(): + <div class="form-row"> + <label>${key}:</label> + ${val | h} + <div style="clear: both"></div> + </div> + %endfor + %endif %endif <div class="form-row"><table width="100%"> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.