1 new changeset in galaxy-central: http://bitbucket.org/galaxy/galaxy-central/changeset/824c59db3c3a/ changeset: 824c59db3c3a user: greg date: 2011-08-15 22:12:12 summary: Add the ability for a tool shed admin to browse repository metadata, and implement fixes for setting repository metadata for repositories with multiple tools. affected #: 4 files (5.8 KB) --- a/lib/galaxy/webapps/community/controllers/admin.py Mon Aug 15 13:28:40 2011 -0400 +++ b/lib/galaxy/webapps/community/controllers/admin.py Mon Aug 15 16:12:12 2011 -0400 @@ -3,7 +3,7 @@ from galaxy.model.orm import * from galaxy.web.framework.helpers import time_ago, iff, grids from galaxy.util import inflector -from common import get_category, get_repository +from common import get_category, get_repository, get_repository_metadata_by_id from repository import RepositoryListGrid, CategoryListGrid import logging log = logging.getLogger( __name__ ) @@ -308,6 +308,65 @@ async_compatible=False ) ) standard_filters = [] +class RepositoryMetadataListGrid( grids.Grid ): + class IdColumn( grids.IntegerColumn ): + def get_value( self, trans, grid, repository_metadata ): + return repository_metadata.id + class RepositoryIdColumn( grids.IntegerColumn ): + def get_value( self, trans, grid, repository_metadata ): + return repository_metadata.repository_id + class ChangesetRevisionColumn( grids.TextColumn ): + def get_value( self, trans, grid, repository_metadata ): + return repository_metadata.changeset_revision + class MetadataColumn( grids.TextColumn ): + def get_value( self, trans, grid, repository_metadata ): + metadata_str = '' + if repository_metadata: + metadata = repository_metadata.metadata + if metadata: + if 'tools' in metadata: + metadata_str += '<b>Tools:</b><br/>' + for tool_metadata_dict in metadata[ 'tools' ]: + metadata_str += '%s <b>%s</b><br/>' % \ + ( tool_metadata_dict[ 'id' ], tool_metadata_dict[ 'version' ] ) + if 'workflows' in metadata: + metadata_str += '<b>Workflows:</b><br/>' + for workflow_metadata_dict in metadata[ 'workflows' ]: + metadata_str += '%s <b>%s</b><br/>' % \ + ( workflow_metadata_dict[ 'name' ], workflow_metadata_dict[ 'format-version' ] ) + return metadata_str + class MaliciousColumn( grids.BooleanColumn ): + def get_value( self, trans, grid, repository_metadata ): + return repository_metadata.malicious + # Grid definition + title = "Repository Metadata" + model_class = model.RepositoryMetadata + template='/webapps/community/repository/grid.mako' + default_sort_key = "repository_id" + columns = [ + IdColumn( "Id", + visible=False, + attach_popup=False ), + RepositoryIdColumn( "Repository Id", + key="repository_id", + attach_popup=False ), + ChangesetRevisionColumn( "Revision", + attach_popup=False ), + MetadataColumn( "Metadata", + attach_popup=False ), + MaliciousColumn( "Malicious", + attach_popup=False ) + ] + operations = [ grids.GridOperation( "Delete", + allow_multiple=True ) ] + standard_filters = [] + default_filter = {} + num_rows_per_page = 50 + preserve_state = False + use_paging = True + def build_initial_query( self, trans, **kwd ): + return trans.sa_session.query( self.model_class ) + class AdminController( BaseController, Admin ): user_list_grid = UserListGrid() @@ -315,9 +374,44 @@ group_list_grid = GroupListGrid() manage_category_list_grid = ManageCategoryListGrid() repository_list_grid = AdminRepositoryListGrid() + repository_metadata_list_grid = RepositoryMetadataListGrid() @web.expose @web.require_admin + def browse_repository_metadata( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd[ 'operation' ].lower() + if operation == "delete": + return self.delete_repository_metadata( trans, **kwd ) + # Render the list view + return self.repository_metadata_list_grid( trans, **kwd ) + @web.expose + @web.require_admin + def delete_repository_metadata( self, trans, **kwd ): + # TODO: Add a javascript confirm before this method executes.... + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + id = kwd.get( 'id', None ) + if id: + ids = util.listify( id ) + count = 0 + for repository_metadata_id in ids: + repository_metadata = get_repository_metadata_by_id( trans, repository_metadata_id ) + trans.sa_session.delete( repository_metadata ) + trans.sa_session.flush() + count += 1 + if count: + message = "Deleted %d repository metadata %s" % ( count, inflector.cond_plural( len( ids ), "record" ) ) + else: + message = "No repository metadata ids received for deleting." + status = 'error' + trans.response.send_redirect( web.url_for( controller='admin', + action='browse_repository_metadata', + message=util.sanitize_text( message ), + status=status ) ) + @web.expose + @web.require_admin def browse_repositories( self, trans, **kwd ): # We add params to the keyword dict in this method in order to rename the param # with an "f-" prefix, simulating filtering by clicking a search link. We have --- a/lib/galaxy/webapps/community/controllers/common.py Mon Aug 15 13:28:40 2011 -0400 +++ b/lib/galaxy/webapps/community/controllers/common.py Mon Aug 15 16:12:12 2011 -0400 @@ -78,52 +78,54 @@ .filter( and_( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ), trans.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) ) \ .first() +def get_repository_metadata_by_id( trans, id ): + """Get repository metadata from the database""" + return trans.sa_session.query( trans.model.RepositoryMetadata ).get( trans.security.decode_id( id ) ) def get_latest_repository_metadata( trans, id ): """Get last metadata defined for a specified repository from the database""" return trans.sa_session.query( trans.model.RepositoryMetadata ) \ .filter( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ) ) \ .order_by( trans.model.RepositoryMetadata.table.c.update_time.desc() ) \ .first() -def set_workflowl_metadata( trans, id, change_set_revision, exported_workflow_dict, metadata_dict ): - # We'll store everything except the workflow steps in the database. - flush_needed = False +def generate_workflow_metadata( trans, id, change_set_revision, exported_workflow_dict, metadata_dict ): + """ + Update the received metadata_dict with changes that have been applied + to the received exported_workflow_dict. Store everything except the + workflow steps in the database. + """ workflow_dict = { 'a_galaxy_workflow' : exported_workflow_dict[ 'a_galaxy_workflow' ], 'name' :exported_workflow_dict[ 'name' ], 'annotation' : exported_workflow_dict[ 'annotation' ], 'format-version' : exported_workflow_dict[ 'format-version' ] } - repository_metadata = get_latest_repository_metadata( trans, id ) - if repository_metadata: - metadata = repository_metadata.metadata - if metadata and 'workflows' in metadata: - metadata_workflows = metadata[ 'workflows' ] - found = False - for workflow_metadata_dict in metadata_workflows: - if 'a_galaxy_workflow' in workflow_metadata_dict and util.string_as_bool( workflow_metadata_dict[ 'a_galaxy_workflow' ] ) and \ - 'name' in workflow_metadata_dict and workflow_metadata_dict[ 'name' ] == exported_workflow_dict[ 'name' ] and \ - 'annotation' in workflow_metadata_dict and workflow_metadata_dict[ 'annotation' ] == exported_workflow_dict[ 'annotation' ] and \ - 'format-version' in workflow_metadata_dict and workflow_metadata_dict[ 'format-version' ] == exported_workflow_dict[ 'format-version' ]: - found = True - break - if not found: - metadata_workflows.append( workflow_dict ) + if 'workflows' in metadata_dict: + metadata_dict[ 'workflows' ].append( workflow_dict ) + else: + metadata_dict[ 'workflows' ] = [ workflow_dict ] + return metadata_dict +def new_workflow_metadata_required( trans, id, metadata_dict ): + """ + TODO: Currently everything about an exported workflow except the name is hard-coded, so + there's no real way to differentiate versions of exported workflows. If this changes at + some future time, this method should be enhanced accordingly... + """ + if 'workflows' in metadata_dict: + repository_metadata = get_latest_repository_metadata( trans, id ) + if repository_metadata: + if repository_metadata.metadata: + # The repository has metadata, so update the workflows value - no new record is needed. + return False else: - if metadata is None: - repository_metadata.metadata = {} - repository_metadata.metadata[ 'workflows' ] = workflow_dict - repository_metadata.changeset_revision = change_set_revision - trans.sa_session.add( repository_metadata ) - if not flush_needed: - flush_needed = True - else: - if 'workflows' in metadata_dict: - metadata_dict[ 'workflows' ].append( workflow_dict ) - else: - metadata_dict[ 'workflows' ] = [ workflow_dict ] - return flush_needed, metadata_dict - -def set_tool_metadata( trans, id, change_set_revision, root, name, tool, metadata_dict ): + # There is no saved repository metadata, so we need to create a new repository_metadata table record. + return True + # The received metadata_dict includes no metadata for workflows, so a new repository_metadata table + # record is not needed. + return False +def generate_tool_metadata( trans, id, change_set_revision, root, name, tool, metadata_dict ): + """ + Update the received metadata_dict with changes that have been + applied to the received tool. + """ tool_requirements = [] - flush_needed = False for tr in tool.requirements: requirement_dict = dict( name=tr.name, type=tr.type, @@ -145,54 +147,42 @@ tool_config=os.path.join( root, name ), requirements=tool_requirements, tests=tool_tests ) - repository_metadata = get_latest_repository_metadata( trans, id ) - if repository_metadata: - # We have the last RepositoryMetadata record generated for this repository. - metadata = repository_metadata.metadata - if metadata and 'tools' in metadata: - # The metadata for 1 or more tools was successfully generated in the past for this repository.. - metadata_tools = metadata[ 'tools' ] - found = False - for tool_metadata_dict in metadata_tools: - if 'id' in tool_metadata_dict and tool_metadata_dict[ 'id' ] == tool.id and \ - 'version' in tool_metadata_dict and tool_metadata_dict[ 'version' ] == tool.version: - # We've found a record for the current tool id and version, so we'll - # update the metadata associated with the record. - found = True - tool_metadata_dict[ 'name' ] = tool.name - tool_metadata_dict[ 'description' ] = tool.description - tool_metadata_dict[ 'version_string_cmd' ] = tool.version_string_cmd - tool_metadata_dict[ 'tool_config' ] = os.path.join( root, name ) - tool_metadata_dict[ 'requirements' ] = tool_requirements - tool_metadata_dict[ 'tests' ] = tool_tests - if not flush_needed: - flush_needed = True - if found: - repository_metadata.changeset_revision = change_set_revision + if 'tools' in metadata_dict: + metadata_dict[ 'tools' ].append( tool_dict ) + else: + metadata_dict[ 'tools' ] = [ tool_dict ] + return metadata_dict +def new_tool_metadata_required( trans, id, metadata_dict ): + """ + Compare the last saved metadata for each tool in the repository with the new metadata + in metadata_dict to determine if a new repository_metadata table record is required, or + if the last saved metadata record can updated instead. + """ + if 'tools' in metadata_dict: + repository_metadata = get_latest_repository_metadata( trans, id ) + if repository_metadata: + metadata = repository_metadata.metadata + if metadata and 'tools' in metadata: + # The metadata for one or more tools was successfully generated in the past + # for this repository, so we compare the version string for each tool id + # in metadata_dict with what was previously saved to see if we need to create + # a new table record or if we can simply update the existing record. + for new_tool_metadata_dict in metadata_dict[ 'tools' ]: + for saved_tool_metadata_dict in metadata[ 'tools' ]: + if new_tool_metadata_dict[ 'id' ] == saved_tool_metadata_dict[ 'id' ]: + if new_tool_metadata_dict[ 'version' ] != saved_tool_metadata_dict[ 'version' ]: + return True else: - if 'tools' in metadata_dict: - metadata_dict[ 'tools' ].append( tool_dict ) - else: - metadata_dict[ 'tools' ] = [ tool_dict ] - if not flush_needed: - flush_needed = True - if not flush_needed: - flush_needed = True + # We have repository metadata that does not include metadata for any tools in the + # repository, so we can update the existing repository metadata. + return False else: - # Either the metadata is Null, or it does not include the key 'tools'. - if metadata is None: - repository_metadata.metadata = {} - repository_metadata.metadata[ 'tools' ] = [ tool_dict ] - repository_metadata.changeset_revision = change_set_revision - trans.sa_session.add( repository_metadata ) - if not flush_needed: - flush_needed = True - else: - if 'tools' in metadata_dict: - metadata_dict[ 'tools' ].append( tool_dict ) - else: - metadata_dict[ 'tools' ] = [ tool_dict ] - return flush_needed, metadata_dict + # There is no saved repository metadata, so we need to create a new repository_metadata + # table record. + return True + # The received metadata_dict includes no metadata for tools, so a new repository_metadata table + # record is not needed. + return False def set_repository_metadata( trans, id, change_set_revision, **kwd ): """Set repository metadata""" message = '' @@ -202,7 +192,6 @@ repo = hg.repository( get_configured_ui(), repo_dir ) change_set = get_change_set( trans, repo, change_set_revision ) invalid_files = [] - flush_needed = False if change_set is not None: metadata_dict = {} for root, dirs, files in os.walk( repo_dir ): @@ -221,7 +210,8 @@ full_path = os.path.abspath( os.path.join( root, name ) ) tool = load_tool( trans, full_path ) if tool is not None: - flush_needed, metadata_dict = set_tool_metadata( trans, id, change_set_revision, root, name, tool, metadata_dict ) + # Update the list metadata dictionaries for tools in metadata_dict. + metadata_dict = generate_tool_metadata( trans, id, change_set_revision, root, name, tool, metadata_dict ) except Exception, e: invalid_files.append( ( name, str( e ) ) ) # Find all exported workflows @@ -233,16 +223,22 @@ workflow_text = fp.read() fp.close() exported_workflow_dict = from_json_string( workflow_text ) - flush_needed, metadata_dict = set_workflowl_metadata( trans, id, change_set_revision, exported_workflow_dict, metadata_dict ) + # Update the list of metadata dictionaries for workflows in metadata_dict. + metadata_dict = generate_workflow_metadata( trans, id, change_set_revision, exported_workflow_dict, metadata_dict ) except Exception, e: invalid_files.append( ( name, str( e ) ) ) if metadata_dict: - # The metadata_dict dictionary will contain items only - # if the repository did not already have metadata set. - repository_metadata = trans.model.RepositoryMetadata( repository.id, change_set_revision, metadata_dict ) - trans.sa_session.add( repository_metadata ) - if not flush_needed: - flush_needed = True + if new_tool_metadata_required( trans, id, metadata_dict ) or new_workflow_metadata_required( trans, id, metadata_dict ): + # Create a new repository_metadata table row. + repository_metadata = trans.model.RepositoryMetadata( repository.id, change_set_revision, metadata_dict ) + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() + else: + # Update the last saved repository_metadata table row. + repository_metadata = get_latest_repository_metadata( trans, id ) + repository_metadata.metadata = metadata_dict + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() else: # change_set is None message = "Repository does not include changeset revision '%s'." % str( change_set_revision ) @@ -284,10 +280,6 @@ correction_msg = exception_msg message += "<b>%s</b> - %s<br/>" % ( tool_file, correction_msg ) status = 'error' - elif flush_needed: - # We only flush if there are no tool config errors, so change sets will only have metadata - # if everything in them is valid. - trans.sa_session.flush() return message, status def get_repository_by_name( trans, name ): """Get a repository from the database via name""" --- a/templates/webapps/community/admin/index.mako Mon Aug 15 13:28:40 2011 -0400 +++ b/templates/webapps/community/admin/index.mako Mon Aug 15 16:12:12 2011 -0400 @@ -43,14 +43,30 @@ <div class="page-container" style="padding: 10px;"><div class="toolMenu"><div class="toolSectionList"> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_categories', webapp='community' )}">Browse by category</a></div> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='admin', action='browse_repositories', webapp='community' )}">Browse all repositories</a></div> + <div class="toolSectionTitle"> + Repositories + </div> + <div class="toolSectionBody"> + <div class="toolSectionBg"> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_categories', webapp='community' )}">Browse by category</a> + </div> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='admin', action='browse_repositories', webapp='community' )}">Browse all repositories</a> + </div> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='admin', action='browse_repository_metadata', webapp='community' )}">Browse metadata</a> + </div> + </div> + </div><div class="toolSectionTitle"> Categories </div><div class="toolSectionBody"><div class="toolSectionBg"> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='admin', action='manage_categories', webapp='community' )}">Manage categories</a></div> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='admin', action='manage_categories', webapp='community' )}">Manage categories</a> + </div></div></div><div class="toolSectionPad"></div> @@ -59,9 +75,15 @@ </div><div class="toolSectionBody"><div class="toolSectionBg"> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='admin', action='users', webapp='community' )}">Manage users</a></div> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='admin', action='groups', webapp='community' )}">Manage groups</a></div> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='admin', action='roles', webapp='community' )}">Manage roles</a></div> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='admin', action='users', webapp='community' )}">Manage users</a> + </div> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='admin', action='groups', webapp='community' )}">Manage groups</a> + </div> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='admin', action='roles', webapp='community' )}">Manage roles</a> + </div></div></div></div> --- a/templates/webapps/community/index.mako Mon Aug 15 13:28:40 2011 -0400 +++ b/templates/webapps/community/index.mako Mon Aug 15 16:12:12 2011 -0400 @@ -45,13 +45,21 @@ <div class="toolMenu"><div class="toolSectionList"><div class="toolSectionPad"></div> - <div class="toolSectionTitle">Repositories</div> + <div class="toolSectionTitle"> + Repositories + </div><div class="toolSectionBody"><div class="toolSectionBg"> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_categories', webapp='community' )}">Browse by category</a></div> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories', webapp='community' )}">Browse all repositories</a></div> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_categories', webapp='community' )}">Browse by category</a> + </div> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories', webapp='community' )}">Browse all repositories</a> + </div> %if trans.user: - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories', operation='my_repositories', webapp='community' )}">Browse my repositories</a></div> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories', operation='my_repositories', webapp='community' )}">Browse my repositories</a> + </div> %endif </div></div> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.