[galaxy-commits] commit/galaxy-central: greg: Add basic UI for managng tool shed repositories installed into a local Galaxy instance from a tool shed. Add the ability to view the list of tools and workflows included in installed tool shed repositories. Ad the ability to import a workflow from an installed tool shed repository into the local Galaxy instance.

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/c3582a9913fa/ changeset: c3582a9913fa user: greg date: 2011-10-28 20:05:14 summary: Add basic UI for managng tool shed repositories installed into a local Galaxy instance from a tool shed. Add the ability to view the list of tools and workflows included in installed tool shed repositories. Ad the ability to import a workflow from an installed tool shed repository into the local Galaxy instance. affected #: 7 files diff -r 07f3f601d645f07b76b196879ecd6d1c68c5b5d4 -r c3582a9913fad4d41b7b7e151163fa4a45371bc1 lib/galaxy/web/base/controller.py --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -2443,13 +2443,6 @@ id = trans.security.decode_id( id ) quota = trans.sa_session.query( trans.model.Quota ).get( id ) return quota -def get_tool_shed_repository( trans, tool_shed, name, owner, changeset_revision ): - return trans.sa_session.query( trans.model.ToolShedRepository ) \ - .filter( and_( trans.model.ToolShedRepository.table.c.tool_shed == tool_shed, - trans.model.ToolShedRepository.table.c.name == name, - trans.model.ToolShedRepository.table.c.owner == owner, - trans.model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ - .first() def handle_sample_tool_data_table_conf_file( trans, filename ): """ Parse the incoming filename and add new entries to the in-memory diff -r 07f3f601d645f07b76b196879ecd6d1c68c5b5d4 -r c3582a9913fad4d41b7b7e151163fa4a45371bc1 lib/galaxy/web/controllers/admin.py --- a/lib/galaxy/web/controllers/admin.py +++ b/lib/galaxy/web/controllers/admin.py @@ -410,7 +410,8 @@ columns = [ NameColumn( "Name", key="name", - attach_popup=True ), + link=( lambda item: dict( operation="manage_repository", id=item.id, webapp="galaxy" ) ), + attach_popup=False ), DescriptionColumn( "Description" ), OwnerColumn( "Owner" ), RevisionColumn( "Revision" ), @@ -426,17 +427,14 @@ key="free-text-search", visible=False, filterable="standard" ) ) - operations = [ grids.GridOperation( "Get updates", - allow_multiple=False, - condition=( lambda item: not item.deleted ), - async_compatible=False ) ] standard_filters = [] default_filter = dict( deleted="False" ) num_rows_per_page = 50 preserve_state = False use_paging = True def build_initial_query( self, trans, **kwd ): - return trans.sa_session.query( self.model_class ) + return trans.sa_session.query( self.model_class ) \ + .filter( self.model_class.table.c.deleted == False ) class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuota, QuotaParamParser ): @@ -678,11 +676,88 @@ return quota, params @web.expose @web.require_admin - def browse_repositories( self, trans, **kwd ): + def browse_tool_shed_repository( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + repository = get_repository( trans, kwd[ 'id' ] ) + relative_install_dir = self.__get_relative_install_dir( trans, repository ) + repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) ) + tool_dicts = [] + workflow_dicts = [] + for root, dirs, files in os.walk( repo_files_dir ): + if not root.find( '.hg' ) >= 0 and not root.find( 'hgrc' ) >= 0: + if '.hg' in dirs: + # Don't visit .hg directories. + dirs.remove( '.hg' ) + if 'hgrc' in files: + # Don't include hgrc files. + files.remove( 'hgrc' ) + for name in files: + # Find all tool configs. + if name.endswith( '.xml' ): + try: + full_path = os.path.abspath( os.path.join( root, name ) ) + tool = trans.app.toolbox.load_tool( full_path ) + if tool is not None: + tool_config = os.path.join( root, name ) + # Handle tool.requirements. + tool_requirements = [] + for tr in tool.requirements: + name=tr.name + type=tr.type + if type == 'fabfile': + version = None + fabfile = tr.fabfile + method = tr.method + else: + version = tr.version + fabfile = None + method = None + requirement_dict = dict( name=name, + type=type, + version=version, + fabfile=fabfile, + method=method ) + tool_requirements.append( requirement_dict ) + tool_dict = dict( id=tool.id, + old_id=tool.old_id, + name=tool.name, + version=tool.version, + description=tool.description, + requirements=tool_requirements, + tool_config=tool_config ) + tool_dicts.append( tool_dict ) + except Exception, e: + # The file is not a Galaxy tool config. + pass + # Find all exported workflows + elif name.endswith( '.ga' ): + try: + full_path = os.path.abspath( os.path.join( root, name ) ) + # Convert workflow data from json + fp = open( full_path, 'rb' ) + workflow_text = fp.read() + fp.close() + workflow_dict = from_json_string( workflow_text ) + if workflow_dict[ 'a_galaxy_workflow' ] == 'true': + workflow_dicts.append( dict( full_path=full_path, workflow_dict=workflow_dict ) ) + except Exception, e: + # The file is not a Galaxy workflow. + pass + return trans.fill_template( '/admin/tool_shed_repository/browse_repository.mako', + repository=repository, + tool_dicts=tool_dicts, + workflow_dicts=workflow_dicts, + message=message, + status=status ) + @web.expose + @web.require_admin + def browse_tool_shed_repositories( self, trans, **kwd ): if 'operation' in kwd: - operation = kwd.pop('operation').lower() - if operation == "get updates": - return self.check_for_updates( trans, **kwd ) + operation = kwd.pop( 'operation' ).lower() + if operation == "manage_repository": + return self.manage_tool_shed_repository( trans, **kwd ) # Render the list view return self.repository_list_grid( trans, **kwd ) @web.expose @@ -850,10 +925,35 @@ status=status ) @web.expose @web.require_admin + def manage_tool_shed_repository( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + repository_id = params.get( 'id', None ) + repository = get_repository( trans, repository_id ) + description = util.restore_text( params.get( 'description', repository.description ) ) + if params.get( 'edit_repository_button', False ): + if description != repository.description: + repository.description = description + trans.sa_session.add( repository ) + trans.sa_session.flush() + message = "The repository information has been updated." + relative_install_dir = self.__get_relative_install_dir( trans, repository ) + if relative_install_dir: + repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) ) + else: + repo_files_dir = 'unknown' + return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako', + repository=repository, + description=description, + repo_files_dir=repo_files_dir, + message=message, + status=status ) + @web.expose + @web.require_admin def check_for_updates( self, trans, **kwd ): params = util.Params( kwd ) - repository_id = params.get( 'id', None ) - repository = get_repository( trans, repository_id ) + repository = get_repository( trans, kwd[ 'id' ] ) galaxy_url = trans.request.host # Send a request to the relevant tool shed to see if there are any updates. # TODO: support https in the following url. @@ -872,26 +972,16 @@ owner = params.get( 'owner', None ) changeset_revision = params.get( 'changeset_revision', None ) latest_changeset_revision = params.get( 'latest_changeset_revision', None ) + repository = get_repository_by_shed_name_owner_changeset_revision( trans, tool_shed_url, name, owner, changeset_revision ) if changeset_revision and latest_changeset_revision: if changeset_revision == latest_changeset_revision: message = "The cloned tool shed repository named '%s' is current (there are no updates available)." % name else: - repository = get_repository_by_name_owner_changeset_revision( trans, name, owner, changeset_revision ) current_working_dir = os.getcwd() - # Get the directory where the repository is cloned. - cleaned_tool_shed_url = self.__clean_tool_shed_url( tool_shed_url ) - partial_cloned_dir = '%s/repos/%s/%s/%s' % ( cleaned_tool_shed_url, owner, name, changeset_revision ) - # Get the relative tool installation paths from each of the shed tool configs. - shed_tool_confs = trans.app.toolbox.shed_tool_confs - relative_cloned_dir = None - # The shed_tool_confs dictionary contains shed_conf_filename : tool_path pairs. - for shed_conf_filename, tool_path in shed_tool_confs.items(): - relative_cloned_dir = os.path.join( tool_path, partial_cloned_dir ) - if os.path.isdir( relative_cloned_dir ): - break - if relative_cloned_dir: + relative_install_dir = self.__get_relative_install_dir( trans, repository ) + if relative_install_dir: # Update the cloned repository to changeset_revision. - repo_files_dir = os.path.join( relative_cloned_dir, name ) + repo_files_dir = os.path.join( relative_install_dir, name ) log.debug( "Updating cloned repository named '%s' from revision '%s' to revision '%s'..." % \ ( name, changeset_revision, latest_changeset_revision ) ) cmd = 'hg pull' @@ -935,9 +1025,23 @@ message = "The latest changeset revision could not be retrieved for the repository named '%s'." % name status = 'error' return trans.response.send_redirect( web.url_for( controller='admin', - action='browse_repositories', + action='manage_tool_shed_repository', + id=trans.security.encode_id( repository.id ), message=message, status=status ) ) + def __get_relative_install_dir( self, trans, repository ): + # Get the directory where the repository is install. + tool_shed = self.__clean_tool_shed_url( repository.tool_shed ) + partial_install_dir = '%s/repos/%s/%s/%s' % ( tool_shed, repository.owner, repository.name, repository.changeset_revision ) + # Get the relative tool installation paths from each of the shed tool configs. + shed_tool_confs = trans.app.toolbox.shed_tool_confs + relative_install_dir = None + # The shed_tool_confs dictionary contains { shed_conf_filename : tool_path } pairs. + for shed_conf_filename, tool_path in shed_tool_confs.items(): + relative_install_dir = os.path.join( tool_path, partial_install_dir ) + if os.path.isdir( relative_install_dir ): + break + return relative_install_dir def __handle_missing_data_table_entry( self, trans, tool_path, sample_files, repository_tools_tups ): # Inspect each tool to see if any have input parameters that are dynamically # generated select lists that require entries in the tool_data_table_conf.xml file. @@ -1057,7 +1161,7 @@ tool_shed = tmp_url.split( 'repos' )[ 0 ].rstrip( '/' ) owner = self.__get_repository_owner( tmp_url ) flush_needed = False - tool_shed_repository = get_tool_shed_repository( trans, tool_shed, name, owner, changeset_revision ) + tool_shed_repository = get_repository_by_shed_name_owner_changeset_revision( trans, tool_shed, name, owner, changeset_revision ) if tool_shed_repository: if tool_shed_repository.deleted: tool_shed_repository.deleted = False @@ -1200,3 +1304,11 @@ trans.model.ToolShedRepository.table.c.owner == owner, trans.model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ .first() +def get_repository_by_shed_name_owner_changeset_revision( trans, tool_shed, name, owner, changeset_revision ): + return trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( and_( trans.model.ToolShedRepository.table.c.tool_shed == tool_shed, + trans.model.ToolShedRepository.table.c.name == name, + trans.model.ToolShedRepository.table.c.owner == owner, + trans.model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ + .first() + diff -r 07f3f601d645f07b76b196879ecd6d1c68c5b5d4 -r c3582a9913fad4d41b7b7e151163fa4a45371bc1 lib/galaxy/web/controllers/workflow.py --- a/lib/galaxy/web/controllers/workflow.py +++ b/lib/galaxy/web/controllers/workflow.py @@ -1095,8 +1095,8 @@ @web.expose def import_workflow( self, trans, **kwd ): """ - Import a workflow by reading an url, uploading a file, or receiving the textual - representation of a workflow. + Import a workflow by reading an url, uploading a file, opening and reading the contents + of a local file, or receiving the textual representation of a workflow via http. """ url = kwd.get( 'url', '' ) workflow_text = kwd.get( 'workflow_text', '' ) @@ -1104,6 +1104,8 @@ message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) import_button = kwd.get( 'import_button', False ) + # The following parameters will have values only if the workflow + # id being imported from a Galaxy tool shed repository. tool_shed_url = kwd.get( 'tool_shed_url', '' ) repository_metadata_id = kwd.get( 'repository_metadata_id', '' ) # The workflow_name parameter is in the request only if the import originated @@ -1111,6 +1113,15 @@ workflow_name = kwd.get( 'workflow_name', '' ) if workflow_name: workflow_name = tool_shed_decode( workflow_name ) + # The following parameters will have a value only if the import originated + # from a tool shed repository installed locally. + local_file = kwd.get( 'local_file', '' ) + repository_id = kwd.get( 'repository_id', '' ) + if local_file and not import_button: + workflow_file = open( local_file, 'rb' ) + workflow_text = workflow_file.read() + workflow_file.close() + import_button = True if tool_shed_url and not import_button: # Use urllib (send another request to the tool shed) to retrieve the workflow. workflow_url = 'http://%s/workflow/import_workflow?repository_metadata_id=%s&workflow_name=%s&webapp=%s&open_for_url=true' % \ @@ -1118,7 +1129,6 @@ response = urllib2.urlopen( workflow_url ) workflow_text = response.read() response.close() - workflow_text = workflow_text import_button = True if import_button: workflow_data = None @@ -1131,6 +1141,7 @@ message = "Failed to open URL: <b>%s</b><br>Exception: %s" % ( url, str( e ) ) status = 'error' elif workflow_text: + # This case occurs when the workflow_text was sent via http from the tool shed. workflow_data = workflow_text else: # Load workflow from browsed file. @@ -1170,7 +1181,7 @@ message += "Imported, but this workflow contains cycles. " status = "error" else: - message += "Workflow '%s' imported successfully. " % workflow.name + message += "Workflow <b>%s</b> imported successfully. " % workflow.name if missing_tool_tups: if trans.user_is_admin(): # A required tool is not available in the local Galaxy instance. @@ -1201,11 +1212,19 @@ pass if tool_shed_url: # We've received the textual representation of a workflow from a Galaxy tool shed. - message = "This workflow has been successfully imported into your local Galaxy instance." + message = "Workflow <b>%s</b> imported successfully." % workflow.name # TODO: support https in the following url. url = 'http://%s/workflow/view_workflow?repository_metadata_id=%s&workflow_name=%s&webapp=%s&message=%s' % \ ( tool_shed_url, repository_metadata_id, tool_shed_encode( workflow_name ), webapp, message ) return trans.response.send_redirect( url ) + elif local_file: + # The workflow was read from a file included with an installed tool shed repository. + message = "Workflow <b>%s</b> imported successfully." % workflow.name + return trans.response.send_redirect( web.url_for( controller='admin', + action='browse_tool_shed_repository', + id=repository_id, + message=message, + status=status ) ) return self.list( trans ) return trans.fill_template( "workflow/import.mako", url=url, diff -r 07f3f601d645f07b76b196879ecd6d1c68c5b5d4 -r c3582a9913fad4d41b7b7e151163fa4a45371bc1 lib/galaxy/webapps/community/controllers/common.py --- a/lib/galaxy/webapps/community/controllers/common.py +++ b/lib/galaxy/webapps/community/controllers/common.py @@ -361,8 +361,9 @@ workflow_text = fp.read() fp.close() exported_workflow_dict = from_json_string( workflow_text ) - # Update the list of metadata dictionaries for workflows in metadata_dict. - metadata_dict = generate_workflow_metadata( trans, id, changeset_revision, exported_workflow_dict, metadata_dict ) + if exported_workflow_dict[ 'a_galaxy_workflow' ] == 'true': + # Update the list of metadata dictionaries for workflows in metadata_dict. + metadata_dict = generate_workflow_metadata( trans, id, changeset_revision, exported_workflow_dict, metadata_dict ) except Exception, e: invalid_files.append( ( name, str( e ) ) ) else: diff -r 07f3f601d645f07b76b196879ecd6d1c68c5b5d4 -r c3582a9913fad4d41b7b7e151163fa4a45371bc1 templates/admin/tool_shed_repository/browse_repository.mako --- /dev/null +++ b/templates/admin/tool_shed_repository/browse_repository.mako @@ -0,0 +1,127 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + +<% from galaxy.web.base.controller import tool_shed_encode, tool_shed_decode %> + +<br/><br/> +<ul class="manage-table-actions"> + <li><a class="action-button" id="repository-${repository.id}-popup" class="menubutton">Repository Actions</a></li> + <div popupmenu="repository-${repository.id}-popup"> + <a class="action-button" href="${h.url_for( controller='admin', action='manage_tool_shed_repository', id=trans.security.encode_id( repository.id ) )}">Manage repository</a> + <a class="action-button" href="${h.url_for( controller='admin', action='check_for_updates', id=trans.security.encode_id( repository.id ) )}">Get updates</a> + </div> +</ul> + +%if message: + ${render_msg( message, status )} +%endif + +<div class="toolForm"> + <div class="toolFormTitle">Installed tool shed repository '${repository.name}'</div> + <div class="toolFormBody"> + + %if tool_dicts: + <div class="form-row"> + <table width="100%"> + <tr bgcolor="#D8D8D8" width="100%"> + <td><b>Tools</b></td> + </tr> + </table> + </div> + <div class="form-row"> + <table class="grid"> + <tr> + <td><b>name</b></td> + <td><b>description</b></td> + <td><b>version</b></td> + <td><b>requirements</b></td> + </tr> + %for tool_dict in tool_dicts: + <tr> + <td>${tool_dict[ 'name' ]}</div> + </td> + <td>${tool_dict[ 'description' ]}</td> + <td>${tool_dict[ 'version' ]}</td> + <td> + <% + if 'requirements' in tool_dict: + requirements = tool_dict[ 'requirements' ] + else: + requirements = None + %> + %if requirements: + <% + requirements_str = '' + for requirement_dict in tool_dict[ 'requirements' ]: + requirements_str += '%s (%s), ' % ( requirement_dict[ 'name' ], requirement_dict[ 'type' ] ) + requirements_str = requirements_str.rstrip( ', ' ) + %> + ${requirements_str} + %else: + none + %endif + </td> + </tr> + %endfor + </table> + </div> + <div style="clear: both"></div> + %endif + %if workflow_dicts: + <div class="form-row"> + <table width="100%"> + <tr bgcolor="#D8D8D8" width="100%"> + <td><b>Workflows</b></td> + </tr> + </table> + </div> + <div style="clear: both"></div> + <div class="form-row"> + <table class="grid"> + <tr> + <td><b>name</b></td> + <td><b>steps</b></td> + <td><b>format-version</b></td> + <td><b>annotation</b></td> + </tr> + <% index = 0 %> + %for wf_dict in workflow_dicts: + <% + full_path = wf_dict[ 'full_path' ] + workflow_dict = wf_dict[ 'workflow_dict' ] + workflow_name = workflow_dict[ 'name' ] + if 'steps' in workflow_dict: + ## Initially steps were not stored in the metadata record. + steps = workflow_dict[ 'steps' ] + else: + steps = [] + format_version = workflow_dict[ 'format-version' ] + annotation = workflow_dict[ 'annotation' ] + %> + <tr> + <td> + <div class="menubutton" style="float: left;" id="workflow-${index}-popup"> + ${workflow_name} + <div popupmenu="workflow-${index}-popup"> + <a class="action-button" href="${h.url_for( controller='workflow', action='import_workflow', local_file=full_path, repository_id=trans.security.encode_id( repository.id ) )}">Import to Galaxy</a> + </div> + </div> + </td> + <td> + %if 'steps' in workflow_dict: + ${len( steps )} + %else: + unknown + %endif + </td> + <td>${format_version}</td> + <td>${annotation}</td> + </tr> + <% index += 1 %> + %endfor + </table> + </div> + <div style="clear: both"></div> + %endif + </div> +</div> diff -r 07f3f601d645f07b76b196879ecd6d1c68c5b5d4 -r c3582a9913fad4d41b7b7e151163fa4a45371bc1 templates/admin/tool_shed_repository/manage_repository.mako --- /dev/null +++ b/templates/admin/tool_shed_repository/manage_repository.mako @@ -0,0 +1,57 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + +<br/><br/> +<ul class="manage-table-actions"> + <li><a class="action-button" id="repository-${repository.id}-popup" class="menubutton">Repository Actions</a></li> + <div popupmenu="repository-${repository.id}-popup"> + <a class="action-button" href="${h.url_for( controller='admin', action='browse_tool_shed_repository', id=trans.security.encode_id( repository.id ) )}">Browse repository</a> + <a class="action-button" href="${h.url_for( controller='admin', action='check_for_updates', id=trans.security.encode_id( repository.id ) )}">Get updates</a> + </div> +</ul> + +%if message: + ${render_msg( message, status )} +%endif + +<div class="toolForm"> + <div class="toolFormTitle">${repository.name}</div> + <div class="toolFormBody"> + <form name="edit_repository" id="edit_repository" action="${h.url_for( controller='admin', action='manage_tool_shed_repository', id=trans.security.encode_id( repository.id ) )}" method="post" > + <div class="form-row"> + <label>Tool shed:</label> + ${repository.tool_shed} + <div style="clear: both"></div> + </div> + <div class="form-row"> + <label>Name:</label> + ${repository.name} + <div style="clear: both"></div> + </div> + <div class="form-row"> + <label>Description:</label> + <input name="description" type="textfield" value="${description}" size="80"/> + <div style="clear: both"></div> + </div> + <div class="form-row"> + <label>Revision:</label> + ${repository.changeset_revision} + </div> + <div class="form-row"> + <label>Owner:</label> + ${repository.owner} + </div> + <div class="form-row"> + <label>Location:</label> + ${repo_files_dir} + </div> + <div class="form-row"> + <label>Deleted:</label> + ${repository.deleted} + </div> + <div class="form-row"> + <input type="submit" name="edit_repository_button" value="Save"/> + </div> + </form> + </div> +</div> diff -r 07f3f601d645f07b76b196879ecd6d1c68c5b5d4 -r c3582a9913fad4d41b7b7e151163fa4a45371bc1 templates/webapps/galaxy/admin/index.mako --- a/templates/webapps/galaxy/admin/index.mako +++ b/templates/webapps/galaxy/admin/index.mako @@ -64,7 +64,7 @@ <div class="toolTitle"><a href="${h.url_for( controller='admin', action='memdump' )}" target="galaxy_main">Profile memory usage</a></div><div class="toolTitle"><a href="${h.url_for( controller='admin', action='jobs' )}" target="galaxy_main">Manage jobs</a></div> %if cloned_repositories: - <div class="toolTitle"><a href="${h.url_for( controller='admin', action='browse_repositories' )}" target="galaxy_main">Manage installed tool shed repositories</a></div> + <div class="toolTitle"><a href="${h.url_for( controller='admin', action='browse_tool_shed_repositories' )}" target="galaxy_main">Manage installed tool shed repositories</a></div> %endif </div></div> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.