1 new commit in galaxy-central:

https://bitbucket.org/galaxy/galaxy-central/commits/6431d6796645/ Changeset: 6431d6796645 User: greg Date: 2013-09-24 18:05:17 Summary: Enhance the Galaxy API to display the list of exported workflows contained in an installed tool shed repository and to import one or more of those exported workflows into Galaxy. This change set is loosely based on John Chilton's pull request # 223 titled "Implement API methods for interfacing with an installed tool shed repository's workflows", but the implementation has been re-worked. All of John's contributed features should be available in this change set. Affected #: 7 files

diff -r 7df8d81544c39cde769a5ead49be182294ba58b1 -r 6431d67966450d1c4e6946d54a94584e343049a4 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2327,9 +2327,12 @@ self.id = None self.user = None

+ class StoredWorkflow( object, Dictifiable): + dict_collection_visible_keys = ( 'id', 'name', 'published' ) dict_element_visible_keys = ( 'id', 'name', 'published' ) + def __init__( self ): self.id = None self.user = None @@ -2357,7 +2360,11 @@ return rval

-class Workflow( object ): +class Workflow( object, Dictifiable ): + + dict_collection_visible_keys = ( 'name', 'has_cycles', 'has_errors' ) + dict_element_visible_keys = ( 'name', 'has_cycles', 'has_errors' ) + def __init__( self ): self.user = None self.name = None @@ -2365,7 +2372,9 @@ self.has_errors = None self.steps = []

+ class WorkflowStep( object ): + def __init__( self ): self.id = None self.type = None @@ -2376,36 +2385,48 @@ self.input_connections = [] self.config = None

+ class WorkflowStepConnection( object ): + def __init__( self ): self.output_step_id = None self.output_name = None self.input_step_id = None self.input_name = None

+ class WorkflowOutput(object): + def __init__( self, workflow_step, output_name): self.workflow_step = workflow_step self.output_name = output_name

+ class StoredWorkflowUserShareAssociation( object ): + def __init__( self ): self.stored_workflow = None self.user = None

+ class StoredWorkflowMenuEntry( object ): + def __init__( self ): self.stored_workflow = None self.user = None self.order_index = None

+ class WorkflowInvocation( object ): pass

+ class WorkflowInvocationStep( object ): pass

+ class MetadataFile( object ): + def __init__( self, dataset = None, name = None ): if isinstance( dataset, HistoryDatasetAssociation ): self.history_dataset = dataset

diff -r 7df8d81544c39cde769a5ead49be182294ba58b1 -r 6431d67966450d1c4e6946d54a94584e343049a4 lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -10,6 +10,7 @@ from tool_shed.galaxy_install import repository_util from tool_shed.util import common_util from tool_shed.util import encoding_util +from tool_shed.util import workflow_util import tool_shed.util.shed_util_common as suc

log = logging.getLogger( __name__ ) @@ -33,6 +34,102 @@ """RESTful controller for interactions with tool shed repositories."""

@web.expose_api + def exported_workflows( self, trans, id, **kwd ): + """ + GET /api/tool_shed_repositories/{encoded_tool_shed_repository_id}/exported_workflows + + Display a list of dictionaries containing information about this tool shed repository's exported workflows. + + :param id: the encoded id of the ToolShedRepository object + """ + # Example URL: http://localhost:8763/api/tool_shed_repositories/f2db41e1fa331b3e/exported_w... + # Since exported workflows are dictionaries with very few attributes that differentiate them from each other, we'll build the + # list based on the following dictionary of those few attributes. + exported_workflows = [] + repository = suc.get_tool_shed_repository_by_id( trans, id ) + metadata = repository.metadata + if metadata: + exported_workflow_tups = metadata.get( 'workflows', [] ) + else: + exported_workflow_tups = [] + for index, exported_workflow_tup in enumerate( exported_workflow_tups ): + # The exported_workflow_tup looks like ( relative_path, exported_workflow_dict ), where the value of relative_path is the location + # on disk (relative to the root of the installed repository) where the exported_workflow_dict file (.ga file) is located. + exported_workflow_dict = exported_workflow_tup[ 1 ] + annotation = exported_workflow_dict.get( 'annotation', '' ) + format_version = exported_workflow_dict.get( 'format-version', '' ) + workflow_name = exported_workflow_dict.get( 'name', '' ) + # Since we don't have an in-memory object with an id, we'll identify the exported workflow via it's location (i.e., index) in the list. + display_dict = dict( index=index, annotation=annotation, format_version=format_version, workflow_name=workflow_name ) + exported_workflows.append( display_dict ) + return exported_workflows + + @web.expose_api + def import_workflow( self, trans, payload, **kwd ): + """ + POST /api/tool_shed_repositories/import_workflow + + Import the specified exported workflow contained in the specified installed tool shed repository into Galaxy. + + :param key: the API key of the Galaxy user with which the imported workflow will be associated. + :param id: the encoded id of the ToolShedRepository object + + The following parameters are included in the payload. + :param index: the index location of the workflow tuple in the list of exported workflows stored in the metadata for the specified repository + """ + api_key = kwd.get( 'key', None ) + if api_key is None: + raise HTTPBadRequest( detail="Missing required parameter 'key' whose value is the API key for the Galaxy user importing the specified workflow." ) + tool_shed_repository_id = kwd.get( 'id', '' ) + if not tool_shed_repository_id: + raise HTTPBadRequest( detail="Missing required parameter 'id'." ) + index = payload.get( 'index', None ) + if index is None: + raise HTTPBadRequest( detail="Missing required parameter 'index'." ) + repository = suc.get_tool_shed_repository_by_id( trans, tool_shed_repository_id ) + exported_workflows = json.from_json_string( self.exported_workflows( trans, tool_shed_repository_id ) ) + # Since we don't have an in-memory object with an id, we'll identify the exported workflow via it's location (i.e., index) in the list. + exported_workflow = exported_workflows[ int( index ) ] + workflow_name = exported_workflow[ 'workflow_name' ] + workflow, status, message = workflow_util.import_workflow( trans, repository, workflow_name ) + if status == 'error': + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + else: + return workflow.to_dict( view='element' ) + + @web.expose_api + def import_workflows( self, trans, **kwd ): + """ + POST /api/tool_shed_repositories/import_workflow + + Import all of the exported workflows contained in the specified installed tool shed repository into Galaxy. + + :param key: the API key of the Galaxy user with which the imported workflows will be associated. + :param id: the encoded id of the ToolShedRepository object + """ + api_key = kwd.get( 'key', None ) + if api_key is None: + raise HTTPBadRequest( detail="Missing required parameter 'key' whose value is the API key for the Galaxy user importing the specified workflow." ) + tool_shed_repository_id = kwd.get( 'id', '' ) + if not tool_shed_repository_id: + raise HTTPBadRequest( detail="Missing required parameter 'id'." ) + repository = suc.get_tool_shed_repository_by_id( trans, tool_shed_repository_id ) + exported_workflows = json.from_json_string( self.exported_workflows( trans, tool_shed_repository_id ) ) + imported_workflow_dicts = [] + for exported_workflow_dict in exported_workflows: + workflow_name = exported_workflow_dict[ 'workflow_name' ] + workflow, status, message = workflow_util.import_workflow( trans, repository, workflow_name ) + if status == 'error': + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + else: + imported_workflow_dicts.append( workflow.to_dict( view='element' ) ) + return imported_workflow_dicts + + @web.expose_api def index( self, trans, **kwd ): """ GET /api/tool_shed_repositories @@ -58,28 +155,6 @@ return message

@web.expose_api - def show( self, trans, id, **kwd ): - """ - GET /api/tool_shed_repositories/{encoded_tool_shed_repsository_id} - Display a dictionary containing information about a specified tool_shed_repository. - - :param id: the encoded id of the ToolShedRepository object - """ - # Example URL: http://localhost:8763/api/tool_shed_repositories/df7a1f0c02a5b08e - try: - tool_shed_repository = suc.get_tool_shed_repository_by_id( trans, id ) - tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) ) - tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', - action='show', - id=trans.security.encode_id( tool_shed_repository.id ) ) - return tool_shed_repository_dict - except Exception, e: - message = "Error in tool_shed_repositories API in index: " + str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message - - @web.expose_api def install_repository_revision( self, trans, payload, **kwd ): """ POST /api/tool_shed_repositories/install_repository_revision @@ -412,3 +487,25 @@ tool_shed_repositories.append( repository_dict ) # Display the list of repaired repositories. return tool_shed_repositories + + @web.expose_api + def show( self, trans, id, **kwd ): + """ + GET /api/tool_shed_repositories/{encoded_tool_shed_repsository_id} + Display a dictionary containing information about a specified tool_shed_repository. + + :param id: the encoded id of the ToolShedRepository object + """ + # Example URL: http://localhost:8763/api/tool_shed_repositories/df7a1f0c02a5b08e + try: + tool_shed_repository = suc.get_tool_shed_repository_by_id( trans, id ) + tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) ) + tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', + action='show', + id=trans.security.encode_id( tool_shed_repository.id ) ) + return tool_shed_repository_dict + except Exception, e: + message = "Error in tool_shed_repositories API in index: " + str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message

diff -r 7df8d81544c39cde769a5ead49be182294ba58b1 -r 6431d67966450d1c4e6946d54a94584e343049a4 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -176,7 +176,10 @@ # Galaxy API for tool shed features. webapp.mapper.resource( 'tool_shed_repository', 'tool_shed_repositories', - member={ 'repair_repository_revision' : 'POST' }, + member={ 'repair_repository_revision' : 'POST', + 'exported_workflows' : 'GET', + 'import_workflow' : 'POST', + 'import_workflows' : 'POST' }, controller='tool_shed_repositories', name_prefix='tool_shed_repository_', path_prefix='/api',

diff -r 7df8d81544c39cde769a5ead49be182294ba58b1 -r 6431d67966450d1c4e6946d54a94584e343049a4 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -358,66 +358,26 @@ @web.expose @web.require_admin def import_workflow( self, trans, workflow_name, repository_id, **kwd ): - """Import a workflow contained in an installed tool shed repository into the Galaxy instance.""" + """Import a workflow contained in an installed tool shed repository into Galaxy.""" message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) if workflow_name: workflow_name = encoding_util.tool_shed_decode( workflow_name ) - repository = suc.get_tool_shed_repository_by_id( trans, repository_id ) - changeset_revision = repository.changeset_revision - metadata = repository.metadata - workflows = metadata.get( 'workflows', [] ) - tools_metadata = metadata.get( 'tools', [] ) - workflow_dict = None - for workflow_data_tuple in workflows: - # The value of workflow_data_tuple is ( relative_path_to_workflow_file, exported_workflow_dict ). - relative_path_to_workflow_file, exported_workflow_dict = workflow_data_tuple - if exported_workflow_dict[ 'name' ] == workflow_name: - # If the exported workflow is available on disk, import it. - if os.path.exists( relative_path_to_workflow_file ): - workflow_file = open( relative_path_to_workflow_file, 'rb' ) - workflow_data = workflow_file.read() - workflow_file.close() - workflow_dict = json.from_json_string( workflow_data ) + repository = suc.get_tool_shed_repository_by_id( trans, repository_id ) + if repository: + workflow, status, message = workflow_util.import_workflow( trans, repository, workflow_name ) + if workflow: + workflow_name = encoding_util.tool_shed_encode( str( workflow.name ) ) else: - # Use the current exported_workflow_dict. - workflow_dict = exported_workflow_dict - break - if workflow_dict: - # Create workflow if possible. - workflow, missing_tool_tups = workflow_util.get_workflow_from_dict( trans=trans, - workflow_dict=workflow_dict, - tools_metadata=tools_metadata, - repository_id=repository_id, - changeset_revision=changeset_revision ) - # Save the workflow in the Galaxy database. - # Pass workflow_dict along to create annotation at this point - stored_workflow = workflow_util.save_workflow( trans, workflow, workflow_dict ) - # Use the latest version of the saved workflow. - workflow = stored_workflow.latest_workflow - if workflow_name: - workflow.name = workflow_name - # Provide user feedback and show workflow list. - if workflow.has_errors: - message += "Imported, but some steps in this workflow have validation errors. " - status = "error" - if workflow.has_cycles: - message += "Imported, but this workflow contains cycles. " - status = "error" + message += 'Unable to locate a workflow named <b>%s</b> within the installed tool shed repository named <b>%s</b>' % \ + ( str( workflow_name ), str( repository.name ) ) + status = 'error' else: - message += "Workflow <b>%s</b> imported successfully. " % workflow.name - if missing_tool_tups: - # TODO: rework this since it is used in the tool shed, but shoudn't be used in Galaxy. - name_and_id_str = '' - for missing_tool_tup in missing_tool_tups: - tool_id, tool_name, other = missing_tool_tup - name_and_id_str += 'name: %s, id: %s' % ( str( tool_id ), str( tool_name ) ) - log.debug( "The following tools required by this workflow are missing from this Galaxy instance: %s" % name_and_id_str ) + message = 'Invalid repository id <b>%s</b> received.' % str( repository_id ) + status = 'error' else: - message += 'The workflow named %s is not included in the metadata for revision %s of repository %s' % \ - ( str( workflow_name ), str( changeset_revision ), str( repository.name ) ) + message = 'The value of workflow_name is required, but was not received.' status = 'error' - workflow_name = encoding_util.tool_shed_encode( workflow.name ), return trans.response.send_redirect( web.url_for( controller='admin_toolshed', action='view_workflow', workflow_name=workflow_name,

diff -r 7df8d81544c39cde769a5ead49be182294ba58b1 -r 6431d67966450d1c4e6946d54a94584e343049a4 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -1896,7 +1896,7 @@ changeset_revisions.append( changeset_revision ) add_tool_versions( trans, encoded_id, repository_metadata, changeset_revisions ) elif len( repo ) == 1 and not invalid_file_tups: - message = "Revision '%s' includes no tools, datatypes or exported workflows for which metadata can " % str( repository.tip( trans.app ) ) + message = "Revision <b>%s</b> includes no Galaxy utilities for which metadata can " % str( repository.tip( trans.app ) ) message += "be defined so this revision cannot be automatically installed into a local Galaxy instance." status = "error" if invalid_file_tups:

diff -r 7df8d81544c39cde769a5ead49be182294ba58b1 -r 6431d67966450d1c4e6946d54a94584e343049a4 lib/tool_shed/util/workflow_util.py --- a/lib/tool_shed/util/workflow_util.py +++ b/lib/tool_shed/util/workflow_util.py @@ -1,4 +1,5 @@ import logging +import os import galaxy.tools import galaxy.tools.parameters import galaxy.webapps.galaxy.controllers.workflow @@ -151,7 +152,7 @@ def new( self, trans, type, tools_metadata=None, tool_id=None ): """Return module for type and (optional) tool_id initialized with new / default state.""" assert type in self.module_types - return self.module_types[type].new( trans, tool_id ) + return self.module_types[ type ].new( trans, tool_id )

def from_dict( self, trans, repository_id, changeset_revision, step_dict, **kwd ): """Return module initialized from the data in dictionary `step_dict`.""" @@ -219,13 +220,11 @@ tool_errors = module.type == 'tool' and not module.tool module_data_inputs = get_workflow_data_inputs( step, module ) module_data_outputs = get_workflow_data_outputs( step, module, workflow.steps ) - step_dict = { - 'id' : step.order_index, - 'data_inputs' : module_data_inputs, - 'data_outputs' : module_data_outputs, - 'position' : step.position, - 'tool_errors' : tool_errors - } + step_dict = { 'id' : step.order_index, + 'data_inputs' : module_data_inputs, + 'data_outputs' : module_data_outputs, + 'position' : step.position, + 'tool_errors' : tool_errors } input_conn_dict = {} for conn in step.input_connections: input_conn_dict[ conn.input_name ] = dict( id=conn.output_step.order_index, output_name=conn.output_name ) @@ -401,8 +400,9 @@ post_job_actions = step_dict.get( 'post_job_actions', {} ) for name, pja_dict in post_job_actions.items(): trans.model.PostJobAction( pja_dict[ 'action_type' ], - step, pja_dict[ 'output_name' ], - pja_dict[ 'action_arguments' ] ) + step, + pja_dict[ 'output_name' ], + pja_dict[ 'action_arguments' ] ) steps.append( step ) steps_by_external_id[ step_dict[ 'id' ] ] = step # Second pass to deal with connections between steps. @@ -433,6 +433,64 @@ break return module_name

+def import_workflow( trans, repository, workflow_name ): + """Import a workflow contained in an installed tool shed repository into Galaxy (this method is called only from Galaxy).""" + status = 'done' + message = '' + changeset_revision = repository.changeset_revision + metadata = repository.metadata + workflows = metadata.get( 'workflows', [] ) + tools_metadata = metadata.get( 'tools', [] ) + workflow_dict = None + for workflow_data_tuple in workflows: + # The value of workflow_data_tuple is ( relative_path_to_workflow_file, exported_workflow_dict ). + relative_path_to_workflow_file, exported_workflow_dict = workflow_data_tuple + if exported_workflow_dict[ 'name' ] == workflow_name: + # If the exported workflow is available on disk, import it. + if os.path.exists( relative_path_to_workflow_file ): + workflow_file = open( relative_path_to_workflow_file, 'rb' ) + workflow_data = workflow_file.read() + workflow_file.close() + workflow_dict = json.from_json_string( workflow_data ) + else: + # Use the current exported_workflow_dict. + workflow_dict = exported_workflow_dict + break + if workflow_dict: + # Create workflow if possible. + workflow, missing_tool_tups = get_workflow_from_dict( trans=trans, + workflow_dict=workflow_dict, + tools_metadata=tools_metadata, + repository_id=repository.id, + changeset_revision=changeset_revision ) + # Save the workflow in the Galaxy database. Pass workflow_dict along to create annotation at this point. + stored_workflow = save_workflow( trans, workflow, workflow_dict ) + # Use the latest version of the saved workflow. + workflow = stored_workflow.latest_workflow + if workflow_name: + workflow.name = workflow_name + # Provide user feedback and show workflow list. + if workflow.has_errors: + message += "Imported, but some steps in this workflow have validation errors. " + status = "error" + if workflow.has_cycles: + message += "Imported, but this workflow contains cycles. " + status = "error" + else: + message += "Workflow <b>%s</b> imported successfully. " % workflow.name + if missing_tool_tups: + name_and_id_str = '' + for missing_tool_tup in missing_tool_tups: + tool_id, tool_name, other = missing_tool_tup + name_and_id_str += 'name: %s, id: %s' % ( str( tool_id ), str( tool_name ) ) + message += "The following tools required by this workflow are missing from this Galaxy instance: %s. " % name_and_id_str + else: + workflow = None + message += 'The workflow named %s is not included in the metadata for revision %s of repository %s' % \ + ( str( workflow_name ), str( changeset_revision ), str( repository.name ) ) + status = 'error' + return workflow, status, message + def save_workflow( trans, workflow, workflow_dict = None): """Use the received in-memory Workflow object for saving to the Galaxy database.""" stored = trans.model.StoredWorkflow()

diff -r 7df8d81544c39cde769a5ead49be182294ba58b1 -r 6431d67966450d1c4e6946d54a94584e343049a4 scripts/api/import_workflows_from_installed_tool_shed_repository.py --- /dev/null +++ b/scripts/api/import_workflows_from_installed_tool_shed_repository.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +""" +Import one or more exported workflows contained within a specified tool shed repository installed into Galaxy. + +Here is a working example of how to use this script to repair a repository installed into Galaxy. +python ./import_workflows_from_installed_tool_shed_repository.py -a 22be3b -l http://localhost:8763/ -n workflow_with_tools -o test -r ef45bb64237e -u http://localhost:9009/ +""" + +import os +import sys +import argparse +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import display +from common import submit + +def clean_url( url ): + if url.find( '//' ) > 0: + # We have an url that includes a protocol, something like: http://localhost:9009 + items = url.split( '//' ) + return items[ 1 ].rstrip( '/' ) + return url.rstrip( '/' ) + +def main( options ): + api_key = options.api + base_galaxy_url = options.local_url.rstrip( '/' ) + base_tool_shed_url = options.tool_shed_url.rstrip( '/' ) + cleaned_tool_shed_url = clean_url( base_tool_shed_url ) + installed_tool_shed_repositories_url = '%s/api/tool_shed_repositories' % base_galaxy_url + tool_shed_repository_id = None + installed_tool_shed_repositories = display( api_key, installed_tool_shed_repositories_url, return_formatted=False ) + for installed_tool_shed_repository in installed_tool_shed_repositories: + tool_shed = str( installed_tool_shed_repository[ 'tool_shed' ] ) + name = str( installed_tool_shed_repository[ 'name' ] ) + owner = str( installed_tool_shed_repository[ 'owner' ] ) + changeset_revision = str( installed_tool_shed_repository[ 'changeset_revision' ] ) + if tool_shed == cleaned_tool_shed_url and name == options.name and owner == options.owner and changeset_revision == options.changeset_revision: + tool_shed_repository_id = installed_tool_shed_repository[ 'id' ] + break + if tool_shed_repository_id: + # Get the list of exported workflows contained in the installed repository. + url = '%s%s' % ( base_galaxy_url, '/api/tool_shed_repositories/%s/exported_workflows' % str( tool_shed_repository_id ) ) + exported_workflows = display( api_key, url, return_formatted=False ) + if exported_workflows: + # Import all of the workflows in the list of exported workflows. + data = {} + # NOTE: to import a single workflow, add an index to data (e.g., + # data[ 'index' ] = 0 + # and change the url to be ~/import_workflow (simgular). For example, + # url = '%s%s' % ( base_galaxy_url, '/api/tool_shed_repositories/%s/import_workflow' % str( tool_shed_repository_id ) ) + url = '%s%s' % ( base_galaxy_url, '/api/tool_shed_repositories/%s/import_workflows' % str( tool_shed_repository_id ) ) + submit( options.api, url, data ) + else: + print "Invalid tool_shed / name / owner / changeset_revision." + +if __name__ == '__main__': + parser = argparse.ArgumentParser( description='Import workflows contained in an installed tool shed repository via the Galaxy API.' ) + parser.add_argument( "-a", "--api", dest="api", required=True, help="API Key" ) + parser.add_argument( "-u", "--url", dest="tool_shed_url", required=True, help="Tool Shed URL" ) + parser.add_argument( "-l", "--local", dest="local_url", required=True, help="URL of the galaxy instance." ) + parser.add_argument( "-n", "--name", required=True, help="Repository name." ) + parser.add_argument( "-o", "--owner", required=True, help="Repository owner." ) + parser.add_argument( "-r", "--revision", dest="changeset_revision", required=True, help="Repository owner." ) + options = parser.parse_args() + main( options )

Repository URL: https://bitbucket.org/galaxy/galaxy-central/

--

This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.