details: http://www.bx.psu.edu/hg/galaxy/rev/eec050533545 changeset: 3736:eec050533545 user: James Taylor <james@jamestaylor.org> date: Mon May 03 17:35:54 2010 -0400 description: Very basic support for exporting and importing workflows among instances diffstat: lib/galaxy/tools/__init__.py | 24 +++-- lib/galaxy/web/__init__.py | 2 +- lib/galaxy/web/controllers/workflow.py | 153 ++++++++++++++++++++++++++++++++- lib/galaxy/web/framework/__init__.py | 9 + lib/galaxy/workflow/modules.py | 14 +- templates/form.mako | 6 +- 6 files changed, 187 insertions(+), 21 deletions(-) diffs (327 lines): diff -r c75c0f9b0bf7 -r eec050533545 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py Mon May 03 16:32:35 2010 -0400 +++ b/lib/galaxy/tools/__init__.py Mon May 03 17:35:54 2010 -0400 @@ -203,7 +203,7 @@ def __init__( self ): self.page = 0 self.inputs = None - def encode( self, tool, app ): + def encode( self, tool, app, secure=True ): """ Convert the data to a string """ @@ -213,18 +213,22 @@ value["__page__"] = self.page value = simplejson.dumps( value ) # Make it secure - a = hmac_new( app.config.tool_secret, value ) - b = binascii.hexlify( value ) - return "%s:%s" % ( a, b ) - def decode( self, value, tool, app ): + if secure: + a = hmac_new( app.config.tool_secret, value ) + b = binascii.hexlify( value ) + return "%s:%s" % ( a, b ) + else: + return value + def decode( self, value, tool, app, secure=True ): """ Restore the state from a string """ - # Extract and verify hash - a, b = value.split( ":" ) - value = binascii.unhexlify( b ) - test = hmac_new( app.config.tool_secret, value ) - assert a == test + if secure: + # Extract and verify hash + a, b = value.split( ":" ) + value = binascii.unhexlify( b ) + test = hmac_new( app.config.tool_secret, value ) + assert a == test # Restore from string values = json_fix( simplejson.loads( value ) ) self.page = values.pop( "__page__" ) diff -r c75c0f9b0bf7 -r eec050533545 lib/galaxy/web/__init__.py --- a/lib/galaxy/web/__init__.py Mon May 03 16:32:35 2010 -0400 +++ b/lib/galaxy/web/__init__.py Mon May 03 17:35:54 2010 -0400 @@ -2,6 +2,6 @@ The Galaxy web application. """ -from framework import expose, json, require_login, require_admin, url_for, error, form, FormBuilder +from framework import expose, json, json_pretty, require_login, require_admin, url_for, error, form, FormBuilder from framework.base import httpexceptions diff -r c75c0f9b0bf7 -r eec050533545 lib/galaxy/web/controllers/workflow.py --- a/lib/galaxy/web/controllers/workflow.py Mon May 03 16:32:35 2010 -0400 +++ b/lib/galaxy/web/controllers/workflow.py Mon May 03 17:35:54 2010 -0400 @@ -18,6 +18,8 @@ from galaxy.model.mapping import desc from galaxy.model.orm import * +import urllib2 + class StoredWorkflowListGrid( grids.Grid ): class StepsColumn( grids.GridColumn ): def get_value(self, trans, grid, workflow): @@ -621,7 +623,6 @@ step_dict['position'] = step.position # Add to return value data['steps'][step.order_index] = step_dict - print data['upgrade_messages'] return data @web.json @@ -660,7 +661,6 @@ workflow.has_errors = True # Stick this in the step temporarily step.temp_input_connections = step_dict['input_connections'] - # Save step annotation. annotation = step_dict[ 'annotation' ] if annotation: @@ -698,6 +698,155 @@ rval['name'] = workflow.name return rval + @web.json_pretty + def export_workflow( self, trans, id ): + """ + Get the latest Workflow for the StoredWorkflow identified by `id` and + encode it as a json string that can be imported back into Galaxy + + This has slightly different information than the above. In particular, + it does not attempt to decode forms and build UIs, it just stores + the raw state. + """ + user = trans.get_user() + id = trans.security.decode_id( id ) + trans.workflow_building_mode = True + # Load encoded workflow from database + stored = trans.sa_session.query( model.StoredWorkflow ).get( id ) + self.security_check( trans.get_user(), stored, False, True ) + workflow = stored.latest_workflow + # Pack workflow data into a dictionary and return + data = {} + data['name'] = workflow.name + data['steps'] = {} + # For each step, rebuild the form and encode the state + for step in workflow.steps: + # Load from database representation + module = module_factory.from_workflow_step( trans, step ) + # Get user annotation. + step_annotation = self.get_item_annotation_obj( trans, trans.user, step ) + annotation_str = "" + if step_annotation: + annotation_str = step_annotation.annotation + # Pack attributes into plain dictionary + step_dict = { + 'id': step.order_index, + 'type': module.type, + 'tool_id': module.get_tool_id(), + 'name': module.get_name(), + 'tool_state': module.get_state( secure=False ), + 'tool_errors': module.get_errors(), + ## 'data_inputs': module.get_data_inputs(), + ## 'data_outputs': module.get_data_outputs(), + 'annotation' : annotation_str + } + # Connections + input_connections = step.input_connections + if step.type is None or step.type == 'tool': + # Determine full (prefixed) names of valid input datasets + data_input_names = {} + def callback( input, value, prefixed_name, prefixed_label ): + if isinstance( input, DataToolParameter ): + data_input_names[ prefixed_name ] = True + visit_input_values( module.tool.inputs, module.state.inputs, callback ) + # Filter + # FIXME: this removes connection without displaying a message currently! + input_connections = [ conn for conn in input_connections if conn.input_name in data_input_names ] + # Encode input connections as dictionary + input_conn_dict = {} + for conn in input_connections: + input_conn_dict[ conn.input_name ] = \ + dict( id=conn.output_step.order_index, output_name=conn.output_name ) + step_dict['input_connections'] = input_conn_dict + # Position + step_dict['position'] = step.position + # Add to return value + data['steps'][step.order_index] = step_dict + return data + + @web.expose + def import_workflow( self, trans, workflow_text=None, url=None ): + if workflow_text is None and url is None: + return form( url_for(), "Import Workflow", submit_text="Import" ) \ + .add_text( "url", "URL to load workflow from", "" ) \ + .add_input( "textarea", "Encoded workflow (as generated by export workflow)", "workflow_text", "" ) + if url: + # Load workflow from external URL + # NOTE: blocks the web thread. + try: + workflow_data = urllib2.urlopen( url ).read() + except Exception, e: + return trans.show_error_message( "Failed to open URL %s<br><br>Message: %s" % ( url, str( e ) ) ) + else: + workflow_data = workflow_text + # Convert incoming workflow data from json + try: + data = simplejson.loads( workflow_data ) + except Exception, e: + return trans.show_error_message( "Data at '%s' does not appear to be a Galaxy workflow<br><br>Message: %s" % ( url, str( e ) ) ) + # Put parameters in workflow mode + trans.workflow_building_mode = True + # Create new workflow from incoming data + workflow = model.Workflow() + # Just keep the last name (user can rename later) + workflow.name = data['name'] + # Assume no errors until we find a step that has some + workflow.has_errors = False + # Create each step + steps = [] + # The editor will provide ids for each step that we don't need to save, + # but do need to use to make connections + steps_by_external_id = {} + # First pass to build step objects and populate basic values + for key, step_dict in data['steps'].iteritems(): + # Create the model class for the step + step = model.WorkflowStep() + steps.append( step ) + steps_by_external_id[ step_dict['id' ] ] = step + # FIXME: Position should be handled inside module + step.position = step_dict['position'] + module = module_factory.from_dict( trans, step_dict, secure=False ) + module.save_to_step( step ) + if step.tool_errors: + workflow.has_errors = True + # Stick this in the step temporarily + step.temp_input_connections = step_dict['input_connections'] + # Save step annotation. + annotation = step_dict[ 'annotation' ] + if annotation: + annotation = sanitize_html( annotation, 'utf-8', 'text/html' ) + self.add_item_annotation( trans, step, annotation ) + # Second pass to deal with connections between steps + for step in steps: + # Input connections + for input_name, conn_dict in step.temp_input_connections.iteritems(): + if conn_dict: + conn = model.WorkflowStepConnection() + conn.input_step = step + conn.input_name = input_name + conn.output_name = conn_dict['output_name'] + conn.output_step = steps_by_external_id[ conn_dict['id'] ] + del step.temp_input_connections + # Order the steps if possible + attach_ordered_steps( workflow, steps ) + # Connect up + stored = model.StoredWorkflow() + stored.name = workflow.name + workflow.stored_workflow = stored + stored.latest_workflow = workflow + stored.user = trans.user + # Persist + trans.sa_session.add( stored ) + trans.sa_session.flush() + # Return something informative + errors = [] + if workflow.has_errors: + return trans.show_warn_message( "Imported, but some steps in this workflow have validation errors" ) + if workflow.has_cycles: + return trans.show_warn_message( "Imported, but this workflow contains cycles" ) + else: + return trans.show_message( "Workflow '%s' imported" % workflow.name ) + @web.json def get_datatypes( self, trans ): ext_to_class_name = dict() diff -r c75c0f9b0bf7 -r eec050533545 lib/galaxy/web/framework/__init__.py --- a/lib/galaxy/web/framework/__init__.py Mon May 03 16:32:35 2010 -0400 +++ b/lib/galaxy/web/framework/__init__.py Mon May 03 17:35:54 2010 -0400 @@ -66,6 +66,15 @@ decorator.exposed = True return decorator +def json_pretty( func ): + def decorator( self, trans, *args, **kwargs ): + trans.response.set_content_type( "text/javascript" ) + return simplejson.dumps( func( self, trans, *args, **kwargs ), indent=4, sort_keys=True ) + if not hasattr(func, '_orig'): + decorator._orig = func + decorator.exposed = True + return decorator + def require_login( verb="perform this action", use_panels=False, webapp='galaxy' ): def argcatcher( func ): def decorator( self, trans, *args, **kwargs ): diff -r c75c0f9b0bf7 -r eec050533545 lib/galaxy/workflow/modules.py --- a/lib/galaxy/workflow/modules.py Mon May 03 16:32:35 2010 -0400 +++ b/lib/galaxy/workflow/modules.py Mon May 03 17:35:54 2010 -0400 @@ -96,7 +96,7 @@ module.state = dict( name="Input Dataset" ) return module @classmethod - def from_dict( Class, trans, d ): + def from_dict( Class, trans, d, secure=False ): module = Class( trans ) state = from_json_string( d["tool_state"] ) module.state = dict( name=state.get( "name", "Input Dataset" ) ) @@ -172,11 +172,11 @@ module.state = module.tool.new_state( trans, all_pages=True ) return module @classmethod - def from_dict( Class, trans, d ): + def from_dict( Class, trans, d, secure=False ): tool_id = d['tool_id'] module = Class( trans, tool_id ) module.state = DefaultToolState() - module.state.decode( d["tool_state"], module.tool, module.trans.app ) + module.state.decode( d["tool_state"], module.tool, module.trans.app, secure=False ) module.errors = d.get( "tool_errors", None ) return module @@ -199,8 +199,8 @@ return self.tool.name def get_tool_id( self ): return self.tool_id - def get_state( self ): - return self.state.encode( self.tool, self.trans.app ) + def get_state( self, secure=True ): + return self.state.encode( self.tool, self.trans.app, secure=secure ) def get_errors( self ): return self.errors def get_tooltip( self ): @@ -278,13 +278,13 @@ """ assert type in self.module_types return self.module_types[type].new( trans, tool_id ) - def from_dict( self, trans, d ): + def from_dict( self, trans, d, **kwargs ): """ Return module initialized from the data in dictionary `d`. """ type = d['type'] assert type in self.module_types - return self.module_types[type].from_dict( trans, d ) + return self.module_types[type].from_dict( trans, d, **kwargs ) def from_workflow_step( self, trans, step ): """ Return module initializd from the WorkflowStep object `step`. diff -r c75c0f9b0bf7 -r eec050533545 templates/form.mako --- a/templates/form.mako Mon May 03 16:32:35 2010 -0400 +++ b/templates/form.mako Mon May 03 17:35:54 2010 -0400 @@ -60,7 +60,11 @@ </label> %endif <div class="form-row-input"> - <input type="${input.type}" name="${input.name}" value="${input.value}" size="40"> + %if input.type == 'textarea': + <textarea name="${input.name}" cols="40">${input.value}</textarea> + %else: + <input type="${input.type}" name="${input.name}" value="${input.value}" size="40"> + %endif </div> %if input.error: <div class="form-row-error-message">${input.error}</div>