details: http://www.bx.psu.edu/hg/galaxy/rev/ad5232a4dbc5 changeset: 2557:ad5232a4dbc5 user: Dan Blankenberg <dan@bx.psu.edu> date: Thu Aug 13 11:00:00 2009 -0400 description: When setting metadata external is configured, using the 'Auto-detect' function on a hda will now launch an external job process. 13 file(s) affected in this change: lib/galaxy/app.py lib/galaxy/datatypes/registry.py lib/galaxy/jobs/__init__.py lib/galaxy/jobs/runners/local.py lib/galaxy/model/__init__.py lib/galaxy/tools/__init__.py lib/galaxy/tools/actions/metadata.py lib/galaxy/tools/actions/upload.py lib/galaxy/web/controllers/root.py static/june_2007_style/blue/history.css static/june_2007_style/history.css.tmpl templates/mobile/history/detail.mako templates/root/history_common.mako diffs (287 lines): diff -r 846f6a7ee80c -r ad5232a4dbc5 lib/galaxy/app.py --- a/lib/galaxy/app.py Wed Aug 12 15:56:03 2009 -0400 +++ b/lib/galaxy/app.py Thu Aug 13 11:00:00 2009 -0400 @@ -37,6 +37,8 @@ self.toolbox = tools.ToolBox( self.config.tool_config, self.config.tool_path, self ) # Load datatype converters self.datatypes_registry.load_datatype_converters( self.toolbox ) + #load external metadata tool + self.datatypes_registry.load_external_metadata_tool( self.toolbox ) # Load datatype indexers self.datatypes_registry.load_datatype_indexers( self.toolbox ) #Load security policy diff -r 846f6a7ee80c -r ad5232a4dbc5 lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py Wed Aug 12 15:56:03 2009 -0400 +++ b/lib/galaxy/datatypes/registry.py Thu Aug 13 11:00:00 2009 -0400 @@ -1,7 +1,7 @@ """ Provides mapping between extensions and datatypes, mime-types, etc. """ -import os +import os, tempfile import logging import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks, chrominfo import galaxy.util @@ -18,6 +18,7 @@ self.datatype_converters = odict() self.datatype_indexers = odict() self.converters = [] + self.set_external_metadata_tool = None self.indexers = [] self.sniff_order = [] self.upload_file_formats = [] @@ -251,6 +252,31 @@ self.datatype_converters[source_datatype][target_datatype] = converter self.log.debug( "Loaded converter: %s", converter.id ) + def load_external_metadata_tool( self, toolbox ): + """Adds a tool which is used to set external metadata""" + #we need to be able to add a job to the queue to set metadata. The queue will currently only accept jobs with an associated tool. + #We'll create a special tool to be used for Auto-Detecting metadata; this is less than ideal, but effective + #Properly building a tool without relying on parsing an XML file is near impossible...so we'll create a temporary file + tool_xml_text = """ + <tool id="__SET_METADATA__" name="Set External Metadata" version="1.0.0" tool_type="set_metadata"> + <type class="SetMetadataTool" module="galaxy.tools"/> + <action module="galaxy.tools.actions.metadata" class="SetMetadataToolAction"/> + <command>$__SET_EXTERNAL_METADATA_COMMAND_LINE__</command> + <inputs> + <param format="data" name="input1" type="data" label="File to set metadata on."/> + <param name="__ORIGINAL_DATASET_STATE__" type="hidden" value=""/> + <param name="__SET_EXTERNAL_METADATA_COMMAND_LINE__" type="hidden" value=""/> + </inputs> + </tool> + """ + tmp_name = tempfile.NamedTemporaryFile() + tmp_name.write( tool_xml_text ) + tmp_name.flush() + set_meta_tool = toolbox.load_tool( tmp_name.name ) + toolbox.tools_by_id[ set_meta_tool.id ] = set_meta_tool + self.set_external_metadata_tool = set_meta_tool + self.log.debug( "Loaded external metadata tool: %s", self.set_external_metadata_tool.id ) + def load_datatype_indexers( self, toolbox ): """Adds indexers from self.indexers to the toolbox from app""" for elem in self.indexers: diff -r 846f6a7ee80c -r ad5232a4dbc5 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py Wed Aug 12 15:56:03 2009 -0400 +++ b/lib/galaxy/jobs/__init__.py Thu Aug 13 11:00:00 2009 -0400 @@ -274,7 +274,7 @@ elif idata.state == idata.states.ERROR: job_wrapper.fail( "input data %d is in error state" % ( idata.hid ) ) return JOB_INPUT_ERROR - elif idata.state != idata.states.OK: + elif idata.state != idata.states.OK and not ( idata.state == idata.states.SETTING_METADATA and job.tool_id is not None and job.tool_id == self.app.datatypes_registry.set_external_metadata_tool.id ): # need to requeue return JOB_WAIT return JOB_READY @@ -543,7 +543,7 @@ # Certain tools require tasks to be completed after job execution # ( this used to be performed in the "exec_after_process" hook, but hooks are deprecated ). if self.tool.tool_type is not None: - self.tool.exec_after_process( self.queue.app, inp_data, out_data, param_dict ) + self.tool.exec_after_process( self.queue.app, inp_data, out_data, param_dict, job = job ) # Call 'exec_after_process' hook self.tool.call_hook( 'exec_after_process', self.queue.app, inp_data=inp_data, out_data=out_data, param_dict=param_dict, diff -r 846f6a7ee80c -r ad5232a4dbc5 lib/galaxy/jobs/runners/local.py --- a/lib/galaxy/jobs/runners/local.py Wed Aug 12 15:56:03 2009 -0400 +++ b/lib/galaxy/jobs/runners/local.py Thu Aug 13 11:00:00 2009 -0400 @@ -104,7 +104,7 @@ #run the metadata setting script here #this is terminatable when output dataset/job is deleted #so that long running set_meta()s can be cancelled without having to reboot the server - if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and self.app.config.set_metadata_externally: + if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and self.app.config.set_metadata_externally and job_wrapper.output_paths: external_metadata_script = job_wrapper.setup_external_metadata( output_fnames = job_wrapper.get_output_fnames(), kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior log.debug( 'executing external set_meta script for job %d: %s' % ( job_wrapper.job_id, external_metadata_script ) ) external_metadata_proc = subprocess.Popen( args = external_metadata_script, diff -r 846f6a7ee80c -r ad5232a4dbc5 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Wed Aug 12 15:56:03 2009 -0400 +++ b/lib/galaxy/model/__init__.py Thu Aug 13 11:00:00 2009 -0400 @@ -330,7 +330,8 @@ OK = 'ok', EMPTY = 'empty', ERROR = 'error', - DISCARDED = 'discarded' ) + DISCARDED = 'discarded', + SETTING_METADATA = 'setting_metadata' ) permitted_actions = get_permitted_actions( filter='DATASET' ) file_path = "/tmp/" engine = None diff -r 846f6a7ee80c -r ad5232a4dbc5 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py Wed Aug 12 15:56:03 2009 -0400 +++ b/lib/galaxy/tools/__init__.py Thu Aug 13 11:00:00 2009 -0400 @@ -1468,7 +1468,7 @@ out_data[ name ] = data return out_data - def exec_after_process( self, app, inp_data, out_data, param_dict ): + def exec_after_process( self, app, inp_data, out_data, param_dict, job = None ): if self.tool_type == 'data_source': name, data = out_data.items()[0] data.set_size() @@ -1572,6 +1572,18 @@ dataset.history.add( new_data ) new_data.flush() return primary_datasets + +class SetMetadataTool( Tool ): + def exec_after_process( self, app, inp_data, out_data, param_dict, job = None ): + for name, dataset in inp_data.iteritems(): + external_metadata = galaxy.datatypes.metadata.JobExternalOutputMetadataWrapper( job ) + if external_metadata.external_metadata_set_successfully( dataset ): + dataset.metadata.from_JSON_dict( external_metadata.get_output_filenames_by_dataset( dataset ).filename_out ) + # If setting external metadata has failed, how can we inform the user? + # For now, we'll leave the default metadata and set the state back to its original. + dataset.datatype.after_edit( dataset ) + dataset.state = param_dict.get( '__ORIGINAL_DATASET_STATE__' ) + dataset.flush() # ---- Utility classes to be factored out ----------------------------------- diff -r 846f6a7ee80c -r ad5232a4dbc5 lib/galaxy/tools/actions/metadata.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/tools/actions/metadata.py Thu Aug 13 11:00:00 2009 -0400 @@ -0,0 +1,48 @@ +from . import ToolAction +from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper + +import logging +log = logging.getLogger( __name__ ) + +class SetMetadataToolAction( ToolAction ): + """Tool action used for setting external metadata on an existing dataset""" + + def execute( self, tool, trans, incoming = {}, set_output_hid = False ): + for name, value in incoming.iteritems(): + if isinstance( value, trans.app.model.HistoryDatasetAssociation ): + dataset = value + dataset_name = name + break + else: + raise Exception( 'The dataset to set metadata on could not be determined.' ) + + # Create the job object + job = trans.app.model.Job() + job.session_id = trans.get_galaxy_session().id + job.history_id = trans.history.id + job.tool_id = tool.id + try: + # For backward compatibility, some tools may not have versions yet. + job.tool_version = tool.version + except: + job.tool_version = "1.0.0" + job.flush() #ensure job.id is available + + #add parameters to job_parameter table + incoming[ '__ORIGINAL_DATASET_STATE__' ] = dataset.state #store original dataset state, so we can restore it. A seperate table might be better (no chance of 'loosing' the original state)? + external_metadata_wrapper = JobExternalOutputMetadataWrapper( job ) + cmd_line = external_metadata_wrapper.setup_external_metadata( dataset, exec_dir = None, tmp_dir = trans.app.config.new_file_path, dataset_files_path = trans.app.model.Dataset.file_path, output_fnames = None, config_root = None, datatypes_config = None, kwds = { 'overwrite' : True } ) + incoming[ '__SET_EXTERNAL_METADATA_COMMAND_LINE__' ] = cmd_line + for name, value in tool.params_to_strings( incoming, trans.app ).iteritems(): + job.add_parameter( name, value ) + #add the dataset to job_to_input_dataset table + job.add_input_dataset( dataset_name, dataset ) + #Need a special state here to show that metadata is being set and also allow the job to run + # i.e. if state was set to 'running' the set metadata job would never run, as it would wait for input (the dataset to set metadata on) to be in a ready state + dataset.state = dataset.states.SETTING_METADATA + trans.app.model.flush() + + # Queue the job for execution + trans.app.job_queue.put( job.id, tool ) + trans.log_event( "Added set external metadata job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id ) + return [] diff -r 846f6a7ee80c -r ad5232a4dbc5 lib/galaxy/tools/actions/upload.py --- a/lib/galaxy/tools/actions/upload.py Wed Aug 12 15:56:03 2009 -0400 +++ b/lib/galaxy/tools/actions/upload.py Thu Aug 13 11:00:00 2009 -0400 @@ -1,4 +1,5 @@ import os, shutil, urllib, StringIO, re, gzip, tempfile, shutil, zipfile +from . import ToolAction from galaxy import datatypes, jobs from galaxy.datatypes import sniff from galaxy import model, util @@ -8,7 +9,7 @@ import logging log = logging.getLogger( __name__ ) -class UploadToolAction( object ): +class UploadToolAction( ToolAction ): # Action for uploading files def __init__( self ): self.empty = False diff -r 846f6a7ee80c -r ad5232a4dbc5 lib/galaxy/web/controllers/root.py --- a/lib/galaxy/web/controllers/root.py Wed Aug 12 15:56:03 2009 -0400 +++ b/lib/galaxy/web/controllers/root.py Thu Aug 13 11:00:00 2009 -0400 @@ -297,10 +297,15 @@ if name not in [ 'name', 'info', 'dbkey' ]: if spec.get( 'default' ): setattr( data.metadata, name, spec.unwrap( spec.get( 'default' ) ) ) - data.set_meta() - data.datatype.after_edit( data ) + if trans.app.config.set_metadata_externally: + msg = 'Attributes have been queued to be updated' + trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming = { 'input1':data } ) + else: + msg = 'Attributes updated' + data.set_meta() + data.datatype.after_edit( data ) trans.app.model.flush() - return trans.show_ok_message( "Attributes updated", refresh_frames=['history'] ) + return trans.show_ok_message( msg, refresh_frames=['history'] ) elif params.convert_data: target_type = kwd.get("target_type", None) if target_type: diff -r 846f6a7ee80c -r ad5232a4dbc5 static/june_2007_style/blue/history.css --- a/static/june_2007_style/blue/history.css Wed Aug 12 15:56:03 2009 -0400 +++ b/static/june_2007_style/blue/history.css Thu Aug 13 11:00:00 2009 -0400 @@ -13,6 +13,8 @@ div.historyItem-empty .state-icon{background:url(history-states.png) no-repeat 0px -25px;} div.historyItem-running{border-color:#AAAA66;background:#FFFFCC;} div.historyItem-running .state-icon{background-image:url(data_running.gif);} +div.historyItem-setting_metadata{border-color:#AAAA66;background:#FFFFCC;} +div.historyItem-setting_metadata .state-icon{background-image:url(data_running.gif);} div.historyItem-upload{border-color:#6666AA;background:#CCCCFF;} div.historyItem-upload .state-icon{background-image:url(data_upload.gif);} div.historyItem-queued{background:#EEEEEE;} diff -r 846f6a7ee80c -r ad5232a4dbc5 static/june_2007_style/history.css.tmpl --- a/static/june_2007_style/history.css.tmpl Wed Aug 12 15:56:03 2009 -0400 +++ b/static/june_2007_style/history.css.tmpl Thu Aug 13 11:00:00 2009 -0400 @@ -72,6 +72,14 @@ } div.historyItem-running { + border-color: $history_running_border; + background: $history_running_bg; + .state-icon { + background-image: url(data_running.gif); + } +} + +div.historyItem-setting_metadata { border-color: $history_running_border; background: $history_running_bg; .state-icon { diff -r 846f6a7ee80c -r ad5232a4dbc5 templates/mobile/history/detail.mako --- a/templates/mobile/history/detail.mako Wed Aug 12 15:56:03 2009 -0400 +++ b/templates/mobile/history/detail.mako Thu Aug 13 11:00:00 2009 -0400 @@ -51,6 +51,8 @@ <div> The job creating this dataset was cancelled before completion. </div> + %elif data_state == 'setting_metadata': + <div>Metadata is being Auto-Detected.</div> %elif data_state == "empty": <div>No data: <i>${data.display_info()}</i></div> %elif data_state == "ok": diff -r 846f6a7ee80c -r ad5232a4dbc5 templates/root/history_common.mako --- a/templates/root/history_common.mako Wed Aug 12 15:56:03 2009 -0400 +++ b/templates/root/history_common.mako Thu Aug 13 11:00:00 2009 -0400 @@ -58,6 +58,8 @@ <div> The job creating this dataset was cancelled before completion. </div> + %elif data_state == 'setting_metadata': + <div>${_('Metadata is being Auto-Detected.')}</div> %elif data_state == "empty": <div>${_('No data: ')}<i>${data.display_info()}</i></div> %elif data_state == "ok":