details:
http://www.bx.psu.edu/hg/galaxy/rev/ad5232a4dbc5
changeset: 2557:ad5232a4dbc5
user: Dan Blankenberg <dan(a)bx.psu.edu>
date: Thu Aug 13 11:00:00 2009 -0400
description:
When setting metadata external is configured, using the 'Auto-detect' function on
a hda will now launch an external job process.
13 file(s) affected in this change:
lib/galaxy/app.py
lib/galaxy/datatypes/registry.py
lib/galaxy/jobs/__init__.py
lib/galaxy/jobs/runners/local.py
lib/galaxy/model/__init__.py
lib/galaxy/tools/__init__.py
lib/galaxy/tools/actions/metadata.py
lib/galaxy/tools/actions/upload.py
lib/galaxy/web/controllers/root.py
static/june_2007_style/blue/history.css
static/june_2007_style/history.css.tmpl
templates/mobile/history/detail.mako
templates/root/history_common.mako
diffs (287 lines):
diff -r 846f6a7ee80c -r ad5232a4dbc5 lib/galaxy/app.py
--- a/lib/galaxy/app.py Wed Aug 12 15:56:03 2009 -0400
+++ b/lib/galaxy/app.py Thu Aug 13 11:00:00 2009 -0400
@@ -37,6 +37,8 @@
self.toolbox = tools.ToolBox( self.config.tool_config, self.config.tool_path,
self )
# Load datatype converters
self.datatypes_registry.load_datatype_converters( self.toolbox )
+ #load external metadata tool
+ self.datatypes_registry.load_external_metadata_tool( self.toolbox )
# Load datatype indexers
self.datatypes_registry.load_datatype_indexers( self.toolbox )
#Load security policy
diff -r 846f6a7ee80c -r ad5232a4dbc5 lib/galaxy/datatypes/registry.py
--- a/lib/galaxy/datatypes/registry.py Wed Aug 12 15:56:03 2009 -0400
+++ b/lib/galaxy/datatypes/registry.py Thu Aug 13 11:00:00 2009 -0400
@@ -1,7 +1,7 @@
"""
Provides mapping between extensions and datatypes, mime-types, etc.
"""
-import os
+import os, tempfile
import logging
import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage,
tracks, chrominfo
import galaxy.util
@@ -18,6 +18,7 @@
self.datatype_converters = odict()
self.datatype_indexers = odict()
self.converters = []
+ self.set_external_metadata_tool = None
self.indexers = []
self.sniff_order = []
self.upload_file_formats = []
@@ -251,6 +252,31 @@
self.datatype_converters[source_datatype][target_datatype] = converter
self.log.debug( "Loaded converter: %s", converter.id )
+ def load_external_metadata_tool( self, toolbox ):
+ """Adds a tool which is used to set external
metadata"""
+ #we need to be able to add a job to the queue to set metadata. The queue will
currently only accept jobs with an associated tool.
+ #We'll create a special tool to be used for Auto-Detecting metadata; this is
less than ideal, but effective
+ #Properly building a tool without relying on parsing an XML file is near
impossible...so we'll create a temporary file
+ tool_xml_text = """
+ <tool id="__SET_METADATA__" name="Set External
Metadata" version="1.0.0" tool_type="set_metadata">
+ <type class="SetMetadataTool"
module="galaxy.tools"/>
+ <action module="galaxy.tools.actions.metadata"
class="SetMetadataToolAction"/>
+ <command>$__SET_EXTERNAL_METADATA_COMMAND_LINE__</command>
+ <inputs>
+ <param format="data" name="input1"
type="data" label="File to set metadata on."/>
+ <param name="__ORIGINAL_DATASET_STATE__"
type="hidden" value=""/>
+ <param name="__SET_EXTERNAL_METADATA_COMMAND_LINE__"
type="hidden" value=""/>
+ </inputs>
+ </tool>
+ """
+ tmp_name = tempfile.NamedTemporaryFile()
+ tmp_name.write( tool_xml_text )
+ tmp_name.flush()
+ set_meta_tool = toolbox.load_tool( tmp_name.name )
+ toolbox.tools_by_id[ set_meta_tool.id ] = set_meta_tool
+ self.set_external_metadata_tool = set_meta_tool
+ self.log.debug( "Loaded external metadata tool: %s",
self.set_external_metadata_tool.id )
+
def load_datatype_indexers( self, toolbox ):
"""Adds indexers from self.indexers to the toolbox from
app"""
for elem in self.indexers:
diff -r 846f6a7ee80c -r ad5232a4dbc5 lib/galaxy/jobs/__init__.py
--- a/lib/galaxy/jobs/__init__.py Wed Aug 12 15:56:03 2009 -0400
+++ b/lib/galaxy/jobs/__init__.py Thu Aug 13 11:00:00 2009 -0400
@@ -274,7 +274,7 @@
elif idata.state == idata.states.ERROR:
job_wrapper.fail( "input data %d is in error state" % (
idata.hid ) )
return JOB_INPUT_ERROR
- elif idata.state != idata.states.OK:
+ elif idata.state != idata.states.OK and not ( idata.state ==
idata.states.SETTING_METADATA and job.tool_id is not None and job.tool_id ==
self.app.datatypes_registry.set_external_metadata_tool.id ):
# need to requeue
return JOB_WAIT
return JOB_READY
@@ -543,7 +543,7 @@
# Certain tools require tasks to be completed after job execution
# ( this used to be performed in the "exec_after_process" hook, but
hooks are deprecated ).
if self.tool.tool_type is not None:
- self.tool.exec_after_process( self.queue.app, inp_data, out_data, param_dict
)
+ self.tool.exec_after_process( self.queue.app, inp_data, out_data, param_dict,
job = job )
# Call 'exec_after_process' hook
self.tool.call_hook( 'exec_after_process', self.queue.app,
inp_data=inp_data,
out_data=out_data, param_dict=param_dict,
diff -r 846f6a7ee80c -r ad5232a4dbc5 lib/galaxy/jobs/runners/local.py
--- a/lib/galaxy/jobs/runners/local.py Wed Aug 12 15:56:03 2009 -0400
+++ b/lib/galaxy/jobs/runners/local.py Thu Aug 13 11:00:00 2009 -0400
@@ -104,7 +104,7 @@
#run the metadata setting script here
#this is terminatable when output dataset/job is deleted
#so that long running set_meta()s can be cancelled without having to reboot the
server
- if job_wrapper.get_state() not in [ model.Job.states.ERROR,
model.Job.states.DELETED ] and self.app.config.set_metadata_externally:
+ if job_wrapper.get_state() not in [ model.Job.states.ERROR,
model.Job.states.DELETED ] and self.app.config.set_metadata_externally and
job_wrapper.output_paths:
external_metadata_script = job_wrapper.setup_external_metadata( output_fnames
= job_wrapper.get_output_fnames(), kwds = { 'overwrite' : False } ) #we don't
want to overwrite metadata that was copied over in init_meta(), as per established
behavior
log.debug( 'executing external set_meta script for job %d: %s' % (
job_wrapper.job_id, external_metadata_script ) )
external_metadata_proc = subprocess.Popen( args = external_metadata_script,
diff -r 846f6a7ee80c -r ad5232a4dbc5 lib/galaxy/model/__init__.py
--- a/lib/galaxy/model/__init__.py Wed Aug 12 15:56:03 2009 -0400
+++ b/lib/galaxy/model/__init__.py Thu Aug 13 11:00:00 2009 -0400
@@ -330,7 +330,8 @@
OK = 'ok',
EMPTY = 'empty',
ERROR = 'error',
- DISCARDED = 'discarded' )
+ DISCARDED = 'discarded',
+ SETTING_METADATA = 'setting_metadata' )
permitted_actions = get_permitted_actions( filter='DATASET' )
file_path = "/tmp/"
engine = None
diff -r 846f6a7ee80c -r ad5232a4dbc5 lib/galaxy/tools/__init__.py
--- a/lib/galaxy/tools/__init__.py Wed Aug 12 15:56:03 2009 -0400
+++ b/lib/galaxy/tools/__init__.py Thu Aug 13 11:00:00 2009 -0400
@@ -1468,7 +1468,7 @@
out_data[ name ] = data
return out_data
- def exec_after_process( self, app, inp_data, out_data, param_dict ):
+ def exec_after_process( self, app, inp_data, out_data, param_dict, job = None ):
if self.tool_type == 'data_source':
name, data = out_data.items()[0]
data.set_size()
@@ -1572,6 +1572,18 @@
dataset.history.add( new_data )
new_data.flush()
return primary_datasets
+
+class SetMetadataTool( Tool ):
+ def exec_after_process( self, app, inp_data, out_data, param_dict, job = None ):
+ for name, dataset in inp_data.iteritems():
+ external_metadata =
galaxy.datatypes.metadata.JobExternalOutputMetadataWrapper( job )
+ if external_metadata.external_metadata_set_successfully( dataset ):
+ dataset.metadata.from_JSON_dict(
external_metadata.get_output_filenames_by_dataset( dataset ).filename_out )
+ # If setting external metadata has failed, how can we inform the user?
+ # For now, we'll leave the default metadata and set the state back to its
original.
+ dataset.datatype.after_edit( dataset )
+ dataset.state = param_dict.get( '__ORIGINAL_DATASET_STATE__' )
+ dataset.flush()
# ---- Utility classes to be factored out -----------------------------------
diff -r 846f6a7ee80c -r ad5232a4dbc5 lib/galaxy/tools/actions/metadata.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/lib/galaxy/tools/actions/metadata.py Thu Aug 13 11:00:00 2009 -0400
@@ -0,0 +1,48 @@
+from . import ToolAction
+from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper
+
+import logging
+log = logging.getLogger( __name__ )
+
+class SetMetadataToolAction( ToolAction ):
+ """Tool action used for setting external metadata on an existing
dataset"""
+
+ def execute( self, tool, trans, incoming = {}, set_output_hid = False ):
+ for name, value in incoming.iteritems():
+ if isinstance( value, trans.app.model.HistoryDatasetAssociation ):
+ dataset = value
+ dataset_name = name
+ break
+ else:
+ raise Exception( 'The dataset to set metadata on could not be
determined.' )
+
+ # Create the job object
+ job = trans.app.model.Job()
+ job.session_id = trans.get_galaxy_session().id
+ job.history_id = trans.history.id
+ job.tool_id = tool.id
+ try:
+ # For backward compatibility, some tools may not have versions yet.
+ job.tool_version = tool.version
+ except:
+ job.tool_version = "1.0.0"
+ job.flush() #ensure job.id is available
+
+ #add parameters to job_parameter table
+ incoming[ '__ORIGINAL_DATASET_STATE__' ] = dataset.state #store original
dataset state, so we can restore it. A seperate table might be better (no chance of
'loosing' the original state)?
+ external_metadata_wrapper = JobExternalOutputMetadataWrapper( job )
+ cmd_line = external_metadata_wrapper.setup_external_metadata( dataset, exec_dir =
None, tmp_dir = trans.app.config.new_file_path, dataset_files_path =
trans.app.model.Dataset.file_path, output_fnames = None, config_root = None,
datatypes_config = None, kwds = { 'overwrite' : True } )
+ incoming[ '__SET_EXTERNAL_METADATA_COMMAND_LINE__' ] = cmd_line
+ for name, value in tool.params_to_strings( incoming, trans.app ).iteritems():
+ job.add_parameter( name, value )
+ #add the dataset to job_to_input_dataset table
+ job.add_input_dataset( dataset_name, dataset )
+ #Need a special state here to show that metadata is being set and also allow the
job to run
+ # i.e. if state was set to 'running' the set metadata job would never
run, as it would wait for input (the dataset to set metadata on) to be in a ready state
+ dataset.state = dataset.states.SETTING_METADATA
+ trans.app.model.flush()
+
+ # Queue the job for execution
+ trans.app.job_queue.put( job.id, tool )
+ trans.log_event( "Added set external metadata job to the job queue, id:
%s" % str(job.id), tool_id=job.tool_id )
+ return []
diff -r 846f6a7ee80c -r ad5232a4dbc5 lib/galaxy/tools/actions/upload.py
--- a/lib/galaxy/tools/actions/upload.py Wed Aug 12 15:56:03 2009 -0400
+++ b/lib/galaxy/tools/actions/upload.py Thu Aug 13 11:00:00 2009 -0400
@@ -1,4 +1,5 @@
import os, shutil, urllib, StringIO, re, gzip, tempfile, shutil, zipfile
+from . import ToolAction
from galaxy import datatypes, jobs
from galaxy.datatypes import sniff
from galaxy import model, util
@@ -8,7 +9,7 @@
import logging
log = logging.getLogger( __name__ )
-class UploadToolAction( object ):
+class UploadToolAction( ToolAction ):
# Action for uploading files
def __init__( self ):
self.empty = False
diff -r 846f6a7ee80c -r ad5232a4dbc5 lib/galaxy/web/controllers/root.py
--- a/lib/galaxy/web/controllers/root.py Wed Aug 12 15:56:03 2009 -0400
+++ b/lib/galaxy/web/controllers/root.py Thu Aug 13 11:00:00 2009 -0400
@@ -297,10 +297,15 @@
if name not in [ 'name', 'info', 'dbkey' ]:
if spec.get( 'default' ):
setattr( data.metadata, name, spec.unwrap( spec.get(
'default' ) ) )
- data.set_meta()
- data.datatype.after_edit( data )
+ if trans.app.config.set_metadata_externally:
+ msg = 'Attributes have been queued to be updated'
+
trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute(
trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming = {
'input1':data } )
+ else:
+ msg = 'Attributes updated'
+ data.set_meta()
+ data.datatype.after_edit( data )
trans.app.model.flush()
- return trans.show_ok_message( "Attributes updated",
refresh_frames=['history'] )
+ return trans.show_ok_message( msg, refresh_frames=['history'] )
elif params.convert_data:
target_type = kwd.get("target_type", None)
if target_type:
diff -r 846f6a7ee80c -r ad5232a4dbc5 static/june_2007_style/blue/history.css
--- a/static/june_2007_style/blue/history.css Wed Aug 12 15:56:03 2009 -0400
+++ b/static/june_2007_style/blue/history.css Thu Aug 13 11:00:00 2009 -0400
@@ -13,6 +13,8 @@
div.historyItem-empty .state-icon{background:url(history-states.png) no-repeat 0px
-25px;}
div.historyItem-running{border-color:#AAAA66;background:#FFFFCC;}
div.historyItem-running .state-icon{background-image:url(data_running.gif);}
+div.historyItem-setting_metadata{border-color:#AAAA66;background:#FFFFCC;}
+div.historyItem-setting_metadata .state-icon{background-image:url(data_running.gif);}
div.historyItem-upload{border-color:#6666AA;background:#CCCCFF;}
div.historyItem-upload .state-icon{background-image:url(data_upload.gif);}
div.historyItem-queued{background:#EEEEEE;}
diff -r 846f6a7ee80c -r ad5232a4dbc5 static/june_2007_style/history.css.tmpl
--- a/static/june_2007_style/history.css.tmpl Wed Aug 12 15:56:03 2009 -0400
+++ b/static/june_2007_style/history.css.tmpl Thu Aug 13 11:00:00 2009 -0400
@@ -72,6 +72,14 @@
}
div.historyItem-running {
+ border-color: $history_running_border;
+ background: $history_running_bg;
+ .state-icon {
+ background-image: url(data_running.gif);
+ }
+}
+
+div.historyItem-setting_metadata {
border-color: $history_running_border;
background: $history_running_bg;
.state-icon {
diff -r 846f6a7ee80c -r ad5232a4dbc5 templates/mobile/history/detail.mako
--- a/templates/mobile/history/detail.mako Wed Aug 12 15:56:03 2009 -0400
+++ b/templates/mobile/history/detail.mako Thu Aug 13 11:00:00 2009 -0400
@@ -51,6 +51,8 @@
<div>
The job creating this dataset was cancelled before completion.
</div>
+ %elif data_state == 'setting_metadata':
+ <div>Metadata is being Auto-Detected.</div>
%elif data_state == "empty":
<div>No data:
<i>${data.display_info()}</i></div>
%elif data_state == "ok":
diff -r 846f6a7ee80c -r ad5232a4dbc5 templates/root/history_common.mako
--- a/templates/root/history_common.mako Wed Aug 12 15:56:03 2009 -0400
+++ b/templates/root/history_common.mako Thu Aug 13 11:00:00 2009 -0400
@@ -58,6 +58,8 @@
<div>
The job creating this dataset was cancelled before completion.
</div>
+ %elif data_state == 'setting_metadata':
+ <div>${_('Metadata is being Auto-Detected.')}</div>
%elif data_state == "empty":
<div>${_('No data:
')}<i>${data.display_info()}</i></div>
%elif data_state == "ok":