galaxy-dist commit 2447b9a4dae3: Make "loc files" more flexible by adding "tool data tables". These are
by commits-noreply＠bitbucket.org
# HG changeset patch -- Bitbucket.org
# Project galaxy-dist
# URL http://bitbucket.org/galaxy/galaxy-dist/overview
# User James Taylor <james(a)jamestaylor.org>
# Date 1278614219 14400
# Node ID 2447b9a4dae30b17df089290a6471f3401fa5f78
# Parent e683c6995fb5d032660d1c28cff2e1bc578d1298
Make "loc files" more flexible by adding "tool data tables". These are
configured at the application level. Specific tabular data files are
specified in a application config file and bound to names, the tools
then refer to these names. Thus users can configure where location
files are located without modifying tool configs.
Also:
- Simpler column name configuration
- Columns can be referred to by name in addition to index in all
dynamic option filters
- A data table can merge multiple files
- Design can support other types of data files
--- a/lib/galaxy/tools/parameters/dynamic_options.py
+++ b/lib/galaxy/tools/parameters/dynamic_options.py
@@ -46,9 +46,9 @@ class StaticValueFilter( Filter ):
Filter.__init__( self, d_option, elem )
self.value = elem.get( "value", None )
assert self.value is not None, "Required 'value' attribute missing from filter"
- self.column = elem.get( "column", None )
- assert self.column is not None, "Required 'column' attribute missing from filter, when loading from file"
- self.column = int ( self.column )
+ column = elem.get( "column", None )
+ assert column is not None, "Required 'column' attribute missing from filter, when loading from file"
+ self.column = d_option.column_spec_to_index( column )
self.keep = string_as_bool( elem.get( "keep", 'True' ) )
def filter_options( self, options, trans, other_values ):
rval = []
@@ -81,11 +81,11 @@ class DataMetaFilter( Filter ):
d_option.has_dataset_dependencies = True
self.key = elem.get( "key", None )
assert self.key is not None, "Required 'key' attribute missing from filter"
- self.column = elem.get( "column", None )
- if self.column is None:
+ column = elem.get( "column", None )
+ if column is None:
assert self.dynamic_option.file_fields is None and self.dynamic_option.dataset_ref_name is None, "Required 'column' attribute missing from filter, when loading from file"
else:
- self.column = int ( self.column )
+ self.column = d_option.column_spec_to_index( column )
self.multiple = string_as_bool( elem.get( "multiple", "False" ) )
self.separator = elem.get( "separator", "," )
def get_dependency_name( self ):
@@ -141,9 +141,9 @@ class ParamValueFilter( Filter ):
Filter.__init__( self, d_option, elem )
self.ref_name = elem.get( "ref", None )
assert self.ref_name is not None, "Required 'ref' attribute missing from filter"
- self.column = elem.get( "column", None )
- assert self.column is not None, "Required 'column' attribute missing from filter"
- self.column = int ( self.column )
+ column = elem.get( "column", None )
+ assert column is not None, "Required 'column' attribute missing from filter"
+ self.column = d_option.column_spec_to_index( column )
self.keep = string_as_bool( elem.get( "keep", 'True' ) )
def get_dependency_name( self ):
return self.ref_name
@@ -168,9 +168,9 @@ class UniqueValueFilter( Filter ):
"""
def __init__( self, d_option, elem ):
Filter.__init__( self, d_option, elem )
- self.column = elem.get( "column", None )
- assert self.column is not None, "Required 'column' attribute missing from filter"
- self.column = int ( self.column )
+ column = elem.get( "column", None )
+ assert column is not None, "Required 'column' attribute missing from filter"
+ self.column = d_option.column_spec_to_index( column )
def get_dependency_name( self ):
return self.dynamic_option.dataset_ref_name
def filter_options( self, options, trans, other_values ):
@@ -196,9 +196,9 @@ class MultipleSplitterFilter( Filter ):
def __init__( self, d_option, elem ):
Filter.__init__( self, d_option, elem )
self.separator = elem.get( "separator", "," )
- self.columns = elem.get( "column", None )
- assert self.columns is not None, "Required 'columns' attribute missing from filter"
- self.columns = [ int ( column ) for column in self.columns.split( "," ) ]
+ columns = elem.get( "column", None )
+ assert columns is not None, "Required 'columns' attribute missing from filter"
+ self.columns = [ d_option.column_spec_to_index( column ) for column in columns.split( "," ) ]
def filter_options( self, options, trans, other_values ):
rval = []
for fields in options:
@@ -302,9 +302,9 @@ class SortByColumnFilter( Filter ):
"""
def __init__( self, d_option, elem ):
Filter.__init__( self, d_option, elem )
- self.column = elem.get( "column", None )
- assert self.column is not None, "Required 'column' attribute missing from filter"
- self.column = int( self.column )
+ column = elem.get( "column", None )
+ assert column is not None, "Required 'column' attribute missing from filter"
+ self.column = d_option.column_spec_to_index( column )
def filter_options( self, options, trans, other_values ):
rval = []
for i, fields in enumerate( options ):
@@ -354,20 +354,25 @@ class DynamicOptions( object ):
data_file = elem.get( 'from_file', None )
dataset_file = elem.get( 'from_dataset', None )
from_parameter = elem.get( 'from_parameter', None )
- if data_file is not None or dataset_file is not None or from_parameter is not None:
- for column_elem in elem.findall( 'column' ):
- name = column_elem.get( 'name', None )
- assert name is not None, "Required 'name' attribute missing from column def"
- index = column_elem.get( 'index', None )
- assert index is not None, "Required 'index' attribute missing from column def"
- index = int( index )
- self.columns[name] = index
- if index > self.largest_index:
- self.largest_index = index
- assert 'value' in self.columns, "Required 'value' column missing from column def"
- if 'name' not in self.columns:
- self.columns['name'] = self.columns['value']
+ tool_data_table_name = elem.get( 'from_data_table', None )
+
+ # Options are defined from a data table loaded by the app
+ self.tool_data_table = None
+ if tool_data_table_name:
+ app = tool_param.tool.app
+ assert tool_data_table_name in app.tool_data_tables, \
+ "Data table named '%s' is required by tool but not configured" % tool_data_table_name
+ self.tool_data_table = app.tool_data_tables[ tool_data_table_name ]
+ # Column definitions are optional, but if provided override those from the table
+ if elem.find( "column" ) is not None:
+ self.parse_column_definitions( elem )
+ else:
+ self.columns = self.tool_data_table.columns
+ # Options are defined by parsing tabular text data from an data file
+ # on disk, a dataset, or the value of another parameter
+ elif data_file is not None or dataset_file is not None or from_parameter is not None:
+ self.parse_column_definitions( elem )
if data_file is not None:
data_file = data_file.strip()
if not os.path.isabs( data_file ):
@@ -388,6 +393,20 @@ class DynamicOptions( object ):
# Load Validators
for validator in elem.findall( 'validator' ):
self.validators.append( validation.Validator.from_element( self.tool_param, validator ) )
+
+ def parse_column_definitions( self, elem ):
+ for column_elem in elem.findall( 'column' ):
+ name = column_elem.get( 'name', None )
+ assert name is not None, "Required 'name' attribute missing from column def"
+ index = column_elem.get( 'index', None )
+ assert index is not None, "Required 'index' attribute missing from column def"
+ index = int( index )
+ self.columns[name] = index
+ if index > self.largest_index:
+ self.largest_index = index
+ assert 'value' in self.columns, "Required 'value' column missing from column def"
+ if 'name' not in self.columns:
+ self.columns['name'] = self.columns['value']
def parse_file_fields( self, reader ):
rval = []
@@ -421,6 +440,8 @@ class DynamicOptions( object ):
assert dataset is not None, "Required dataset '%s' missing from input" % self.dataset_ref_name
if not dataset: return [] #no valid dataset in history
options = self.parse_file_fields( open( dataset.file_name ) )
+ elif self.tool_data_table:
+ options = self.tool_data_table.get_fields()
else:
options = list( self.file_fields )
for filter in self.filters:
@@ -429,7 +450,7 @@ class DynamicOptions( object ):
def get_options( self, trans, other_values ):
rval = []
- if self.file_fields is not None or self.dataset_ref_name is not None:
+ if self.file_fields is not None or self.tool_data_table is not None or self.dataset_ref_name is not None:
options = self.get_fields( trans, other_values )
for fields in options:
rval.append( ( fields[self.columns['name']], fields[self.columns['value']], False ) )
@@ -437,3 +458,15 @@ class DynamicOptions( object ):
for filter in self.filters:
rval = filter.filter_options( rval, trans, other_values )
return rval
+
+ def column_spec_to_index( self, column_spec ):
+ """
+ Convert a column specification (as read from the config file), to an
+ index. A column specification can just be a number, a column name, or
+ a column alias.
+ """
+ # Name?
+ if column_spec in self.columns:
+ return self.columns[column_spec]
+ # Int?
+ return int( column_spec )
--- a/tools/sr_mapping/bowtie_wrapper.xml
+++ b/tools/sr_mapping/bowtie_wrapper.xml
@@ -192,10 +192,13 @@
</param><when value="indexed"><param name="index" type="select" label="Select a reference genome" help="if your genome of interest is not listed - contact Galaxy team">
+ <options from_data_table="bowtie_indexes"/>
+ <!--
<options from_file="bowtie_indices.loc"><column name="value" index="1" /><column name="name" index="0" /></options>
+ --></param></when><when value="history">
--- a/lib/galaxy/util/__init__.py
+++ b/lib/galaxy/util/__init__.py
@@ -231,13 +231,17 @@ def rst_to_html( s ):
log.warn( str )
return docutils.core.publish_string( s, writer=HTMLFragWriter(), settings_overrides=dict( warning_stream=FakeStream() ) )
-def xml_text(root, name):
+def xml_text(root, name=None):
"""Returns the text inside an element"""
- # Try attribute first
- val = root.get(name)
- if val: return val
- # Then try as element
- elem = root.find(name)
+ if name is not None:
+ # Try attribute first
+ val = root.get(name)
+ if val:
+ return val
+ # Then try as element
+ elem = root.find(name)
+ else:
+ elem = root
if elem is not None and elem.text:
text = ''.join(elem.text.splitlines())
return text.strip()
--- a/setup.sh
+++ b/setup.sh
@@ -7,6 +7,7 @@ SAMPLES="
datatypes_conf.xml.sample
reports_wsgi.ini.sample
tool_conf.xml.sample
+tool_data_table_conf.xml.sample
universe_wsgi.ini.sample
tool-data/alignseq.loc.sample
tool-data/annotation_profiler_options.xml.sample
--- a/tools/sr_mapping/bwa_wrapper.xml
+++ b/tools/sr_mapping/bwa_wrapper.xml
@@ -34,10 +34,13 @@
</param><when value="indexed"><param name="indices" type="select" label="Select a reference genome">
+ <options from_data_table="bwa_indexes"/>
+ <!--
<options from_file="bwa_index.loc"><column name="value" index="1" /><column name="name" index="0" /></options>
+ --></param></when><when value="history">
--- a/lib/galaxy/app.py
+++ b/lib/galaxy/app.py
@@ -2,6 +2,7 @@ import sys, os, atexit
from galaxy import config, jobs, util, tools, web
import galaxy.tools.search
+import galaxy.tools.data
from galaxy.web import security
import galaxy.model
import galaxy.datatypes.registry
@@ -36,6 +37,8 @@ class UniverseApplication( object ):
self.security = security.SecurityHelper( id_secret=self.config.id_secret )
# Tag handler
self.tag_handler = GalaxyTagHandler()
+ # Tool data tables
+ self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_table_config_path )
# Initialize the tools
self.toolbox = tools.ToolBox( self.config.tool_config, self.config.tool_path, self )
# Search support for tools
--- a/lib/galaxy/config.py
+++ b/lib/galaxy/config.py
@@ -48,6 +48,7 @@ class Configuration( object ):
self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "tool-data" ), os.getcwd() )
self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root )
self.tool_config = resolve_path( kwargs.get( 'tool_config_file', 'tool_conf.xml' ), self.root )
+ self.tool_data_table_config_path = resolve_path( kwargs.get( 'tool_data_table_config_path', 'tool_data_table_conf.xml' ), self.root )
self.tool_secret = kwargs.get( "tool_secret", "" )
self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" )
self.set_metadata_externally = string_as_bool( kwargs.get( "set_metadata_externally", "False" ) )
--- a/tools/maf/interval2maf.xml
+++ b/tools/maf/interval2maf.xml
@@ -32,22 +32,24 @@
</when><when value="cached"><param name="mafType" type="select" label="Choose alignments">
- <options from_file="maf_index.loc">
+ <options from_data_table="indexed_maf_files">
+ <!--
<column name="name" index="0"/><column name="value" index="1"/><column name="dbkey" index="2"/><column name="species" index="3"/>
- <filter type="data_meta" ref="input1" key="dbkey" column="2" multiple="True" separator=","/>
+ -->
+ <filter type="data_meta" ref="input1" key="dbkey" column="dbkey" multiple="True" separator=","/><validator type="no_options" message="No alignments are available for the build associated with the selected interval file"/></options></param><param name="species" type="select" display="checkboxes" multiple="true" label="Choose species" help="Select species to be included in the final alignment">
- <options from_file="maf_index.loc">
+ <options from_data_table="indexed_maf_files"><column name="uid" index="1"/><column name="value" index="3"/><column name="name" index="3"/>
- <filter type="param_value" ref="mafType" name="uid" column="1"/>
- <filter type="multiple_splitter" column="3" separator=","/>
+ <filter type="param_value" ref="mafType" column="uid"/>
+ <filter type="multiple_splitter" column="name" separator=","/></options></param></when>
