1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/changeset/71031bf3105c/
changeset: 71031bf3105c
user: jgoecks
date: 2012-02-14 18:27:47
summary: Better error message when bias correction/sequence data cannot be used in Cufflinks/compare/merge/diff.
affected #: 5 files
diff -r 33c780b4c1455628ec135b0ba98e285efcb89ed8 -r 71031bf3105c6557256bf2957ef5565ad672355d tools/ngs_rna/cuffcompare_wrapper.py
--- a/tools/ngs_rna/cuffcompare_wrapper.py
+++ b/tools/ngs_rna/cuffcompare_wrapper.py
@@ -58,16 +58,21 @@
# Set/link to sequence file.
if options.use_seq_data:
- cached_seqs_pointer_file = os.path.join( options.index_dir, 'sam_fa_indices.loc' )
- if not os.path.exists( cached_seqs_pointer_file ):
- stop_err( 'The required file (%s) does not exist.' % cached_seqs_pointer_file )
- # If found for the dbkey, seq_path will look something like /galaxy/data/equCab2/sam_index/equCab2.fa,
- # and the equCab2.fa file will contain fasta sequences.
- seq_path = check_seq_file( options.dbkey, cached_seqs_pointer_file )
if options.ref_file != 'None':
+ # Sequence data from history.
# Create symbolic link to ref_file so that index will be created in working directory.
seq_path = "ref.fa"
os.symlink( options.ref_file, seq_path )
+ else:
+ # Sequence data from loc file.
+ cached_seqs_pointer_file = os.path.join( options.index_dir, 'sam_fa_indices.loc' )
+ if not os.path.exists( cached_seqs_pointer_file ):
+ stop_err( 'The required file (%s) does not exist.' % cached_seqs_pointer_file )
+ # If found for the dbkey, seq_path will look something like /galaxy/data/equCab2/sam_index/equCab2.fa,
+ # and the equCab2.fa file will contain fasta sequences.
+ seq_path = check_seq_file( options.dbkey, cached_seqs_pointer_file )
+ if seq_path == '':
+ stop_err( 'No sequence data found for dbkey %s, so sequence data cannot be used.' % options.dbkey )
# Build command.
diff -r 33c780b4c1455628ec135b0ba98e285efcb89ed8 -r 71031bf3105c6557256bf2957ef5565ad672355d tools/ngs_rna/cuffdiff_wrapper.py
--- a/tools/ngs_rna/cuffdiff_wrapper.py
+++ b/tools/ngs_rna/cuffdiff_wrapper.py
@@ -121,16 +121,21 @@
# If doing bias correction, set/link to sequence file.
if options.do_bias_correction:
- cached_seqs_pointer_file = os.path.join( options.index_dir, 'sam_fa_indices.loc' )
- if not os.path.exists( cached_seqs_pointer_file ):
- stop_err( 'The required file (%s) does not exist.' % cached_seqs_pointer_file )
- # If found for the dbkey, seq_path will look something like /galaxy/data/equCab2/sam_index/equCab2.fa,
- # and the equCab2.fa file will contain fasta sequences.
- seq_path = check_seq_file( options.dbkey, cached_seqs_pointer_file )
if options.ref_file != 'None':
+ # Sequence data from history.
# Create symbolic link to ref_file so that index will be created in working directory.
seq_path = os.path.join( tmp_output_dir, "ref.fa" )
os.symlink( options.ref_file, seq_path )
+ else:
+ # Sequence data from loc file.
+ cached_seqs_pointer_file = os.path.join( options.index_dir, 'sam_fa_indices.loc' )
+ if not os.path.exists( cached_seqs_pointer_file ):
+ stop_err( 'The required file (%s) does not exist.' % cached_seqs_pointer_file )
+ # If found for the dbkey, seq_path will look something like /galaxy/data/equCab2/sam_index/equCab2.fa,
+ # and the equCab2.fa file will contain fasta sequences.
+ seq_path = check_seq_file( options.dbkey, cached_seqs_pointer_file )
+ if seq_path == '':
+ stop_err( 'No sequence data found for dbkey %s, so bias correction cannot be used.' % options.dbkey )
# Build command.
diff -r 33c780b4c1455628ec135b0ba98e285efcb89ed8 -r 71031bf3105c6557256bf2957ef5565ad672355d tools/ngs_rna/cuffdiff_wrapper.xml
--- a/tools/ngs_rna/cuffdiff_wrapper.xml
+++ b/tools/ngs_rna/cuffdiff_wrapper.xml
@@ -96,8 +96,8 @@
</param><conditional name="bias_correction"><param name="do_bias_correction" type="select" label="Perform Bias Correction" help="Bias detection and correction can significantly improve accuracy of transcript abundance estimates.">
+ <option value="No">No</option><option value="Yes">Yes</option>
- <option value="No">No</option></param><when value="Yes"><conditional name="seq_source">
diff -r 33c780b4c1455628ec135b0ba98e285efcb89ed8 -r 71031bf3105c6557256bf2957ef5565ad672355d tools/ngs_rna/cufflinks_wrapper.py
--- a/tools/ngs_rna/cufflinks_wrapper.py
+++ b/tools/ngs_rna/cufflinks_wrapper.py
@@ -79,16 +79,21 @@
# If doing bias correction, set/link to sequence file.
if options.do_bias_correction:
- cached_seqs_pointer_file = os.path.join( options.index_dir, 'sam_fa_indices.loc' )
- if not os.path.exists( cached_seqs_pointer_file ):
- stop_err( 'The required file (%s) does not exist.' % cached_seqs_pointer_file )
- # If found for the dbkey, seq_path will look something like /galaxy/data/equCab2/sam_index/equCab2.fa,
- # and the equCab2.fa file will contain fasta sequences.
- seq_path = check_seq_file( options.dbkey, cached_seqs_pointer_file )
if options.ref_file != 'None':
+ # Sequence data from history.
# Create symbolic link to ref_file so that index will be created in working directory.
seq_path = "ref.fa"
- os.symlink( options.ref_file, seq_path )
+ os.symlink( options.ref_file, seq_path )
+ else:
+ # Sequence data from loc file.
+ cached_seqs_pointer_file = os.path.join( options.index_dir, 'sam_fa_indices.loc' )
+ if not os.path.exists( cached_seqs_pointer_file ):
+ stop_err( 'The required file (%s) does not exist.' % cached_seqs_pointer_file )
+ # If found for the dbkey, seq_path will look something like /galaxy/data/equCab2/sam_index/equCab2.fa,
+ # and the equCab2.fa file will contain fasta sequences.
+ seq_path = check_seq_file( options.dbkey, cached_seqs_pointer_file )
+ if seq_path == '':
+ stop_err( 'No sequence data found for dbkey %s, so bias correction cannot be used.' % options.dbkey )
# Build command.
diff -r 33c780b4c1455628ec135b0ba98e285efcb89ed8 -r 71031bf3105c6557256bf2957ef5565ad672355d tools/ngs_rna/cuffmerge_wrapper.py
--- a/tools/ngs_rna/cuffmerge_wrapper.py
+++ b/tools/ngs_rna/cuffmerge_wrapper.py
@@ -59,16 +59,21 @@
# Set/link to sequence file.
if options.use_seq_data:
- cached_seqs_pointer_file = os.path.join( options.index_dir, 'sam_fa_indices.loc' )
- if not os.path.exists( cached_seqs_pointer_file ):
- stop_err( 'The required file (%s) does not exist.' % cached_seqs_pointer_file )
- # If found for the dbkey, seq_path will look something like /galaxy/data/equCab2/sam_index/equCab2.fa,
- # and the equCab2.fa file will contain fasta sequences.
- seq_path = check_seq_file( options.dbkey, cached_seqs_pointer_file )
if options.ref_file != 'None':
+ # Sequence data from history.
# Create symbolic link to ref_file so that index will be created in working directory.
seq_path = "ref.fa"
os.symlink( options.ref_file, seq_path )
+ else:
+ # Sequence data from loc file.
+ cached_seqs_pointer_file = os.path.join( options.index_dir, 'sam_fa_indices.loc' )
+ if not os.path.exists( cached_seqs_pointer_file ):
+ stop_err( 'The required file (%s) does not exist.' % cached_seqs_pointer_file )
+ # If found for the dbkey, seq_path will look something like /galaxy/data/equCab2/sam_index/equCab2.fa,
+ # and the equCab2.fa file will contain fasta sequences.
+ seq_path = check_seq_file( options.dbkey, cached_seqs_pointer_file )
+ if seq_path == '':
+ stop_err( 'No sequence data found for dbkey %s, so sequence data cannot be used.' % options.dbkey )
# Build command.
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/changeset/a00a36e7cf4c/
changeset: a00a36e7cf4c
user: greg
date: 2012-02-10 22:33:19
summary: Do not allow conflicting datatypes included in a tool shed repository currently being installed to override datattypes already in the datatypes registry. Also, a bit of cleanup in handling datatypes included in repositories being activated / installed / deactivated / uninstalled. A bit more cleanup is still necessary.
affected #: 3 files
diff -r 62fc9e05383514811e5add9db5f9ad6560a06b20 -r a00a36e7cf4c3b73a0ee2a411b065213f19cf3df lib/galaxy/datatypes/registry.py
--- a/lib/galaxy/datatypes/registry.py
+++ b/lib/galaxy/datatypes/registry.py
@@ -47,11 +47,13 @@
self.datatype_elems = []
self.sniffer_elems = []
self.xml_filename = None
- def load_datatypes( self, root_dir=None, config=None, deactivate=False ):
+ def load_datatypes( self, root_dir=None, config=None, deactivate=False, override=True ):
"""
- Parse a datatypes XML file located at root_dir/config. If deactivate is True, an installed
- tool shed repository that includes proprietary datatypes is being deactivated, so appropriate
- loaded datatypes will be removed from the registry.
+ Parse a datatypes XML file located at root_dir/config. If deactivate is True, an installed tool shed
+ repository that includes proprietary datatypes is being deactivated, so appropriate loaded datatypes
+ will be removed from the registry. The value of override will be False when a tool shed repository is
+ being installed. Since installation is occurring after the datatypes registry has been initialized, its
+ contents cannot be overridden by new introduced conflicting data types.
"""
def __import_module( full_path, datatype_module ):
sys.path.insert( 0, full_path )
@@ -116,9 +118,13 @@
self.datatype_elems.append( elem )
if extension and extension in self.datatypes_by_extension and deactivate:
# We are deactivating an installed tool shed repository, so eliminate the datatype from the registry.
+ # TODO: Handle deactivating datatype converters, etc before removing from self.datatypes_by_extension.
self.log.debug( "Removing datatype with extension '%s' from the registry." % extension )
del self.datatypes_by_extension[ extension ]
- elif extension and ( dtype or type_extension ):
+ can_process_datatype = False
+ else:
+ can_process_datatype = ( extension and ( dtype or type_extension ) ) and ( extension not in self.datatypes_by_extension or override )
+ if can_process_datatype:
if dtype:
fields = dtype.split( ':' )
datatype_module = fields[0]
@@ -194,6 +200,11 @@
else:
if elem not in self.display_app_containers:
self.display_app_containers.append( elem )
+ elif not deactivate:
+ # A new tool shed repository that contains proprietary datatypes is being installed, and since installation
+ # is occurring after the datatypes registry has been initialized, its contents cannot be overridden by new
+ # introduced conflicting data types.
+ self.log.warning( "Ignoring conflicting datatype with extension '%s' from %s." % ( extension, config ) )
except Exception, e:
if deactivate:
self.log.warning( "Error deactivating datatype with extension '%s': %s" % ( extension, str( e ) ) )
@@ -233,14 +244,22 @@
self.log.debug( "Deactivated sniffer for datatype '%s'" % dtype )
else:
# See if we have a conflicting sniffer already loaded.
+ conflict = False
for conflict_loc, sniffer_class in enumerate( self.sniff_order ):
if sniffer_class.__class__ == aclass.__class__:
# We have a conflicting sniffer, so replace the one previously loaded.
- del self.sniff_order[ conflict_loc ]
- self.log.debug( "Replaced conflicting sniffer for datatype '%s'" % dtype )
+ conflict = True
+ if override:
+ del self.sniff_order[ conflict_loc ]
+ self.log.debug( "Replaced conflicting sniffer for datatype '%s'" % dtype )
break
- self.sniff_order.append( aclass )
- self.log.debug( "Loaded sniffer for datatype '%s'" % dtype )
+ if conflict:
+ if override:
+ self.sniff_order.append( aclass )
+ self.log.debug( "Loaded sniffer for datatype '%s'" % dtype )
+ else:
+ self.sniff_order.append( aclass )
+ self.log.debug( "Loaded sniffer for datatype '%s'" % dtype )
except Exception, exc:
if deactivate:
self.log.warning( "Error deactivating sniffer for datatype '%s': %s" % ( dtype, str( exc ) ) )
diff -r 62fc9e05383514811e5add9db5f9ad6560a06b20 -r a00a36e7cf4c3b73a0ee2a411b065213f19cf3df lib/galaxy/util/shed_util.py
--- a/lib/galaxy/util/shed_util.py
+++ b/lib/galaxy/util/shed_util.py
@@ -527,12 +527,14 @@
if display_path:
# Load or deactivate proprietary datatype display applications
app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict, deactivate=deactivate )
-def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=False ):
+def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=False, override=True ):
"""
Parse a proprietary datatypes config (a datatypes_conf.xml file included in an installed tool shed repository) and
add information to appropriate elements that will enable proprietary datatype class modules, datatypes converters
- and display application to be discovered and properly imported by the datatypes registry. This method is used by
- the InstallManager, which does not have access to trans.
+ and display application to be discovered and properly imported by the datatypes registry. The value of override will
+ be False when a tool shed repository is being installed. Since installation is occurring after the datatypes registry
+ has been initialized, its contents cannot be overridden by conflicting data types. This method is used by the InstallManager,
+ which does not have access to trans.
"""
tree = util.parse_xml( datatypes_config )
datatypes_config_root = tree.getroot()
@@ -597,7 +599,7 @@
else:
proprietary_datatypes_config = datatypes_config
# Load proprietary datatypes
- app.datatypes_registry.load_datatypes( root_dir=app.config.root, config=proprietary_datatypes_config, deactivate=deactivate )
+ app.datatypes_registry.load_datatypes( root_dir=app.config.root, config=proprietary_datatypes_config, deactivate=deactivate, override=override )
try:
os.unlink( proprietary_datatypes_config )
except:
@@ -728,7 +730,8 @@
if 'datatypes_config' in metadata_dict:
datatypes_config = os.path.abspath( metadata_dict[ 'datatypes_config' ] )
# Load data types required by tools.
- converter_path, display_path = alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir )
+ override = not new_install
+ converter_path, display_path = alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, override=override )
if converter_path or display_path:
# Create a dictionary of tool shed repository related information.
repository_dict = create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed,
diff -r 62fc9e05383514811e5add9db5f9ad6560a06b20 -r a00a36e7cf4c3b73a0ee2a411b065213f19cf3df lib/galaxy/web/controllers/admin_toolshed.py
--- a/lib/galaxy/web/controllers/admin_toolshed.py
+++ b/lib/galaxy/web/controllers/admin_toolshed.py
@@ -616,9 +616,6 @@
break
for elem in sections_to_load:
trans.app.toolbox.load_section_tag_set( elem, trans.app.toolbox.tool_panel, tool_path )
- if repository.includes_datatypes:
- # Load proprietary datatypes.
- load_datatype_items( trans.app, repository, relative_install_dir )
if uninstalled:
message = 'The <b>%s</b> repository has been reinstalled.' % repository.name
else:
@@ -656,8 +653,6 @@
if tool_section:
trans.app.toolbox.tool_panel[ section_key ] = tool_section
log.debug( "Appended reactivated tool to section: %s" % tool_section.name )
- shed_tool_conf, tool_path, relative_install_dir = self.__get_tool_path_and_relative_install_dir( trans, repository )
- load_datatype_items( trans.app, repository, relative_install_dir )
@web.expose
@web.require_admin
def manage_repository( self, trans, **kwd ):
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/changeset/62fc9e053835/
changeset: 62fc9e053835
user: greg
date: 2012-02-10 20:52:41
summary: Fixes for handling proprietary datatypes included in installed tool shed repositories - import proprietary class modules within the registry.
affected #: 2 files
diff -r ce17eb369f95ccaf2c8331e1a69fe34b1203cff1 -r 62fc9e05383514811e5add9db5f9ad6560a06b20 lib/galaxy/datatypes/registry.py
--- a/lib/galaxy/datatypes/registry.py
+++ b/lib/galaxy/datatypes/registry.py
@@ -1,8 +1,7 @@
"""
Provides mapping between extensions and datatypes, mime-types, etc.
"""
-import os, tempfile
-import logging
+import os, sys, tempfile, threading, logging
import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks, chrominfo, binary, assembly, ngsindex, wsf
import galaxy.util
from galaxy.util.odict import odict
@@ -20,8 +19,7 @@
self.datatype_converters = odict()
# Converters defined in local datatypes_conf.xml
self.converters = []
- # Converters defined in datatypes_conf.xml included
- # in installed tool shed repositories.
+ # Converters defined in datatypes_conf.xml included in installed tool shed repositories.
self.proprietary_converters = []
self.converter_deps = {}
self.available_tracks = []
@@ -44,17 +42,24 @@
# The 'default' display_path defined in local datatypes_conf.xml
self.display_applications_path = None
self.inherit_display_application_by_class = []
+ # Keep a list of imported proprietary datatype class modules.
+ self.imported_modules = []
self.datatype_elems = []
self.sniffer_elems = []
self.xml_filename = None
- def load_datatypes( self, root_dir=None, config=None, imported_modules=None, deactivate=False ):
+ def load_datatypes( self, root_dir=None, config=None, deactivate=False ):
"""
- Parse a datatypes XML file located at root_dir/config. If imported_modules is received, it
- is a list of imported datatypes class files included in an installed tool shed repository.
- If deactivate is received as True, an installed tool shed repository that includes proprietary
- datatypes is being deactivated, so relevant loaded datatypes will be removed from the registry.
+ Parse a datatypes XML file located at root_dir/config. If deactivate is True, an installed
+ tool shed repository that includes proprietary datatypes is being deactivated, so appropriate
+ loaded datatypes will be removed from the registry.
"""
+ def __import_module( full_path, datatype_module ):
+ sys.path.insert( 0, full_path )
+ imported_module = __import__( datatype_module )
+ sys.path.pop( 0 )
+ return imported_module
if root_dir and config:
+ handling_proprietary_datatypes = False
# Parse datatypes_conf.xml
tree = galaxy.util.parse_xml( config )
root = tree.getroot()
@@ -73,6 +78,11 @@
if not self.display_applications_path:
self.display_path_attr = registration.get( 'display_path', 'display_applications' )
self.display_applications_path = os.path.join( root_dir, self.display_path_attr )
+ # Proprietary datatype's <registration> tag may have special attributes, proprietary_converter_path and proprietary_display_path.
+ proprietary_converter_path = registration.get( 'proprietary_converter_path', None )
+ proprietary_display_path = registration.get( 'proprietary_display_path', None )
+ if proprietary_converter_path or proprietary_display_path and not handling_proprietary_datatypes:
+ handling_proprietary_datatypes = True
for elem in registration.findall( 'datatype' ):
try:
extension = elem.get( 'extension', None )
@@ -81,6 +91,12 @@
mimetype = elem.get( 'mimetype', None )
display_in_upload = elem.get( 'display_in_upload', False )
make_subclass = galaxy.util.string_as_bool( elem.get( 'subclass', False ) )
+ # Proprietary datatypes included in installed tool shed repositories will include two special attributes
+ # (proprietary_path and proprietary_datatype_module) if they depend on proprietary datatypes classes.
+ proprietary_path = elem.get( 'proprietary_path', None )
+ proprietary_datatype_module = elem.get( 'proprietary_datatype_module', None )
+ if proprietary_path or proprietary_datatype_module and not handling_proprietary_datatypes:
+ handling_proprietary_datatypes = True
if deactivate:
# We are deactivating an installed tool shed repository, so eliminate the
# datatype elem from the in-memory list of datatype elems.
@@ -108,12 +124,21 @@
datatype_module = fields[0]
datatype_class_name = fields[1]
datatype_class = None
- if imported_modules:
- # See if one of the imported modules contains the datatype class name.
- for imported_module in imported_modules:
+ if proprietary_path and proprietary_datatype_module:
+ # We need to change the value of sys.path, so do it in a way that is thread-safe.
+ lock = threading.Lock()
+ lock.acquire( True )
+ try:
+ imported_module = __import_module( proprietary_path, proprietary_datatype_module )
+ if imported_module not in self.imported_modules:
+ self.imported_modules.append( imported_module )
if hasattr( imported_module, datatype_class_name ):
datatype_class = getattr( imported_module, datatype_class_name )
- break
+ except Exception, e:
+ full_path = os.path.join( full_path, proprietary_datatype_module )
+ self.log.debug( "Exception importing proprietary code file %s: %s" % ( str( full_path ), str( e ) ) )
+ finally:
+ lock.release()
if datatype_class is None:
# The datatype class name must be contained in one of the datatype modules in the Galaxy distribution.
fields = datatype_module.split( '.' )
@@ -130,14 +155,14 @@
self.datatypes_by_extension[ extension ] = datatype_class()
if mimetype is None:
# Use default mime type as per datatype spec
- mimetype = self.datatypes_by_extension[extension].get_mime()
- self.mimetypes_by_extension[extension] = mimetype
+ mimetype = self.datatypes_by_extension[ extension ].get_mime()
+ self.mimetypes_by_extension[ extension ] = mimetype
if hasattr( datatype_class, "get_track_type" ):
self.available_tracks.append( extension )
if display_in_upload:
self.upload_file_formats.append( extension )
# Max file size cut off for setting optional metadata
- self.datatypes_by_extension[extension].max_optional_metadata_filesize = elem.get( 'max_optional_metadata_filesize', None )
+ self.datatypes_by_extension[ extension ].max_optional_metadata_filesize = elem.get( 'max_optional_metadata_filesize', None )
for converter in elem.findall( 'converter' ):
# Build the list of datatype converters which will later be loaded into the calling app's toolbox.
converter_config = converter.get( 'file', None )
@@ -148,7 +173,8 @@
self.converter_deps[extension] = {}
self.converter_deps[extension][target_datatype] = depends_on.split(',')
if converter_config and target_datatype:
- if imported_modules:
+ #if imported_modules:
+ if proprietary_converter_path:
self.proprietary_converters.append( ( converter_config, extension, target_datatype ) )
else:
self.converters.append( ( converter_config, extension, target_datatype ) )
@@ -161,7 +187,8 @@
mimetype = composite_file.get( 'mimetype', None )
self.datatypes_by_extension[extension].add_composite_file( name, optional=optional, mimetype=mimetype )
for display_app in elem.findall( 'display' ):
- if imported_modules:
+ #if imported_modules:
+ if proprietary_display_path:
if elem not in self.proprietary_display_app_containers:
self.proprietary_display_app_containers.append( elem )
else:
@@ -185,9 +212,10 @@
datatype_module = fields[0]
datatype_class_name = fields[1]
module = None
- if imported_modules:
+ #if imported_modules:
+ if handling_proprietary_datatypes:
# See if one of the imported modules contains the datatype class name.
- for imported_module in imported_modules:
+ for imported_module in self.imported_modules:
if hasattr( imported_module, datatype_class_name ):
module = imported_module
break
@@ -197,13 +225,6 @@
for comp in datatype_module.split( '.' )[ 1: ]:
module = getattr( module, comp )
aclass = getattr( module, datatype_class_name )()
- # See if we have a conflicting sniffer already loaded.
- conflict_loc = None
- conflict = False
- for conflict_loc, sniffer_class in enumerate( self.sniff_order ):
- if sniffer_class.__class__ == aclass.__class__:
- conflict = True
- break
if deactivate:
for sniffer_class in self.sniff_order:
if sniffer_class.__class__ == aclass.__class__:
@@ -211,19 +232,21 @@
break
self.log.debug( "Deactivated sniffer for datatype '%s'" % dtype )
else:
- if conflict:
- # We have a conflicting sniffer, so replace the one previously loaded.
- del self.sniff_order[ conflict_loc ]
- self.sniff_order.append( aclass )
- self.log.debug( "Replaced conflicting sniffer for datatype '%s'" % dtype )
- else:
- self.sniff_order.append( aclass )
- self.log.debug( "Loaded sniffer for datatype '%s'" % dtype )
+ # See if we have a conflicting sniffer already loaded.
+ for conflict_loc, sniffer_class in enumerate( self.sniff_order ):
+ if sniffer_class.__class__ == aclass.__class__:
+ # We have a conflicting sniffer, so replace the one previously loaded.
+ del self.sniff_order[ conflict_loc ]
+ self.log.debug( "Replaced conflicting sniffer for datatype '%s'" % dtype )
+ break
+ self.sniff_order.append( aclass )
+ self.log.debug( "Loaded sniffer for datatype '%s'" % dtype )
except Exception, exc:
if deactivate:
self.log.warning( "Error deactivating sniffer for datatype '%s': %s" % ( dtype, str( exc ) ) )
else:
self.log.warning( "Error appending sniffer for datatype '%s' to sniff_order: %s" % ( dtype, str( exc ) ) )
+ self.upload_file_formats.sort()
# Persist the xml form of the registry into a temporary file so that it
# can be loaded from the command line by tools and set_metadata processing.
self.to_xml_file()
@@ -385,8 +408,7 @@
app's toolbox.
"""
if installed_repository_dict:
- # Load converters defined by datatypes_conf.xml
- # included in installed tool shed repository.
+ # Load converters defined by datatypes_conf.xml included in installed tool shed repository.
converters = self.proprietary_converters
else:
# Load converters defined by local datatypes_conf.xml.
@@ -438,10 +460,9 @@
If deactivate is False, add display applications from self.display_app_containers or
self.proprietary_display_app_containers to appropriate datatypes. If deactivate is
True, eliminates relevant display applications from appropriate datatypes.
- """
+ """
if installed_repository_dict:
- # Load display applications defined by datatypes_conf.xml
- # included in installed tool shed repository.
+ # Load display applications defined by datatypes_conf.xml included in installed tool shed repository.
datatype_elems = self.proprietary_display_app_containers
else:
# Load display applications defined by local datatypes_conf.xml.
@@ -452,7 +473,8 @@
display_file = display_app.get( 'file', None )
if installed_repository_dict:
display_path = installed_repository_dict[ 'display_path' ]
- config_path = os.path.join( display_path, display_file )
+ display_file_head, display_file_tail = os.path.split( display_file )
+ config_path = os.path.join( display_path, display_file_tail )
else:
config_path = os.path.join( self.display_applications_path, display_file )
try:
diff -r ce17eb369f95ccaf2c8331e1a69fe34b1203cff1 -r 62fc9e05383514811e5add9db5f9ad6560a06b20 lib/galaxy/util/shed_util.py
--- a/lib/galaxy/util/shed_util.py
+++ b/lib/galaxy/util/shed_util.py
@@ -1,4 +1,4 @@
-import os, sys, tempfile, shutil, subprocess, threading, logging
+import os, tempfile, shutil, subprocess, logging
from datetime import date, datetime, timedelta
from time import strftime
from galaxy import util
@@ -511,7 +511,7 @@
metadata = repository.metadata
datatypes_config = metadata.get( 'datatypes_config', None )
if datatypes_config:
- converter_path, display_path = load_datatypes( app, datatypes_config, relative_install_dir, deactivate=deactivate )
+ converter_path, display_path = alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=deactivate )
if converter_path or display_path:
# Create a dictionary of tool shed repository related information.
repository_dict = create_repository_dict_for_proprietary_datatypes( tool_shed=repository.tool_shed,
@@ -527,15 +527,13 @@
if display_path:
# Load or deactivate proprietary datatype display applications
app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict, deactivate=deactivate )
-def load_datatypes( app, datatypes_config, relative_install_dir, deactivate=False ):
- # This method is used by the InstallManager, which does not have access to trans.
- def __import_module( relative_path, datatype_module ):
- sys.path.insert( 0, relative_path )
- imported_module = __import__( datatype_module )
- sys.path.pop( 0 )
- return imported_module
- imported_modules = []
- # Parse datatypes_config.
+def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=False ):
+ """
+ Parse a proprietary datatypes config (a datatypes_conf.xml file included in an installed tool shed repository) and
+ add information to appropriate elements that will enable proprietary datatype class modules, datatypes converters
+ and display application to be discovered and properly imported by the datatypes registry. This method is used by
+ the InstallManager, which does not have access to trans.
+ """
tree = util.parse_xml( datatypes_config )
datatypes_config_root = tree.getroot()
# Path to datatype converters
@@ -551,6 +549,7 @@
# <datatype_file name="gmap.py"/>
# <datatype_file name="metagenomics.py"/>
# </datatype_files>
+ # We'll add attributes to the datatype tag sets so that the modules can be properly imported by the datatypes registry.
for elem in datatype_files.findall( 'datatype_file' ):
datatype_file_name = elem.get( 'name', None )
if datatype_file_name:
@@ -563,76 +562,91 @@
break
break
if datatype_class_modules:
- # Import each of the datatype class modules.
+ registration = datatypes_config_root.find( 'registration' )
+ converter_path, display_path = get_converter_and_display_paths( registration, relative_install_dir )
+ if converter_path:
+ registration.attrib[ 'proprietary_converter_path' ] = converter_path
+ if display_path:
+ registration.attrib[ 'proprietary_display_path' ] = display_path
for relative_path_to_datatype_file_name in datatype_class_modules:
relative_head, relative_tail = os.path.split( relative_path_to_datatype_file_name )
- registration = datatypes_config_root.find( 'registration' )
- # Get the module by parsing the <datatype> tag.
for elem in registration.findall( 'datatype' ):
- # A 'type' attribute is currently required. The attribute
- # should be something like one of the following:
+ # Handle 'type' attribute which should be something like one of the following:
# type="gmap:GmapDB"
# type="galaxy.datatypes.gmap:GmapDB"
dtype = elem.get( 'type', None )
if dtype:
fields = dtype.split( ':' )
- datatype_module = fields[ 0 ]
- if datatype_module.find( '.' ) >= 0:
- # Handle the case where datatype_module is "galaxy.datatypes.gmap"
- datatype_module = datatype_module.split( '.' )[ -1 ]
- datatype_class_name = fields[ 1 ]
- # We need to change the value of sys.path, so do it in a way that is thread-safe.
- lock = threading.Lock()
- lock.acquire( True )
- try:
- imported_module = __import_module( relative_head, datatype_module )
- if imported_module not in imported_modules:
- imported_modules.append( imported_module )
- except Exception, e:
- log.debug( "Exception importing datatypes code file %s: %s" % ( str( relative_path_to_datatype_file_name ), str( e ) ) )
- finally:
- lock.release()
- # Handle data type converters and display applications.
- for elem in registration.findall( 'datatype' ):
- if not converter_path:
- # If any of the <datatype> tag sets contain <converter> tags, set the converter_path
- # if it is not already set. This requires developers to place all converters in the
- # same subdirectory within the repository hierarchy.
- for converter in elem.findall( 'converter' ):
- converter_config = converter.get( 'file', None )
- if converter_config:
- for root, dirs, files in os.walk( relative_install_dir ):
- if root.find( '.hg' ) < 0:
- for name in files:
- if name == converter_config:
- converter_path = root
- break
- if converter_path:
- break
- if not display_path:
- # If any of the <datatype> tag sets contain <display> tags, set the display_path
- # if it is not already set. This requires developers to place all display acpplications
- # in the same subdirectory within the repository hierarchy.
- for display_app in elem.findall( 'display' ):
- display_config = display_app.get( 'file', None )
- if display_config:
- for root, dirs, files in os.walk( relative_install_dir ):
- if root.find( '.hg' ) < 0:
- for name in files:
- if name == display_config:
- display_path = root
- break
- if display_path:
- break
- if converter_path and display_path:
- break
+ proprietary_datatype_module = fields[ 0 ]
+ if proprietary_datatype_module.find( '.' ) >= 0:
+ # Handle the case where datatype_module is "galaxy.datatypes.gmap".
+ proprietary_datatype_module = proprietary_datatype_module.split( '.' )[ -1 ]
+ # The value of proprietary_path must be an absolute path due to job_working_directory.
+ elem.attrib[ 'proprietary_path' ] = os.path.abspath( relative_head )
+ elem.attrib[ 'proprietary_datatype_module' ] = proprietary_datatype_module
+
+ sniffers = datatypes_config_root.find( 'sniffers' )
+ fd, proprietary_datatypes_config = tempfile.mkstemp()
+ os.write( fd, '<?xml version="1.0"?>\n' )
+ os.write( fd, '<datatypes>\n' )
+ os.write( fd, '%s' % util.xml_to_string( registration ) )
+ os.write( fd, '%s' % util.xml_to_string( sniffers ) )
+ os.write( fd, '</datatypes>\n' )
+ os.close( fd )
+ os.chmod( proprietary_datatypes_config, 0644 )
else:
- # The repository includes a dataypes_conf.xml file, but no code file that
- # contains data type classes. This implies that the data types in datayptes_conf.xml
- # are all subclasses of data types that are in the distribution.
- imported_modules = []
+ proprietary_datatypes_config = datatypes_config
# Load proprietary datatypes
- app.datatypes_registry.load_datatypes( root_dir=app.config.root, config=datatypes_config, imported_modules=imported_modules, deactivate=deactivate )
+ app.datatypes_registry.load_datatypes( root_dir=app.config.root, config=proprietary_datatypes_config, deactivate=deactivate )
+ try:
+ os.unlink( proprietary_datatypes_config )
+ except:
+ pass
+ return converter_path, display_path
+def get_converter_and_display_paths( registration_elem, relative_install_dir ):
+ """
+ Find the relative path to data type converters and display
+ applications included in installed tool shed repositories.
+ """
+ converter_path = None
+ display_path = None
+ for elem in registration_elem.findall( 'datatype' ):
+ if not converter_path:
+ # If any of the <datatype> tag sets contain <converter> tags, set the converter_path
+ # if it is not already set. This requires developers to place all converters in the
+ # same subdirectory within the repository hierarchy.
+ for converter in elem.findall( 'converter' ):
+ converter_config = converter.get( 'file', None )
+ if converter_config:
+ relative_head, relative_tail = os.path.split( converter_config )
+ for root, dirs, files in os.walk( relative_install_dir ):
+ if root.find( '.hg' ) < 0:
+ for name in files:
+ if name == relative_tail:
+ # The value of converter_path must be absolute due to job_working_directory.
+ converter_path = os.path.abspath( root )
+ break
+ if converter_path:
+ break
+ if not display_path:
+ # If any of the <datatype> tag sets contain <display> tags, set the display_path
+ # if it is not already set. This requires developers to place all display acpplications
+ # in the same subdirectory within the repository hierarchy.
+ for display_app in elem.findall( 'display' ):
+ display_config = display_app.get( 'file', None )
+ if display_config:
+ relative_head, relative_tail = os.path.split( display_config )
+ for root, dirs, files in os.walk( relative_install_dir ):
+ if root.find( '.hg' ) < 0:
+ for name in files:
+ if name == relative_tail:
+ # The value of display_path must be absolute due to job_working_directory.
+ display_path = os.path.abspath( root )
+ break
+ if display_path:
+ break
+ if converter_path and display_path:
+ break
return converter_path, display_path
def load_repository_contents( app, repository_name, description, owner, changeset_revision, tool_path, repository_clone_url, relative_install_dir,
current_working_dir, tmp_name, tool_shed=None, tool_section=None, shed_tool_conf=None, new_install=True, dist_to_shed=False ):
@@ -714,7 +728,7 @@
if 'datatypes_config' in metadata_dict:
datatypes_config = os.path.abspath( metadata_dict[ 'datatypes_config' ] )
# Load data types required by tools.
- converter_path, display_path = load_datatypes( app, datatypes_config, relative_install_dir )
+ converter_path, display_path = alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir )
if converter_path or display_path:
# Create a dictionary of tool shed repository related information.
repository_dict = create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed,
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
3 new commits in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/changeset/eb7a31e38a4c/
changeset: eb7a31e38a4c
branch: dynamic_options_memory_fix
user: jmchilton
date: 2012-02-07 17:35:17
summary: Patch dynamic_options.py to just read just the first megabyte of an
input dataset (if it is larger than that). Otherwise Galaxy will
attempt the read the entire file into memory, this can causes crashes
and freezeups without the user even selecting a bad input (for
instance, if a large FASTQ file is the top most history item and a txt
input is sought, Galaxy will freezeup as soon as the tool is clicked
because it will try to load the options for the top item immediately
by default).
Though the implementation is not as clean, I am restricting it to a
megabyte instead of a number of lines/options in case the top most history
item has no line breaks. I imagine in that case the same problem could
occur even if the input is restricted to a reasonable number of lines
(say 100).
affected #: 1 file
diff -r 74b6319b38b4b3876d0b81ef1296bb5afc729cc1 -r eb7a31e38a4cf9968d115984ab2ded908d3ddc9e lib/galaxy/tools/parameters/dynamic_options.py
--- a/lib/galaxy/tools/parameters/dynamic_options.py
+++ b/lib/galaxy/tools/parameters/dynamic_options.py
@@ -500,7 +500,16 @@
dataset = other_values.get( self.dataset_ref_name, None )
assert dataset is not None, "Required dataset '%s' missing from input" % self.dataset_ref_name
if not dataset: return [] #no valid dataset in history
- options = self.parse_file_fields( open( dataset.file_name ) )
+ # Ensure parsing dynamic options does not consume more than a megabyte worth memory.
+ file_size = os.path.getsize( path )
+ if os.path.getsize( path ) < 1048576:
+ options = self.parse_file_fields( open( path ) )
+ else:
+ # Pass just the first megabyte to parse_file_fields.
+ import StringIO
+ log.warn( "Attempting to load options from large file, reading just first megabyte" )
+ contents = open( path, 'r' ).read( megabyte )
+ options = self.parse_file_fields( StringIO.StringIO( contents ) )
elif self.tool_data_table:
options = self.tool_data_table.get_fields()
else:
https://bitbucket.org/galaxy/galaxy-central/changeset/53e55b49a24a/
changeset: 53e55b49a24a
branch: dynamic_options_memory_fix
user: jmchilton
date: 2012-02-07 17:38:00
summary: Fix a typo made with last commit.
affected #: 1 file
diff -r eb7a31e38a4cf9968d115984ab2ded908d3ddc9e -r 53e55b49a24a8e9a2f00b1c594c4e54c226e8865 lib/galaxy/tools/parameters/dynamic_options.py
--- a/lib/galaxy/tools/parameters/dynamic_options.py
+++ b/lib/galaxy/tools/parameters/dynamic_options.py
@@ -501,6 +501,7 @@
assert dataset is not None, "Required dataset '%s' missing from input" % self.dataset_ref_name
if not dataset: return [] #no valid dataset in history
# Ensure parsing dynamic options does not consume more than a megabyte worth memory.
+ path = dataset.file_name
file_size = os.path.getsize( path )
if os.path.getsize( path ) < 1048576:
options = self.parse_file_fields( open( path ) )
https://bitbucket.org/galaxy/galaxy-central/changeset/046157d1ed8a/
changeset: 046157d1ed8a
branch: dynamic_options_memory_fix
user: jmchilton
date: 2012-02-07 17:50:22
summary: One more fixed typo.
affected #: 1 file
diff -r 53e55b49a24a8e9a2f00b1c594c4e54c226e8865 -r 046157d1ed8af7ff178c6b1edee9512deccbc0ce lib/galaxy/tools/parameters/dynamic_options.py
--- a/lib/galaxy/tools/parameters/dynamic_options.py
+++ b/lib/galaxy/tools/parameters/dynamic_options.py
@@ -509,7 +509,7 @@
# Pass just the first megabyte to parse_file_fields.
import StringIO
log.warn( "Attempting to load options from large file, reading just first megabyte" )
- contents = open( path, 'r' ).read( megabyte )
+ contents = open( path, 'r' ).read( 1048576 )
options = self.parse_file_fields( StringIO.StringIO( contents ) )
elif self.tool_data_table:
options = self.tool_data_table.get_fields()
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/changeset/9a568ceeb9f5/
changeset: 9a568ceeb9f5
user: fubar
date: 2012-02-10 00:40:00
summary: Updates to FastQC wrapper
1) improvement suggested by bwlang to pass the history file name in as the file name to fastqc so it appears in the outputs instead of the previously unhelpful internal galaxy filename
2) lack of html output from fastqc or return of non-zero error code now triggers job failure - outputs may still be there but at least it goes red.
affected #: 2 files
diff -r abae2caf3da253353fbdcc077a7ce3cd64a38e54 -r 9a568ceeb9f5e281e7ae3c09d6049ac7d16183d4 tools/rgenetics/rgFastQC.py
--- a/tools/rgenetics/rgFastQC.py
+++ b/tools/rgenetics/rgFastQC.py
@@ -37,6 +37,7 @@
fastqc_data.txt per_base_gc_content.png per_base_sequence_content.png sequence_length_distribution.png warning.png
"""
+ serr = ''
dummy,tlog = tempfile.mkstemp(prefix='rgFastQClog')
sout = open(tlog, 'w')
fastq = os.path.basename(self.opts.input)
@@ -45,12 +46,18 @@
cl.append('-f %s' % self.opts.informat)
if self.opts.contaminants <> None :
cl.append('-c %s' % self.opts.contaminants)
- cl.append(self.opts.input)
+ # patch suggested by bwlang https://bitbucket.org/galaxy/galaxy-central/pull-request/30
+ # use a symlink in a temporary directory so that the FastQC report reflects the history input file name
+ fastqinfilename = os.path.basename(self.opts.inputfilename).replace(' ','_')
+ link_name = os.path.join(self.opts.outputdir, fastqinfilename)
+ os.symlink(self.opts.input, link_name)
+ cl.append(link_name)
p = subprocess.Popen(' '.join(cl), shell=True, stderr=sout, stdout=sout, cwd=self.opts.outputdir)
- return_value = p.wait()
+ retval = p.wait()
sout.close()
runlog = open(tlog,'r').readlines()
os.unlink(tlog)
+ os.unlink(link_name)
flist = os.listdir(self.opts.outputdir) # fastqc plays games with its output directory name. eesh
odpath = None
for f in flist:
@@ -65,7 +72,8 @@
rep = open(hpath,'r').readlines() # for our new html file but we need to insert our stuff after the <body> tag
except:
pass
- if hpath == None:
+ if hpath == None:
+ serr = '\n'.join(runlog)
res = ['## odpath=%s: No output found in %s. Output for the run was:<pre>\n' % (odpath,hpath),]
res += runlog
res += ['</pre>\n',
@@ -74,7 +82,7 @@
'It is also possible that the log shows that fastqc is not installed?<br/>\n',
'If that is the case, please tell the relevant Galaxy administrator that it can be snarfed from<br/>\n',
'http://www.bioinformatics.bbsrc.ac.uk/projects/fastqc/<br/>\n',]
- return res
+ return res,1,serr
self.fix_fastqcimages(odpath)
flist = os.listdir(self.opts.outputdir) # these have now been fixed
excludefiles = ['tick.png','warning.png','fastqc_icon.png','error.png']
@@ -84,7 +92,7 @@
rep[i] = rep[i].replace('Images/','')
html = self.fix_fastqc(rep,flist,runlog)
- return html
+ return html,retval,serr
@@ -129,6 +137,7 @@
if __name__ == '__main__':
op = optparse.OptionParser()
op.add_option('-i', '--input', default=None)
+ op.add_option('-j', '--inputfilename', default=None)
op.add_option('-o', '--htmloutput', default=None)
op.add_option('-d', '--outputdir', default="/tmp/shortread")
op.add_option('-f', '--informat', default='fastq')
@@ -141,9 +150,12 @@
if not os.path.exists(opts.outputdir):
os.makedirs(opts.outputdir)
f = FastQC(opts)
- html = f.run_fastqc()
+ html,retval,serr = f.run_fastqc()
f = open(opts.htmloutput, 'w')
f.write(''.join(html))
f.close()
+ if retval <> 0:
+ print >> sys.stderr, serr # indicate failure
+
diff -r abae2caf3da253353fbdcc077a7ce3cd64a38e54 -r 9a568ceeb9f5e281e7ae3c09d6049ac7d16183d4 tools/rgenetics/rgFastQC.xml
--- a/tools/rgenetics/rgFastQC.xml
+++ b/tools/rgenetics/rgFastQC.xml
@@ -1,7 +1,7 @@
-<tool name="Fastqc: Fastqc QC" id="fastqc" version="0.3">
+<tool name="Fastqc: Fastqc QC" id="fastqc" version="0.4"><description>using FastQC from Babraham</description><command interpreter="python">
- rgFastQC.py -i $input_file -d $html_file.files_path -o $html_file -n "$out_prefix" -f $input_file.ext -e ${GALAXY_DATA_INDEX_DIR}/shared/jars/FastQC/fastqc
+ rgFastQC.py -i $input_file -d $html_file.files_path -o $html_file -n "$out_prefix" -f $input_file.ext -j $input_file.name -e ${GALAXY_DATA_INDEX_DIR}/shared/jars/FastQC/fastqc
#if $contaminants.dataset and str($contaminants) > ''
-c "$contaminants"
#end if
@@ -16,7 +16,7 @@
help="tab delimited file with 2 columns: name and sequence. For example: Illumina Small RNA RT Primer CAAGCAGAAGACGGCATACGA"/></inputs><outputs>
- <data format="html" name="html_file" label="${out_prefix}.html" />
+ <data format="html" name="html_file" label="${out_prefix}_${on_string}.html" /></outputs><tests><test>
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.