1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/changeset/5fc775a03f0b/
changeset: 5fc775a03f0b
user: greg
date: 2012-10-22 16:43:59
summary: Apply patches from John Chilton to resolve the issues where (1) Galaxy doesn't load tool sections from multiple files in a consistent manner, and (2) John's standing request in the now-defunct Galaxy bitbucket bug tracker to allow directories to appear in the tool_config_file option.
affected #: 2 files
diff -r d2c998e06f0ef7b117bf2e05150bbed877d5dcb3 -r 5fc775a03f0bcb083b375629518924138aed1a88 lib/galaxy/config.py
--- a/lib/galaxy/config.py
+++ b/lib/galaxy/config.py
@@ -313,7 +313,7 @@
if self.migrated_tools_config not in tool_configs:
tool_configs.append( self.migrated_tools_config )
for path in tool_configs:
- if not os.path.isfile( path ):
+ if not os.path.exists( path ):
raise ConfigurationError("File not found: %s" % path )
if not os.path.isfile( self.datatypes_config ):
raise ConfigurationError("File not found: %s" % self.datatypes_config )
diff -r d2c998e06f0ef7b117bf2e05150bbed877d5dcb3 -r 5fc775a03f0bcb083b375629518924138aed1a88 lib/galaxy/tools/__init__.py
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -73,6 +73,7 @@
self.workflows_by_id = {}
# In-memory dictionary that defines the layout of the tool panel.
self.tool_panel = odict()
+ self.index = 0
# File that contains the XML section and tool tags from all tool panel config files integrated into a
# single file that defines the tool panel layout. This file can be changed by the Galaxy administrator
# (in a way similar to the single tool_conf.xml file in the past) to alter the layout of the tool panel.
@@ -87,7 +88,14 @@
self.tool_root_dir = tool_root_dir
self.app = app
self.init_dependency_manager()
- for config_filename in listify( config_filenames ):
+ config_filenames = listify( config_filenames )
+ for config_filename in config_filenames:
+ if os.path.isdir( config_filename ):
+ directory_contents = sorted( os.listdir( config_filename ) )
+ directory_config_files = [ config_file for config_file in directory_contents if config_file.endswith( ".xml" ) ]
+ config_filenames.remove( config_filename )
+ config_filenames.extend( directory_config_files )
+ for config_filename in config_filenames:
try:
self.init_tools( config_filename )
except:
@@ -134,7 +142,9 @@
tool_path = self.tool_root_dir
# Only load the panel_dict under certain conditions.
load_panel_dict = not self.integrated_tool_panel_config_has_contents
- for index, elem in enumerate( root ):
+ for _, elem in enumerate( root ):
+ index = self.index
+ self.index += 1
if parsing_shed_tool_conf:
config_elems.append( elem )
if elem.tag == 'tool':
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
4 new commits in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/changeset/4daa56624e2a/
changeset: 4daa56624e2a
branch: zero-merge
user: peterjc
date: 2012-10-19 15:02:54
summary: Error when attempt to merge zero files.
Previously the code would try to call cat with no arguments,
which would wait for stdin and so do nothing and stall.
This error condition could be triggered with a cluster job
when Galaxy didn't get any output back.
affected #: 1 file
diff -r 4cd4f8bef29d17a7bca74e235031060e2289ebdc -r 4daa56624e2a7dbe16c3024eb0871040d1f98b8d lib/galaxy/datatypes/data.py
--- a/lib/galaxy/datatypes/data.py
+++ b/lib/galaxy/datatypes/data.py
@@ -536,7 +536,9 @@
TODO: Do we need to merge gzip files using gzjoin? cat seems to work,
but might be brittle. Need to revisit this.
"""
- if len(split_files) == 1:
+ if not split_files:
+ raise ValueError('Asked to merge zero files as %s' % output_file)
+ elif len(split_files) == 1:
cmd = 'mv -f %s %s' % ( split_files[0], output_file )
else:
cmd = 'cat %s > %s' % ( ' '.join(split_files), output_file )
https://bitbucket.org/galaxy/galaxy-central/changeset/273c02b75ac1/
changeset: 273c02b75ac1
branch: zero-merge
user: peterjc
date: 2012-10-19 15:51:37
summary: Don't attempt to merge zero files
affected #: 1 file
diff -r 4daa56624e2a7dbe16c3024eb0871040d1f98b8d -r 273c02b75ac110937c3a0045785df41cef863fff lib/galaxy/jobs/splitters/multi.py
--- a/lib/galaxy/jobs/splitters/multi.py
+++ b/lib/galaxy/jobs/splitters/multi.py
@@ -115,6 +115,7 @@
try:
working_directory = job_wrapper.working_directory
task_dirs = [os.path.join(working_directory, x) for x in os.listdir(working_directory) if x.startswith('task_')]
+ assert task_dirs, "Should be at least one sub-task!"
# TODO: Output datasets can be very complex. This doesn't handle metadata files
outputs = job_wrapper.get_output_hdas_and_fnames()
pickone_done = []
@@ -129,10 +130,16 @@
# Just include those files f in the output list for which the
# file f exists; some files may not exist if a task fails.
output_files = [ f for f in output_files if os.path.exists(f) ]
- log.debug('files %s ' % output_files)
- output_type.merge(output_files, output_file_name)
- log.debug('merge finished: %s' % output_file_name)
- pass # TODO: merge all the files
+ if output_files:
+ log.debug('files %s ' % output_files)
+ if len(output_files) < len(task_dirs):
+ log.debug('merging only %i out of expected %i files for %s'
+ % (len(output_files), len(task_dirs), output_file_name))
+ output_type.merge(output_files, output_file_name)
+ log.debug('merge finished: %s' % output_file_name)
+ else:
+ log.debug('nothing to merge for %s (expected %i files)'
+ % (output_file_name, len(task_dirs)))
elif output in pickone_outputs:
# just pick one of them
if output not in pickone_done:
https://bitbucket.org/galaxy/galaxy-central/changeset/dbd7336b62c9/
changeset: dbd7336b62c9
branch: zero-merge
user: peterjc
date: 2012-10-19 16:23:02
summary: Log nothing to merge to stderr
affected #: 1 file
diff -r 273c02b75ac110937c3a0045785df41cef863fff -r dbd7336b62c9abf939ea4081482c744b56838e31 lib/galaxy/jobs/splitters/multi.py
--- a/lib/galaxy/jobs/splitters/multi.py
+++ b/lib/galaxy/jobs/splitters/multi.py
@@ -138,8 +138,10 @@
output_type.merge(output_files, output_file_name)
log.debug('merge finished: %s' % output_file_name)
else:
- log.debug('nothing to merge for %s (expected %i files)'
- % (output_file_name, len(task_dirs)))
+ msg = 'nothing to merge for %s (expected %i files)' \
+ % (output_file_name, len(task_dirs))
+ log.debug(msg)
+ stderr += msg + "\n"
elif output in pickone_outputs:
# just pick one of them
if output not in pickone_done:
https://bitbucket.org/galaxy/galaxy-central/changeset/ae9d794074bc/
changeset: ae9d794074bc
user: smcmanus
date: 2012-10-19 21:53:49
summary: Merged in peterjc/galaxy-central/zero-merge (pull request #78)
affected #: 2 files
diff -r 6db344e2ce554f4332b048dd41bb158e26c49fed -r ae9d794074bcc4732601e480ff54d53b0b2ee6e7 lib/galaxy/datatypes/data.py
--- a/lib/galaxy/datatypes/data.py
+++ b/lib/galaxy/datatypes/data.py
@@ -536,7 +536,9 @@
TODO: Do we need to merge gzip files using gzjoin? cat seems to work,
but might be brittle. Need to revisit this.
"""
- if len(split_files) == 1:
+ if not split_files:
+ raise ValueError('Asked to merge zero files as %s' % output_file)
+ elif len(split_files) == 1:
cmd = 'mv -f %s %s' % ( split_files[0], output_file )
else:
cmd = 'cat %s > %s' % ( ' '.join(split_files), output_file )
diff -r 6db344e2ce554f4332b048dd41bb158e26c49fed -r ae9d794074bcc4732601e480ff54d53b0b2ee6e7 lib/galaxy/jobs/splitters/multi.py
--- a/lib/galaxy/jobs/splitters/multi.py
+++ b/lib/galaxy/jobs/splitters/multi.py
@@ -115,6 +115,7 @@
try:
working_directory = job_wrapper.working_directory
task_dirs = [os.path.join(working_directory, x) for x in os.listdir(working_directory) if x.startswith('task_')]
+ assert task_dirs, "Should be at least one sub-task!"
# TODO: Output datasets can be very complex. This doesn't handle metadata files
outputs = job_wrapper.get_output_hdas_and_fnames()
pickone_done = []
@@ -129,10 +130,18 @@
# Just include those files f in the output list for which the
# file f exists; some files may not exist if a task fails.
output_files = [ f for f in output_files if os.path.exists(f) ]
- log.debug('files %s ' % output_files)
- output_type.merge(output_files, output_file_name)
- log.debug('merge finished: %s' % output_file_name)
- pass # TODO: merge all the files
+ if output_files:
+ log.debug('files %s ' % output_files)
+ if len(output_files) < len(task_dirs):
+ log.debug('merging only %i out of expected %i files for %s'
+ % (len(output_files), len(task_dirs), output_file_name))
+ output_type.merge(output_files, output_file_name)
+ log.debug('merge finished: %s' % output_file_name)
+ else:
+ msg = 'nothing to merge for %s (expected %i files)' \
+ % (output_file_name, len(task_dirs))
+ log.debug(msg)
+ stderr += msg + "\n"
elif output in pickone_outputs:
# just pick one of them
if output not in pickone_done:
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.