2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/ebebacd0ccc4/ changeset: ebebacd0ccc4 user: jmchilton date: 2012-11-10 23:08:05 summary: Allow a datatype's merge method (used for parallelism) to take in an additional argument - the output_dataset object - if needed. No existing merge methods should need to be modified, the only modification to Galaxy's effective runtime behavior will occur if a datatype is defined which takes in a third keyword keyword argument called output_dataset. This change can enable the composite dataset parallelism pioneered by Jorrit Boekel, but one can imagine other potential uses (setting metadata derived from multiple files for instance). affected #: 1 file diff -r 5fdb5348d968f5bb38aedba23e26bdb9032a0c0b -r ebebacd0ccc45e5c0f26e0e7ceb55d70a0270dec lib/galaxy/jobs/splitters/multi.py --- a/lib/galaxy/jobs/splitters/multi.py +++ b/lib/galaxy/jobs/splitters/multi.py @@ -1,4 +1,5 @@ import os, logging, shutil +import inspect from galaxy import model, util @@ -125,7 +126,8 @@ output_file_name = str(outputs[output][1]) base_output_name = os.path.basename(output_file_name) if output in merge_outputs: - output_type = outputs[output][0].datatype + output_dataset = outputs[output][0] + output_type = output_dataset.datatype output_files = [os.path.join(dir,base_output_name) for dir in task_dirs] # Just include those files f in the output list for which the # file f exists; some files may not exist if a task fails. @@ -135,7 +137,13 @@ if len(output_files) < len(task_dirs): log.debug('merging only %i out of expected %i files for %s' % (len(output_files), len(task_dirs), output_file_name)) - output_type.merge(output_files, output_file_name) + # First two args to merge always output_files and path of dataset. More + # complicated merge methods may require more parameters. Set those up here. + extra_merge_arg_names = inspect.getargspec( output_type.merge ).args[2:] + extra_merge_args = {} + if "output_dataset" in extra_merge_arg_names: + extra_merge_args["output_dataset"] = output_dataset + output_type.merge(output_files, output_file_name, **extra_merge_args) log.debug('merge finished: %s' % output_file_name) else: msg = 'nothing to merge for %s (expected %i files)' \ https://bitbucket.org/galaxy/galaxy-central/changeset/ae3469a91321/ changeset: ae3469a91321 user: dannon date: 2012-12-04 15:45:52 summary: Merged in galaxyp/galaxy-central-optional-merge-args (pull request #83) affected #: 1 file diff -r 7bd130c33ebc218d97df83a63555abed222b301b -r ae3469a913214c4d5d5e6a1c5b449df978056972 lib/galaxy/jobs/splitters/multi.py --- a/lib/galaxy/jobs/splitters/multi.py +++ b/lib/galaxy/jobs/splitters/multi.py @@ -1,4 +1,5 @@ import os, logging, shutil +import inspect from galaxy import model, util @@ -125,7 +126,8 @@ output_file_name = str(outputs[output][1]) base_output_name = os.path.basename(output_file_name) if output in merge_outputs: - output_type = outputs[output][0].datatype + output_dataset = outputs[output][0] + output_type = output_dataset.datatype output_files = [os.path.join(dir,base_output_name) for dir in task_dirs] # Just include those files f in the output list for which the # file f exists; some files may not exist if a task fails. @@ -135,7 +137,13 @@ if len(output_files) < len(task_dirs): log.debug('merging only %i out of expected %i files for %s' % (len(output_files), len(task_dirs), output_file_name)) - output_type.merge(output_files, output_file_name) + # First two args to merge always output_files and path of dataset. More + # complicated merge methods may require more parameters. Set those up here. + extra_merge_arg_names = inspect.getargspec( output_type.merge ).args[2:] + extra_merge_args = {} + if "output_dataset" in extra_merge_arg_names: + extra_merge_args["output_dataset"] = output_dataset + output_type.merge(output_files, output_file_name, **extra_merge_args) log.debug('merge finished: %s' % output_file_name) else: msg = 'nothing to merge for %s (expected %i files)' \ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.