4 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b040f24f2dff/ Changeset: b040f24f2dff User: gvk Date: 2014-11-21 18:35:02+00:00 Summary: Enhance the collect_primary_datasets() method to allow assignment of the defined output parameter for 3rd-party tools that manage tool outputs internally. Affected #: 1 file diff -r 7cb167299a13cc273a794f19363f32083096efdb -r b040f24f2dff8e748ff4385637916363d43ce1e2 lib/galaxy/tools/parameters/output_collect.py --- a/lib/galaxy/tools/parameters/output_collect.py +++ b/lib/galaxy/tools/parameters/output_collect.py @@ -5,7 +5,6 @@ import glob import json - from galaxy import jobs from galaxy import util from galaxy.util import odict @@ -30,8 +29,10 @@ # Loop through output file names, looking for generated primary # datasets in form of: # 'primary_associatedWithDatasetID_designation_visibility_extension(_DBKEY)' + primary_output_assigned = False + new_outdata_name = None primary_datasets = {} - for name, outdata in output.items(): + for output_index, ( name, outdata ) in enumerate( output.items() ): dataset_collectors = tool.outputs[ name ].dataset_collectors if name in tool.outputs else [ DEFAULT_DATASET_COLLECTOR ] filenames = odict.odict() if 'new_file_path' in app.config.collect_outputs_from: @@ -56,14 +57,20 @@ continue if extra_file_collector.match( outdata, filename ): filenames[ path ] = extra_file_collector - for filename, extra_file_collector in filenames.iteritems(): - if not name in primary_datasets: - primary_datasets[name] = {} + for filename_index, ( filename, extra_file_collector ) in enumerate( filenames.iteritems() ): fields_match = extra_file_collector.match( outdata, os.path.basename( filename ) ) if not fields_match: # Before I guess pop() would just have thrown an IndexError raise Exception( "Problem parsing metadata fields for file %s" % filename ) designation = fields_match.designation + if filename_index == 0 and extra_file_collector.assign_primary_output and output_index == 0: + new_outdata_name = fields_match.name or "%s (%s)" % ( outdata.name, designation ) + # Move data from temp location to dataset location + app.object_store.update_from_file( outdata.dataset, file_name=filename, create=True ) + primary_output_assigned = True + continue + if not name in primary_datasets: + primary_datasets[ name ] = {} visible = fields_match.visible ext = fields_match.ext if ext == "input": @@ -145,6 +152,13 @@ dataset.history.add_dataset( new_data ) sa_session.add( new_data ) sa_session.flush() + if primary_output_assigned: + outdata.name = new_outdata_name + outdata.init_meta() + outdata.set_meta() + outdata.set_peek() + sa_session.add( outdata ) + sa_session.flush() return primary_datasets @@ -178,6 +192,7 @@ self.default_ext = kwargs.get( "ext", None ) self.default_visible = util.asbool( kwargs.get( "visible", None ) ) self.directory = kwargs.get( "directory", None ) + self.assign_primary_output = util.asbool( kwargs.get( 'assign_primary_output', False ) ) def pattern_for_dataset( self, dataset_instance=None ): token_replacement = r'\d+' https://bitbucket.org/galaxy/galaxy-central/commits/45f92195479a/ Changeset: 45f92195479a User: jmchilton Date: 2014-11-26 03:01:49+00:00 Summary: Make dynamic discovery of output datasets deterministic. Should aid in testing and ... lets say ... reproducibility because that is popular in these parts. Affected #: 1 file diff -r b040f24f2dff8e748ff4385637916363d43ce1e2 -r 45f92195479ad49c8bf5a9564bfc6078dbc59bc1 lib/galaxy/tools/parameters/output_collect.py --- a/lib/galaxy/tools/parameters/output_collect.py +++ b/lib/galaxy/tools/parameters/output_collect.py @@ -51,7 +51,7 @@ raise Exception( "Problem with tool configuration, attempting to pull in datasets from outside working directory." ) if not os.path.isdir( directory ): continue - for filename in os.listdir( directory ): + for filename in sorted( os.listdir( directory ) ): path = os.path.join( directory, filename ) if not os.path.isfile( path ): continue https://bitbucket.org/galaxy/galaxy-central/commits/ef0e65b55ebb/ Changeset: ef0e65b55ebb User: jmchilton Date: 2014-11-26 03:13:17+00:00 Summary: Implement tool test demonstrating pull request 569. The new assign_primary_output attribute on discover_datasets tag. Affected #: 2 files diff -r 45f92195479ad49c8bf5a9564bfc6078dbc59bc1 -r ef0e65b55ebba822c8420f825e4b306e8018d7cb test/functional/tools/multi_output_assign_primary.xml --- /dev/null +++ b/test/functional/tools/multi_output_assign_primary.xml @@ -0,0 +1,34 @@ +<tool id="multi_output_assign_primary" name="multi_output_assign_primary" description="multi_output_assign_primary" version="0.1.0"> + <command> + echo "1" > sample1.report.tsv; + echo "2" > sample2.report.tsv; + echo "3" > sample3.report.tsv; + </command> + <inputs> + <param name="num_param" type="integer" value="7" /> + <param name="input" type="data" /> + </inputs> + <outputs> + <data format="tabular" name="sample"> + <discover_datasets pattern="(?P<designation>.+)\.report\.tsv" ext="tabular" visible="true" assign_primary_output="true" /> + </data> + </outputs> + <tests> + <test> + <param name="num_param" value="7" /> + <param name="input" ftype="txt" value="simple_line.txt"/> + <output name="sample"> + <assert_contents> + <has_line line="1" /> + </assert_contents> + <!-- no sample1 it was consumed by named output "sample" --> + <discovered_dataset designation="sample2" ftype="tabular"> + <assert_contents><has_line line="2" /></assert_contents> + </discovered_dataset> + <discovered_dataset designation="sample3" ftype="tabular"> + <assert_contents><has_line line="3" /></assert_contents> + </discovered_dataset> + </output> + </test> + </tests> +</tool> diff -r 45f92195479ad49c8bf5a9564bfc6078dbc59bc1 -r ef0e65b55ebba822c8420f825e4b306e8018d7cb test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -9,6 +9,7 @@ <tool file="multi_select.xml" /><tool file="multi_output.xml" /><tool file="multi_output_configured.xml" /> + <tool file="multi_output_assign_primary.xml" /><tool file="composite_output.xml" /><tool file="metadata.xml" /><tool file="output_order.xml" /> https://bitbucket.org/galaxy/galaxy-central/commits/30132ca365ef/ Changeset: 30132ca365ef User: jmchilton Date: 2014-11-26 03:14:18+00:00 Summary: Merge pull request 569 with test implemented to demonstrate. Affected #: 3 files diff -r e4e1f70bc2d1a84219038b7e58b933a94b424da7 -r 30132ca365effac14fe5fd419ed39d20b83fcbc6 lib/galaxy/tools/parameters/output_collect.py --- a/lib/galaxy/tools/parameters/output_collect.py +++ b/lib/galaxy/tools/parameters/output_collect.py @@ -5,7 +5,6 @@ import glob import json - from galaxy import jobs from galaxy import util from galaxy.util import odict @@ -30,8 +29,10 @@ # Loop through output file names, looking for generated primary # datasets in form of: # 'primary_associatedWithDatasetID_designation_visibility_extension(_DBKEY)' + primary_output_assigned = False + new_outdata_name = None primary_datasets = {} - for name, outdata in output.items(): + for output_index, ( name, outdata ) in enumerate( output.items() ): dataset_collectors = tool.outputs[ name ].dataset_collectors if name in tool.outputs else [ DEFAULT_DATASET_COLLECTOR ] filenames = odict.odict() if 'new_file_path' in app.config.collect_outputs_from: @@ -50,20 +51,26 @@ raise Exception( "Problem with tool configuration, attempting to pull in datasets from outside working directory." ) if not os.path.isdir( directory ): continue - for filename in os.listdir( directory ): + for filename in sorted( os.listdir( directory ) ): path = os.path.join( directory, filename ) if not os.path.isfile( path ): continue if extra_file_collector.match( outdata, filename ): filenames[ path ] = extra_file_collector - for filename, extra_file_collector in filenames.iteritems(): - if not name in primary_datasets: - primary_datasets[name] = {} + for filename_index, ( filename, extra_file_collector ) in enumerate( filenames.iteritems() ): fields_match = extra_file_collector.match( outdata, os.path.basename( filename ) ) if not fields_match: # Before I guess pop() would just have thrown an IndexError raise Exception( "Problem parsing metadata fields for file %s" % filename ) designation = fields_match.designation + if filename_index == 0 and extra_file_collector.assign_primary_output and output_index == 0: + new_outdata_name = fields_match.name or "%s (%s)" % ( outdata.name, designation ) + # Move data from temp location to dataset location + app.object_store.update_from_file( outdata.dataset, file_name=filename, create=True ) + primary_output_assigned = True + continue + if not name in primary_datasets: + primary_datasets[ name ] = {} visible = fields_match.visible ext = fields_match.ext if ext == "input": @@ -145,6 +152,13 @@ dataset.history.add_dataset( new_data ) sa_session.add( new_data ) sa_session.flush() + if primary_output_assigned: + outdata.name = new_outdata_name + outdata.init_meta() + outdata.set_meta() + outdata.set_peek() + sa_session.add( outdata ) + sa_session.flush() return primary_datasets @@ -178,6 +192,7 @@ self.default_ext = kwargs.get( "ext", None ) self.default_visible = util.asbool( kwargs.get( "visible", None ) ) self.directory = kwargs.get( "directory", None ) + self.assign_primary_output = util.asbool( kwargs.get( 'assign_primary_output', False ) ) def pattern_for_dataset( self, dataset_instance=None ): token_replacement = r'\d+' diff -r e4e1f70bc2d1a84219038b7e58b933a94b424da7 -r 30132ca365effac14fe5fd419ed39d20b83fcbc6 test/functional/tools/multi_output_assign_primary.xml --- /dev/null +++ b/test/functional/tools/multi_output_assign_primary.xml @@ -0,0 +1,34 @@ +<tool id="multi_output_assign_primary" name="multi_output_assign_primary" description="multi_output_assign_primary" version="0.1.0"> + <command> + echo "1" > sample1.report.tsv; + echo "2" > sample2.report.tsv; + echo "3" > sample3.report.tsv; + </command> + <inputs> + <param name="num_param" type="integer" value="7" /> + <param name="input" type="data" /> + </inputs> + <outputs> + <data format="tabular" name="sample"> + <discover_datasets pattern="(?P<designation>.+)\.report\.tsv" ext="tabular" visible="true" assign_primary_output="true" /> + </data> + </outputs> + <tests> + <test> + <param name="num_param" value="7" /> + <param name="input" ftype="txt" value="simple_line.txt"/> + <output name="sample"> + <assert_contents> + <has_line line="1" /> + </assert_contents> + <!-- no sample1 it was consumed by named output "sample" --> + <discovered_dataset designation="sample2" ftype="tabular"> + <assert_contents><has_line line="2" /></assert_contents> + </discovered_dataset> + <discovered_dataset designation="sample3" ftype="tabular"> + <assert_contents><has_line line="3" /></assert_contents> + </discovered_dataset> + </output> + </test> + </tests> +</tool> diff -r e4e1f70bc2d1a84219038b7e58b933a94b424da7 -r 30132ca365effac14fe5fd419ed39d20b83fcbc6 test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -9,6 +9,7 @@ <tool file="multi_select.xml" /><tool file="multi_output.xml" /><tool file="multi_output_configured.xml" /> + <tool file="multi_output_assign_primary.xml" /><tool file="composite_output.xml" /><tool file="metadata.xml" /><tool file="output_order.xml" /> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.