6 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/496776fd6b87/ changeset: 496776fd6b87 user: jmchilton date: 2012-10-05 06:50:33 summary: Extend batch workflow mode to allow multiple selection of datasets for multiple parameters. Two modes are implemented on backend - one that runs workflow on every combination of inputs ([cartesian] product mode) and the other that matches inputs (matched mode). UI is left as an exercise for the reader. affected #: 2 files diff -r 210c39f4bf7f2d91c68df84dd9ccecb2ff9f93aa -r 496776fd6b87544d1434ceeb78cd54ba5ca32f43 lib/galaxy/webapps/galaxy/controllers/workflow.py --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -1301,7 +1301,7 @@ ## % ( workflow_name, web.url_for( action='editor', id=trans.security.encode_id(stored.id) ) ) ) @web.expose - def run( self, trans, id, history_id=None, hide_fixed_params=False, **kwargs ): + def run( self, trans, id, history_id=None, multiple_input_mode="product", hide_fixed_params=False, **kwargs ): stored = self.get_stored_workflow( trans, id, check_ownership=False ) user = trans.get_user() if stored.user != user: @@ -1340,23 +1340,9 @@ if kwargs: # If kwargs were provided, the states for each step should have # been POSTed - # Get the kwarg keys for data inputs - input_keys = filter(lambda a: a.endswith('|input'), kwargs) - # Example: prefixed='2|input' - # Check if one of them is a list - multiple_input_key = None - multiple_inputs = [None] - for input_key in input_keys: - if isinstance(kwargs[input_key], list): - multiple_input_key = input_key - multiple_inputs = kwargs[input_key] # List to gather values for the template invocations=[] - for input_number, single_input in enumerate(multiple_inputs): - # Example: single_input='1', single_input='2', etc... - # 'Fix' the kwargs, to have only the input for this iteration - if multiple_input_key: - kwargs[multiple_input_key] = single_input + for kwargs in _expand_multiple_inputs(kwargs, mode=multiple_input_mode): for step in workflow.steps: step.upgrade_messages = {} # Connections by input name @@ -2074,3 +2060,63 @@ cleanup( "", inputs, values ) return associations +def _expand_multiple_inputs(kwargs, mode): + input_combos = _build_input_combos(kwargs, mode) + for input_combo in input_combos: + for key, value in input_combo.iteritems(): + kwargs[key] = value + yield kwargs + +def _build_input_combos(kwargs, mode): + if mode == "product": + return _build_input_combos_product(kwargs) + else: # mode == "matched" + return _build_input_combos_matched(kwargs) + +def _build_input_combos_matched(kwargs): + (single_inputs, multi_inputs) = _split_inputs(kwargs) + matched_multi_inputs = [] + + first_multi_input_key = multi_inputs.keys()[0] + first_multi_value = multi_inputs.pop(first_multi_input_key) + + for value in first_multi_value: + new_inputs = _copy_and_extend_inputs(single_inputs, first_multi_input_key, value) + matched_multi_inputs.append(new_inputs) + + for multi_input_key, multi_input_values in multi_inputs.iteritems(): + if len(multi_input_values) != len(first_multi_value): + raise Exception("Failed to match up multi-select inputs, must select equal number of data files in each multiselect") + for index, value in enumerate(multi_input_values): + matched_multi_inputs[index][multi_input_key] = value + return matched_multi_inputs + +def _build_input_combos_product(kwargs): + (single_inputs, multi_inputs) = _split_inputs(kwargs) + combos = [single_inputs] + for multi_input_key, multi_input_value in multi_inputs.iteritems(): + iter_combos = [] + + for combo in combos: + for input_value in multi_input_value: + iter_combos.append(_copy_and_extend_inputs(combo, multi_input_key, input_value)) + + combos = iter_combos + return combos + +def _copy_and_extend_inputs(inputs, key, value): + new_inputs = dict(inputs) + new_inputs[key] = value + return new_inputs + +def _split_inputs(kwargs): + input_keys = filter(lambda a: a.endswith('|input'), kwargs) + single_inputs = {} + multi_inputs = {} + for input_key in input_keys: + input_val = kwargs[input_key] + if isinstance(input_val, list): + multi_inputs[input_key] = input_val + else: + single_inputs[input_key] = input_val + return (single_inputs, multi_inputs) diff -r 210c39f4bf7f2d91c68df84dd9ccecb2ff9f93aa -r 496776fd6b87544d1434ceeb78cd54ba5ca32f43 templates/workflow/run.mako --- a/templates/workflow/run.mako +++ b/templates/workflow/run.mako @@ -36,6 +36,7 @@ select.removeAttr('multiple').removeAttr('size'); placeholder = 'type to filter'; } else { + // Comment out the following line to multiple batch input workflows in UI. $('.multiinput').addClass('disabled'); $('.multiinput', select.closest('.form-row')).removeClass('disabled'); select.attr('multiple', 'multiple').attr('size', 8); @@ -336,6 +337,29 @@ <form id="tool_form" name="tool_form" method="POST"> ## <input type="hidden" name="workflow_name" value="${h.to_unicode( workflow.name ) | h}" /> +<!-- TODO: Implement UI for selecting between product and matched mode + for batch workflows in multiple inputs are selected for 2 or more + params. + + 1) Delete this line above: $('.multiinput').addClass('disabled'); + 2) Allow user to select between product and matched mode. + + If user selected 5 inputs for one param and 5 inputs for another + in matched mode that will be run the workflow 5 times matching + each input and in product mode it will run the workflow 25 times + with every combination of input pairs. If user selects 6 inputs + for one param and 4 for another, in product mode 24 workflows + will run and in matched mode the submission will fail. + + In matched mode the inputs are matched from top to bottom + regardless of the order they are actually select in. This + behavior is I assume the desired behavior but I have only tested + it in chrome, care should be taken to test behavior on other + browsers and augment UI to ensure numbers of inputs matches + up. +--> +<input type="hidden" name="multiple_input_mode" value="matched" /><!-- product or matched --> + %if wf_parms: <div class="metadataForm"> https://bitbucket.org/galaxy/galaxy-central/changeset/a7201a335abe/ changeset: a7201a335abe user: jmchilton date: 2012-10-06 04:41:07 summary: Fix for case where no inputs are batched. affected #: 1 file diff -r 496776fd6b87544d1434ceeb78cd54ba5ca32f43 -r a7201a335abe14357ed621e8d70cc03c6ff26dc9 lib/galaxy/webapps/galaxy/controllers/workflow.py --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -2075,6 +2075,9 @@ def _build_input_combos_matched(kwargs): (single_inputs, multi_inputs) = _split_inputs(kwargs) + if len(multi_inputs) == 0: + return [{}] + matched_multi_inputs = [] first_multi_input_key = multi_inputs.keys()[0] https://bitbucket.org/galaxy/galaxy-central/changeset/5fb934e1bd7d/ changeset: 5fb934e1bd7d user: jmchilton date: 2012-10-06 17:57:04 summary: Fix multi-batch mode when sending workflows to new histories. affected #: 1 file diff -r a7201a335abe14357ed621e8d70cc03c6ff26dc9 -r 5fb934e1bd7ddb6f28ea71ffb0ba61d63810eb66 lib/galaxy/webapps/galaxy/controllers/workflow.py --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -1342,7 +1342,7 @@ # been POSTed # List to gather values for the template invocations=[] - for kwargs in _expand_multiple_inputs(kwargs, mode=multiple_input_mode): + for (kwargs, multi_input_keys) in _expand_multiple_inputs(kwargs, mode=multiple_input_mode) for step in workflow.steps: step.upgrade_messages = {} # Connections by input name @@ -1385,9 +1385,9 @@ nh_name = kwargs['new_history_name'] else: nh_name = "History from %s workflow" % workflow.name - if multiple_input_key: - mx_ds_name = trans.sa_session.query(trans.app.model.HistoryDatasetAssociation).get( single_input ).name - nh_name = '%s on %s' % (nh_name, mx_ds_name) + instance_inputs = [kwargs[multi_input_key] for multi_input_key in multi_input_keys] + instance_ds_names = [trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( instance_input ).name for instance_input in instance_inputs] + nh_name = '%s%s' % (nh_name, _build_workflow_on_str( instance_ds_names )) new_history = trans.app.model.History( user=trans.user, name=nh_name ) new_history.copy_tags_from(trans.user, trans.get_history()) trans.sa_session.add( new_history ) @@ -2060,12 +2060,23 @@ cleanup( "", inputs, values ) return associations +def _build_workflow_on_str(instance_ds_names): + # Returns suffix for new histories based on multi input iteration + num_multi_inputs = len(instance_ds_names) + if num_multi_inputs == 0: + return "" + elif num_multi_inputs == 1: + return " on %s" % instance_ds_names[0] + else: + return " on %s and %s" % (", ".join(instance_ds_names[0:-1]), instance_ds_names[-1]) + + def _expand_multiple_inputs(kwargs, mode): - input_combos = _build_input_combos(kwargs, mode) + (input_combos, multi_inputs) = _build_input_combos(kwargs, mode) for input_combo in input_combos: for key, value in input_combo.iteritems(): kwargs[key] = value - yield kwargs + yield (kwargs, multi_inputs.keys()) def _build_input_combos(kwargs, mode): if mode == "product": @@ -2076,23 +2087,25 @@ def _build_input_combos_matched(kwargs): (single_inputs, multi_inputs) = _split_inputs(kwargs) if len(multi_inputs) == 0: - return [{}] - + return ([{}], {}) + matched_multi_inputs = [] - + first_multi_input_key = multi_inputs.keys()[0] - first_multi_value = multi_inputs.pop(first_multi_input_key) - + first_multi_value = multi_inputs.get(first_multi_input_key) + for value in first_multi_value: new_inputs = _copy_and_extend_inputs(single_inputs, first_multi_input_key, value) matched_multi_inputs.append(new_inputs) - + for multi_input_key, multi_input_values in multi_inputs.iteritems(): + if multi_input_key == first_multi_input_key: + continue if len(multi_input_values) != len(first_multi_value): raise Exception("Failed to match up multi-select inputs, must select equal number of data files in each multiselect") for index, value in enumerate(multi_input_values): matched_multi_inputs[index][multi_input_key] = value - return matched_multi_inputs + return (matched_multi_inputs, multi_inputs) def _build_input_combos_product(kwargs): (single_inputs, multi_inputs) = _split_inputs(kwargs) @@ -2105,7 +2118,7 @@ iter_combos.append(_copy_and_extend_inputs(combo, multi_input_key, input_value)) combos = iter_combos - return combos + return (combos, multi_inputs) def _copy_and_extend_inputs(inputs, key, value): new_inputs = dict(inputs) https://bitbucket.org/galaxy/galaxy-central/changeset/05289b5ea824/ changeset: 05289b5ea824 user: jmchilton date: 2012-10-06 18:07:03 summary: Typo introduced at last second, I swear am testing this stuff before committing. affected #: 1 file diff -r 5fb934e1bd7ddb6f28ea71ffb0ba61d63810eb66 -r 05289b5ea82479cd2c25ef8b67f36c953e72b431 lib/galaxy/webapps/galaxy/controllers/workflow.py --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -1342,7 +1342,7 @@ # been POSTed # List to gather values for the template invocations=[] - for (kwargs, multi_input_keys) in _expand_multiple_inputs(kwargs, mode=multiple_input_mode) + for (kwargs, multi_input_keys) in _expand_multiple_inputs(kwargs, mode=multiple_input_mode): for step in workflow.steps: step.upgrade_messages = {} # Connections by input name https://bitbucket.org/galaxy/galaxy-central/changeset/6b945a18ad4c/ changeset: 6b945a18ad4c user: dannon date: 2012-10-15 16:01:55 summary: Merge affected #: 2 files diff -r 2d12f10c87c73cb0ca2d9ef227613230d80736a9 -r 6b945a18ad4cc30920e83dcc21e6d25a656fdb5a lib/galaxy/webapps/galaxy/controllers/workflow.py --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -1301,7 +1301,7 @@ ## % ( workflow_name, web.url_for( action='editor', id=trans.security.encode_id(stored.id) ) ) ) @web.expose - def run( self, trans, id, history_id=None, hide_fixed_params=False, **kwargs ): + def run( self, trans, id, history_id=None, multiple_input_mode="product", hide_fixed_params=False, **kwargs ): stored = self.get_stored_workflow( trans, id, check_ownership=False ) user = trans.get_user() if stored.user != user: @@ -1340,23 +1340,9 @@ if kwargs: # If kwargs were provided, the states for each step should have # been POSTed - # Get the kwarg keys for data inputs - input_keys = filter(lambda a: a.endswith('|input'), kwargs) - # Example: prefixed='2|input' - # Check if one of them is a list - multiple_input_key = None - multiple_inputs = [None] - for input_key in input_keys: - if isinstance(kwargs[input_key], list): - multiple_input_key = input_key - multiple_inputs = kwargs[input_key] # List to gather values for the template invocations=[] - for input_number, single_input in enumerate(multiple_inputs): - # Example: single_input='1', single_input='2', etc... - # 'Fix' the kwargs, to have only the input for this iteration - if multiple_input_key: - kwargs[multiple_input_key] = single_input + for (kwargs, multi_input_keys) in _expand_multiple_inputs(kwargs, mode=multiple_input_mode): for step in workflow.steps: step.upgrade_messages = {} # Connections by input name @@ -1399,9 +1385,9 @@ nh_name = kwargs['new_history_name'] else: nh_name = "History from %s workflow" % workflow.name - if multiple_input_key: - mx_ds_name = trans.sa_session.query(trans.app.model.HistoryDatasetAssociation).get( single_input ).name - nh_name = '%s on %s' % (nh_name, mx_ds_name) + instance_inputs = [kwargs[multi_input_key] for multi_input_key in multi_input_keys] + instance_ds_names = [trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( instance_input ).name for instance_input in instance_inputs] + nh_name = '%s%s' % (nh_name, _build_workflow_on_str( instance_ds_names )) new_history = trans.app.model.History( user=trans.user, name=nh_name ) new_history.copy_tags_from(trans.user, trans.get_history()) trans.sa_session.add( new_history ) @@ -2074,3 +2060,79 @@ cleanup( "", inputs, values ) return associations +def _build_workflow_on_str(instance_ds_names): + # Returns suffix for new histories based on multi input iteration + num_multi_inputs = len(instance_ds_names) + if num_multi_inputs == 0: + return "" + elif num_multi_inputs == 1: + return " on %s" % instance_ds_names[0] + else: + return " on %s and %s" % (", ".join(instance_ds_names[0:-1]), instance_ds_names[-1]) + + +def _expand_multiple_inputs(kwargs, mode): + (input_combos, multi_inputs) = _build_input_combos(kwargs, mode) + for input_combo in input_combos: + for key, value in input_combo.iteritems(): + kwargs[key] = value + yield (kwargs, multi_inputs.keys()) + +def _build_input_combos(kwargs, mode): + if mode == "product": + return _build_input_combos_product(kwargs) + else: # mode == "matched" + return _build_input_combos_matched(kwargs) + +def _build_input_combos_matched(kwargs): + (single_inputs, multi_inputs) = _split_inputs(kwargs) + if len(multi_inputs) == 0: + return ([{}], {}) + + matched_multi_inputs = [] + + first_multi_input_key = multi_inputs.keys()[0] + first_multi_value = multi_inputs.get(first_multi_input_key) + + for value in first_multi_value: + new_inputs = _copy_and_extend_inputs(single_inputs, first_multi_input_key, value) + matched_multi_inputs.append(new_inputs) + + for multi_input_key, multi_input_values in multi_inputs.iteritems(): + if multi_input_key == first_multi_input_key: + continue + if len(multi_input_values) != len(first_multi_value): + raise Exception("Failed to match up multi-select inputs, must select equal number of data files in each multiselect") + for index, value in enumerate(multi_input_values): + matched_multi_inputs[index][multi_input_key] = value + return (matched_multi_inputs, multi_inputs) + +def _build_input_combos_product(kwargs): + (single_inputs, multi_inputs) = _split_inputs(kwargs) + combos = [single_inputs] + for multi_input_key, multi_input_value in multi_inputs.iteritems(): + iter_combos = [] + + for combo in combos: + for input_value in multi_input_value: + iter_combos.append(_copy_and_extend_inputs(combo, multi_input_key, input_value)) + + combos = iter_combos + return (combos, multi_inputs) + +def _copy_and_extend_inputs(inputs, key, value): + new_inputs = dict(inputs) + new_inputs[key] = value + return new_inputs + +def _split_inputs(kwargs): + input_keys = filter(lambda a: a.endswith('|input'), kwargs) + single_inputs = {} + multi_inputs = {} + for input_key in input_keys: + input_val = kwargs[input_key] + if isinstance(input_val, list): + multi_inputs[input_key] = input_val + else: + single_inputs[input_key] = input_val + return (single_inputs, multi_inputs) diff -r 2d12f10c87c73cb0ca2d9ef227613230d80736a9 -r 6b945a18ad4cc30920e83dcc21e6d25a656fdb5a templates/workflow/run.mako --- a/templates/workflow/run.mako +++ b/templates/workflow/run.mako @@ -36,6 +36,7 @@ select.removeAttr('multiple').removeAttr('size'); placeholder = 'type to filter'; } else { + // Comment out the following line to multiple batch input workflows in UI. $('.multiinput').addClass('disabled'); $('.multiinput', select.closest('.form-row')).removeClass('disabled'); select.attr('multiple', 'multiple').attr('size', 8); @@ -336,6 +337,29 @@ <form id="tool_form" name="tool_form" method="POST"> ## <input type="hidden" name="workflow_name" value="${h.to_unicode( workflow.name ) | h}" /> +<!-- TODO: Implement UI for selecting between product and matched mode + for batch workflows in multiple inputs are selected for 2 or more + params. + + 1) Delete this line above: $('.multiinput').addClass('disabled'); + 2) Allow user to select between product and matched mode. + + If user selected 5 inputs for one param and 5 inputs for another + in matched mode that will be run the workflow 5 times matching + each input and in product mode it will run the workflow 25 times + with every combination of input pairs. If user selects 6 inputs + for one param and 4 for another, in product mode 24 workflows + will run and in matched mode the submission will fail. + + In matched mode the inputs are matched from top to bottom + regardless of the order they are actually select in. This + behavior is I assume the desired behavior but I have only tested + it in chrome, care should be taken to test behavior on other + browsers and augment UI to ensure numbers of inputs matches + up. +--> +<input type="hidden" name="multiple_input_mode" value="matched" /><!-- product or matched --> + %if wf_parms: <div class="metadataForm"> https://bitbucket.org/galaxy/galaxy-central/changeset/e93cbadb0893/ changeset: e93cbadb0893 user: dannon date: 2012-10-15 16:43:10 summary: Merge affected #: 1 file diff -r 6b945a18ad4cc30920e83dcc21e6d25a656fdb5a -r e93cbadb089348fc00734a3e6693e179e0716b5e templates/workflow/editor.mako --- a/templates/workflow/editor.mako +++ b/templates/workflow/editor.mako @@ -60,7 +60,6 @@ } // Init tool options. - %if trans.app.toolbox_search.enabled: make_popupmenu( $("#tools-options-button"), { ## Search tools menu item. <% @@ -163,7 +162,6 @@ } this.lastValue = this.value; }); - %endif // Canvas overview management canvas_manager = new CanvasManager( $("#canvas-viewport"), $("#overview") ); Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.