- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: jmchilton: Extend tools API to allow use of data params with multiple='true' with multiple values.
by commits-noreply＠bitbucket.org 22 Nov '13

22 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/211f30207d48/ Changeset: 211f30207d48 User: jmchilton Date: 2013-11-22 09:09:28 Summary: Extend tools API to allow use of data params with multiple='true' with multiple values. Likewise extend API driven tool testing framework to support these, add sample test cases. Also fixup english on a log warning. Affected #: 4 files diff -r 4ddfe2108c00f3ba6ff4eb097986297a2b5c98f2 -r 211f30207d48101a1250983d4f0f55c937f00a2e lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1739,7 +1739,13 @@ if isinstance( value, str ) and value.find( "," ) > 0: value = [ int( value_part ) for value_part in value.split( "," ) ] if isinstance( value, list ): - rval = [ trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( v ) for v in value ] + rval = [] + for single_value in value: + if isinstance( single_value, dict ): + assert single_value['src'] == 'hda' + rval.append( trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( trans.app.security.decode_id( single_value[ 'id' ] ) ) ) + else: + rval.append( trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( single_value ) ) elif isinstance( value, trans.app.model.HistoryDatasetAssociation ): rval = value elif isinstance( value, dict ) and 'src' in value and 'id' in value: diff -r 4ddfe2108c00f3ba6ff4eb097986297a2b5c98f2 -r 211f30207d48101a1250983d4f0f55c937f00a2e lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -179,10 +179,12 @@ raw_input = context.extract_value( raw_inputs ) if raw_input: (name, param_value, param_extra) = raw_input + param_value = self.__split_if_str( param_value ) if isinstance( value, basic.DataToolParameter ): - processed_value = [ self.__add_uploaded_dataset( context.for_state(), param_value, param_extra, value ) ] + if not isinstance(param_value, list): + param_value = [ param_value ] + processed_value = [ self.__add_uploaded_dataset( context.for_state(), v, param_extra, value ) for v in param_value ] else: - param_value = self.__split_if_str( param_value ) processed_value = param_value expanded_inputs[ context.for_state() ] = processed_value return expanded_inputs @@ -309,7 +311,7 @@ for metadata_elem in output_elem.findall( 'metadata' ): metadata[ metadata_elem.get('name') ] = metadata_elem.get( 'value' ) if not (assert_list or file or extra_files or metadata): - raise Exception( "Test output defines not checks (e.g. must have a 'file' check against, assertions to check, etc...)") + raise Exception( "Test output defines nothting to check (e.g. must have a 'file' check against, assertions to check, etc...)") attributes['assert_list'] = assert_list attributes['extra_files'] = extra_files attributes['metadata'] = metadata diff -r 4ddfe2108c00f3ba6ff4eb097986297a2b5c98f2 -r 211f30207d48101a1250983d4f0f55c937f00a2e test/base/interactor.py --- a/test/base/interactor.py +++ b/test/base/interactor.py @@ -133,9 +133,13 @@ inputs_tree = testdef.inputs.copy() for key, value in inputs_tree.iteritems(): values = [value] if not isinstance(value, list) else value + new_values = [] for value in values: if value in self.uploads: - inputs_tree[ key ] = self.uploads[ value ] + new_values.append( self.uploads[ value ] ) + else: + new_values.append( value ) + inputs_tree[ key ] = new_values # # HACK: Flatten single-value lists. Required when using expand_grouping for key, value in inputs_tree.iteritems(): diff -r 4ddfe2108c00f3ba6ff4eb097986297a2b5c98f2 -r 211f30207d48101a1250983d4f0f55c937f00a2e test/functional/tools/multi_data_param.xml --- a/test/functional/tools/multi_data_param.xml +++ b/test/functional/tools/multi_data_param.xml @@ -18,13 +18,23 @@ <output name="out1" file="simple_line.txt" /><output name="out2" file="simple_line_alternative.txt" /></test> - <test><param name="f1" value="simple_line.txt,simple_line.txt" /><param name="f2" value="simple_line_alternative.txt" /><output name="out1" file="simple_line_x2.txt" /> - <output name="out1" file="simple_line_alternative.txt" /> + <output name="out2" file="simple_line_alternative.txt" /></test> - --></tests></tool> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Add test case for data param with multiple=True even if just 1 file used.
by commits-noreply＠bitbucket.org 22 Nov '13

22 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/4ddfe2108c00/ Changeset: 4ddfe2108c00 User: jmchilton Date: 2013-11-22 08:27:47 Summary: Add test case for data param with multiple=True even if just 1 file used. Add fix to twill driver to fix this use case. Affected #: 3 files diff -r b2feff0cd2e7dbc3d9ba4b5bed55bd744406c6b1 -r 4ddfe2108c00f3ba6ff4eb097986297a2b5c98f2 test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -1305,7 +1305,15 @@ control.get( name=elem ).selected = True else: for elem in control_value: - control.get( name=elem ).selected = True + try: + # Doubt this case would ever work, but want + # to preserve backward compat. + control.get( name=elem ).selected = True + except Exception: + # ... anyway this is really what we want to + # do, probably even want to try the len( + # elem ) > 30 check below. + control.get( label=elem ).selected = True else: # control.is_of_kind( "singlelist" ) for elem in control_value: try: diff -r b2feff0cd2e7dbc3d9ba4b5bed55bd744406c6b1 -r 4ddfe2108c00f3ba6ff4eb097986297a2b5c98f2 test/functional/tools/multi_data_param.xml --- /dev/null +++ b/test/functional/tools/multi_data_param.xml @@ -0,0 +1,30 @@ +<tool id="multi_data_param" name="multi_data_param" version="0.1.0"> + <command> + cat #for $f in $f1# ${f} #end for# >> $out1; + cat #for $f in $f2# ${f} #end for# >> $out2 + </command> + <inputs> + <param name="f1" type="data" format="txt" multiple="true" /> + <param name="f2" type="data" format="txt" multiple="true" /> + </inputs> + <outputs> + <data format="txt" name="out1" /> + <data format="txt" name="out2" /> + </outputs> + <tests> + <test> + <param name="f1" value="simple_line.txt" /> + <param name="f2" value="simple_line_alternative.txt" /> + <output name="out1" file="simple_line.txt" /> + <output name="out2" file="simple_line_alternative.txt" /> + </test> +  + </tests> +</tool> diff -r b2feff0cd2e7dbc3d9ba4b5bed55bd744406c6b1 -r 4ddfe2108c00f3ba6ff4eb097986297a2b5c98f2 test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -14,4 +14,5 @@ <tool file="disambiguate_repeats.xml" /><tool file="min_repeat.xml" /><tool file="implicit_default_conds.xml" /> + <tool file="multi_data_param.xml" /></toolbox> \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Allow API driven tests via install_and_test_tool_shed_repositories framework.
by commits-noreply＠bitbucket.org 22 Nov '13

22 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b2feff0cd2e7/ Changeset: b2feff0cd2e7 User: jmchilton Date: 2013-11-22 07:50:37 Summary: Allow API driven tests via install_and_test_tool_shed_repositories framework. Greg/Dave - I don't have the infrastructure/knowledge to test this changeset, so sorry if it breaks something, but this is what I believe is nessecary if you want to set GALAXY_TEST_DEFAULT_INTERACTOR=api or if anyone posts a tool to the tool shed that forces the issue by marking a test case with 'interactor="api"'. Affected #: 1 file diff -r d896ca45280fbc7764fa2fe82fe10d7537d858dc -r b2feff0cd2e7dbc3d9ba4b5bed55bd744406c6b1 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -91,6 +91,7 @@ default_galaxy_test_port_min = 10000 default_galaxy_test_port_max = 10999 default_galaxy_test_host = '127.0.0.1' +default_galaxy_master_api_key = "123456" # should this serve static resources (scripts, images, styles, etc.) STATIC_ENABLED = True @@ -792,6 +793,7 @@ # Write the embedded web application's specific configuration to a temporary file. This is necessary in order for # the external metadata script to find the right datasets. kwargs = dict( admin_users = 'test(a)bx.psu.edu', + master_api_key = default_galaxy_master_api_key, allow_user_creation = True, allow_user_deletion = True, allow_library_path_paste = True, @@ -1181,7 +1183,7 @@ test_toolbox.toolbox = app.toolbox # Generate the test methods for this installed repository. We need to pass in True here, or it will look # in $GALAXY_HOME/test-data for test data, which may result in missing or invalid test files. - test_toolbox.build_tests( testing_shed_tools=True ) + test_toolbox.build_tests( testing_shed_tools=True, master_api_key=default_galaxy_master_api_key ) # Set up nose to run the generated functional tests. test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) test_config.configure( sys.argv ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Fixes related to implicit defaults of param values.
by commits-noreply＠bitbucket.org 22 Nov '13

22 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d896ca45280f/ Changeset: d896ca45280f User: jmchilton Date: 2013-11-22 06:37:22 Summary: Fixes related to implicit defaults of param values. Logic errors related to them being contained in repeat blocks and to picking the top value in a select by default when no other value is marked as default. Other small adjustments - add another sample tool demonstrating the problem and clean up error messages. Affected #: 3 files diff -r 1c306a40b1c3f6f3abc51f44d2bc77a5bb1d6695 -r d896ca45280fbc7764fa2fe82fe10d7537d858dc lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -78,13 +78,15 @@ if selected: default_option = name else: - default_option = test_param.static_options[0] + first_option = test_param.static_options[0] + first_option_value = first_option[1] + default_option = first_option_value matches_declared_value = lambda case_value: case_value == default_option else: # No explicit value for this param and cannot determine a # default - give up. Previously this would just result in a key # error exception. - msg = "Failed to find test parameter specification required for conditional %s" % cond + msg = "Failed to find test parameter value specification required for conditional %s" % cond.name raise Exception( msg ) # Check the tool's defined cases against predicate to determine @@ -93,7 +95,9 @@ if matches_declared_value( case.value ): return case else: - log.info("Failed to find case matching test parameter specification for cond %s. Remainder of test behavior is unspecified." % cond) + msg_template = "%s - Failed to find case matching value (%s) for test parameter specification for conditional %s. Remainder of test behavior is unspecified." + msg = msg_template % ( self.tool.id, declared_value, cond.name ) + log.info( msg ) def __split_if_str( self, value ): split = isinstance(value, str) @@ -143,10 +147,20 @@ case_value = raw_input[ 1 ] if raw_input else None case = self.__matching_case_for_value( value, case_value ) if case: - expanded_value = self.__split_if_str( case.value ) - expanded_inputs[ case_context.for_state() ] = expanded_value for input_name, input_value in case.inputs.items(): - expanded_inputs.update( self.__process_raw_inputs( { input_name: input_value }, raw_inputs, parent_context=cond_context ) ) + case_inputs = self.__process_raw_inputs( { input_name: input_value }, raw_inputs, parent_context=cond_context ) + expanded_inputs.update( case_inputs ) + expanded_case_value = self.__split_if_str( case.value ) + if case_value is not None: + # A bit tricky here - we are growing inputs with value + # that may be implicit (i.e. not defined by user just + # a default defined in tool). So we do not want to grow + # expanded_inputs and risk repeat block viewing this + # as a new instance with value defined and hence enter + # an infinite loop - hence the "case_value is not None" + # check. + expanded_inputs[ case_context.for_state() ] = expanded_case_value + elif isinstance( value, grouping.Repeat ): repeat_index = 0 while True: diff -r 1c306a40b1c3f6f3abc51f44d2bc77a5bb1d6695 -r d896ca45280fbc7764fa2fe82fe10d7537d858dc test/functional/tools/implicit_default_conds.xml --- /dev/null +++ b/test/functional/tools/implicit_default_conds.xml @@ -0,0 +1,49 @@ +<tool id="implicit_default_conds" name="implicit_default_conds"> + <command> + echo "$param_group[0].p1.val" >> $out_file1; + echo "$param_group[0].p2.val" >> $out_file1; + </command> + <inputs> + <repeat name="param_group" title="Param Group" min="1"> + <conditional name="p1"> + <param name="type" type="select"> + <option value="default">THE DEFAULT</option> + <option value="different">A different value</option> + </param> + <when value="default"> + <param name="val" value="p1default" type="text" /> + </when> + <when value="different"> + <param name="val" value="p1different" type="text" /> + </when> + </conditional> + <conditional name="p2"> + <param name="type" type="select"> + <option value="default">THE DEFAULT</option> + <option value="different" selected="true">A different value</option> + </param> + <when value="default"> + <param name="val" value="p2default" type="text" /> + </when> + <when value="different"> + <param name="val" value="p2different" type="text" /> + </when> + </conditional> + <param name="int_param" type="integer" value="8" /> + </repeat> + </inputs> + <outputs> + <data name="out_file1" format="txt" /> + </outputs> + <tests> + <test> + <param name="int_param" value="7" /> + <output name="out_file1"> + <assert_contents> + <has_line line="p1default" /> + <has_line line="p2different" /> + </assert_contents> + </output> + </test> + </tests> +</tool> diff -r 1c306a40b1c3f6f3abc51f44d2bc77a5bb1d6695 -r d896ca45280fbc7764fa2fe82fe10d7537d858dc test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -13,4 +13,5 @@ <tool file="output_order.xml" /><tool file="disambiguate_repeats.xml" /><tool file="min_repeat.xml" /> + <tool file="implicit_default_conds.xml" /></toolbox> \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Clean up tool framework test tools.
by commits-noreply＠bitbucket.org 22 Nov '13

22 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1c306a40b1c3/ Changeset: 1c306a40b1c3 User: jmchilton Date: 2013-11-22 04:17:04 Summary: Clean up tool framework test tools. Add example of nested conditional disambiugation. Affected #: 3 files diff -r 4615f7424dcb4d2b61be5f388c761f45a1302a03 -r 1c306a40b1c3f6f3abc51f44d2bc77a5bb1d6695 test/functional/tools/composite.xml --- a/test/functional/tools/composite.xml +++ b/test/functional/tools/composite.xml @@ -1,5 +1,4 @@ -<tool id="velvet_inspired" name="velvet_inspired" version="1.0.0"> - <description>Velvet sequence assembler for very short reads</description> +<tool id="composite" version="1.0.0"><command>cat '$input.extra_files_path/Sequences' > $output</command><inputs><param name="input" type="data" format="velvet" label="Velvet Dataset" help="Prepared by velveth."/> diff -r 4615f7424dcb4d2b61be5f388c761f45a1302a03 -r 1c306a40b1c3f6f3abc51f44d2bc77a5bb1d6695 test/functional/tools/disambiguate_cond.xml --- a/test/functional/tools/disambiguate_cond.xml +++ b/test/functional/tools/disambiguate_cond.xml @@ -1,6 +1,4 @@ -<tool id="handle_cond" name="handle_cond"> - <description>tail-to-head</description> -  +<tool id="disambiguate_cond" name="disambiguate_cond"><command> echo "$p1.p1v $p2.p2v $p3.p3v" > $out_file1; cat "$files.p4.file" >> $out_file1; </command> @@ -61,5 +59,29 @@ </assert_contents></output></test> + <test> + <conditional name="p1"> + <param name="use" value="False"/> + </conditional> + <conditional name="p2"> + <param name="use" value="True"/> + </conditional> + <conditional name="p3"> + <param name="use" value="False"/> + </conditional> + <conditional name="files"> + <param name="attach_files" value="True" /> + <conditional name="p4"> + <param name="use" value="True"/> + <param name="file" value="simple_line_alternative.txt" /> + </conditional> + </conditional> + <output name="out_file1"> + <assert_contents> + <has_line line="7 4 7" /> + <has_line line="This is a different line of text." /> + </assert_contents> + </output> + </test></tests></tool> diff -r 4615f7424dcb4d2b61be5f388c761f45a1302a03 -r 1c306a40b1c3f6f3abc51f44d2bc77a5bb1d6695 test/functional/tools/multi_repeats.xml --- a/test/functional/tools/multi_repeats.xml +++ b/test/functional/tools/multi_repeats.xml @@ -1,5 +1,4 @@ -<tool id="multirepeat" name="multirepeat"> - <description>tail-to-head</description> +<tool id="multi_repeats" name="multi_repeats"><command> cat $input1 #for $q in $queries# ${q.input2} #end for# #for $q in $more_queries# ${q.more_queries_input} #end for# > $out_file1 </command> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Fix API handling of tools containing repeat statement with min set.
by commits-noreply＠bitbucket.org 22 Nov '13

22 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/4615f7424dcb/ Changeset: 4615f7424dcb User: jmchilton Date: 2013-11-22 03:18:47 Summary: Fix API handling of tools containing repeat statement with min set. Affected #: 3 files diff -r f302dae15be9185139d8cc17cd0208a448e7e9b0 -r 4615f7424dcb4d2b61be5f388c761f45a1302a03 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1982,6 +1982,7 @@ group_errors = [ ] any_group_errors = False rep_index = 0 + del group_state[:] # Clear prepopulated defaults if repeat.min set. while True: rep_name = "%s_%d" % ( key, rep_index ) if not any( [ incoming_key.startswith(rep_name) for incoming_key in incoming.keys() ] ): diff -r f302dae15be9185139d8cc17cd0208a448e7e9b0 -r 4615f7424dcb4d2b61be5f388c761f45a1302a03 test/functional/tools/min_repeat.xml --- /dev/null +++ b/test/functional/tools/min_repeat.xml @@ -0,0 +1,27 @@ +<tool id="min_repeat" name="min_repeat"> + <command> + cat #for $q in $queries# ${q.input} #end for# > $out_file1 ; + cat #for $q in $queries2# ${q.input2} #end for# > $out_file2 + </command> + <inputs> + <repeat name="queries" title="Dataset" min="1"> + <param name="input" type="data" label="Select" /> + </repeat> + <repeat name="queries2" title="Dataset" min="1"> + <param name="input2" type="data" label="Select" /> + </repeat> + </inputs> + <outputs> + <data name="out_file1" format="txt" /> + <data name="out_file2" format="txt" /> + </outputs> + <tests> + <test> + <param name="input" value="simple_line.txt"/> + <param name="input" value="simple_line.txt"/> + <param name="input2" value="simple_line_alternative.txt"/> + <output name="out_file1" file="simple_line_x2.txt"/> + <output name="out_file2" file="simple_line_alternative.txt"/> + </test> + </tests> +</tool> diff -r f302dae15be9185139d8cc17cd0208a448e7e9b0 -r 4615f7424dcb4d2b61be5f388c761f45a1302a03 test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -12,4 +12,5 @@ <tool file="metadata.xml" /><tool file="output_order.xml" /><tool file="disambiguate_repeats.xml" /> + <tool file="min_repeat.xml" /></toolbox> \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 44 new changesets
by commits-noreply＠bitbucket.org 22 Nov '13

22 Nov '13

44 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/fd2ae1a670a6/ Changeset: fd2ae1a670a6 User: jmchilton Date: 2013-11-22 02:17:12 Summary: Refactor twill tool test driver to encode hda id earlier. And pass an encoded id to display action in root controller. Would definitely like to simplify or better yet remove root.display if not used anywhere else, it should be coming in through dataset controller. Affected #: 2 files diff -r 5471fa863712ba3f89ad9adceb194daf0baeafa2 -r fd2ae1a670a6caf79ad3e2baa3d3d4242bef27d6 lib/galaxy/webapps/galaxy/controllers/root.py --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -172,12 +172,16 @@ ## ---- Dataset display / editing ---------------------------------------- @web.expose - def display( self, trans, id=None, hid=None, tofile=None, toext=".txt", **kwd ): + def display( self, trans, id=None, hid=None, tofile=None, toext=".txt", encoded_id=None, **kwd ): """Returns data directly into the browser. Sets the mime-type according to the extension. + + Used by the twill tool test driver - used anywhere else? Would like to drop hid + argument and path if unneeded now. Likewise, would like to drop encoded_id=XXX + and use assume id is encoded (likely id wouldn't be coming in encoded if this + is used anywhere else though.) """ - #TODO: unused? #TODO: unencoded id if hid is not None: try: @@ -192,6 +196,8 @@ else: raise Exception( "No dataset with hid '%d'" % hid ) else: + if encoded_id and not id: + id = trans.security.decode_id( encoded_id ) try: data = trans.sa_session.query( self.app.model.HistoryDatasetAssociation ).get( id ) except: diff -r 5471fa863712ba3f89ad9adceb194daf0baeafa2 -r fd2ae1a670a6caf79ad3e2baa3d3d4242bef27d6 test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -826,8 +826,10 @@ if ext != test_ext: raise AssertionError( errmsg ) else: + hda_id = self.security.encode_id( elem.get( 'id' ) ) self.home() - self.visit_page( "display?hid=" + hid ) + # See not in controllers/root.py about encoded_id. + self.visit_page( "display?encoded_id=%s" % hda_id ) data = self.last_page() if attributes is not None and attributes.get( "assert_list", None ) is not None: try: @@ -918,7 +920,7 @@ if base_name is None: base_name = os.path.split(file_name)[-1] temp_name = self.makeTfname(fname=base_name) - self.visit_url( "%s/datasets/%s/display/%s" % ( self.url, self.security.encode_id( hda_id ), base_name ) ) + self.visit_url( "%s/datasets/%s/display/%s" % ( self.url, hda_id, base_name ) ) data = self.last_page() file( temp_name, 'wb' ).write( data ) if self.keepOutdir > '': https://bitbucket.org/galaxy/galaxy-central/commits/5c5eaf0c19da/ Changeset: 5c5eaf0c19da User: jmchilton Date: 2013-11-22 02:17:12 Summary: Refactor shared code in functional_tests.py into a method. Affected #: 1 file diff -r fd2ae1a670a6caf79ad3e2baa3d3d4242bef27d6 -r 5c5eaf0c19da5b347d05a1f664e1fc394a680735 scripts/functional_tests.py --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -415,6 +415,16 @@ os.environ[ 'GALAXY_TEST_SAVE' ] = galaxy_test_save # Pass in through script setenv, will leave a copy of ALL test validate files os.environ[ 'GALAXY_TEST_HOST' ] = galaxy_test_host + + def _run_functional_test( testing_shed_tools=None ): + functional.test_toolbox.toolbox = app.toolbox + functional.test_toolbox.build_tests( testing_shed_tools=testing_shed_tools ) + test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() ) + test_config.configure( sys.argv ) + result = run_tests( test_config ) + success = result.wasSuccessful() + return success + if testing_migrated_tools or testing_installed_tools: shed_tools_dict = {} if testing_migrated_tools: @@ -438,12 +448,7 @@ for installed_tool_panel_config in installed_tool_panel_configs: tool_configs.append( installed_tool_panel_config ) app.toolbox = tools.ToolBox( tool_configs, app.config.tool_path, app ) - functional.test_toolbox.toolbox = app.toolbox - functional.test_toolbox.build_tests( testing_shed_tools=True ) - test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() ) - test_config.configure( sys.argv ) - result = run_tests( test_config ) - success = result.wasSuccessful() + success = _run_functional_test( testing_shed_tools=True ) try: os.unlink( tmp_tool_panel_conf ) except: @@ -453,14 +458,9 @@ except: log.info( "Unable to remove file: %s" % galaxy_tool_shed_test_file ) else: - functional.test_toolbox.toolbox = app.toolbox - functional.test_toolbox.build_tests() if galaxy_test_file_dir: os.environ[ 'GALAXY_TEST_FILE_DIR' ] = galaxy_test_file_dir - test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() ) - test_config.configure( sys.argv ) - result = run_tests( test_config ) - success = result.wasSuccessful() + success = _run_functional_test( ) except: log.exception( "Failure running tests" ) https://bitbucket.org/galaxy/galaxy-central/commits/faf49c10ea03/ Changeset: faf49c10ea03 User: jmchilton Date: 2013-11-22 02:17:12 Summary: Small simplification to test/functional/test_toolbox.py. Affected #: 1 file diff -r 5c5eaf0c19da5b347d05a1f664e1fc394a680735 -r faf49c10ea033cf7fcde67ebfd426d9e344ea396 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -12,10 +12,12 @@ class ToolTestCase( TwillTestCase ): """Abstract test case that runs tests based on a `galaxy.tools.test.ToolTest`""" - def do_it( self, testdef, shed_tool_id=None ): + def do_it( self, testdef ): """ Run through a tool test case. """ + shed_tool_id = self.shed_tool_id + self.__handle_test_def_errors( testdef ) latest_history = self.__setup_test_history() @@ -206,22 +208,21 @@ for i, tool_id in enumerate( toolbox.tools_by_id ): tool = toolbox.get_tool( tool_id ) if tool.tests: + shed_tool_id = None if not testing_shed_tools else tool.id # Create a new subclass of ToolTestCase, dynamically adding methods # named test_tool_XXX that run each test defined in the tool config. name = "TestForTool_" + tool.id.replace( ' ', '_' ) baseclasses = ( ToolTestCase, ) namespace = dict() for j, testdef in enumerate( tool.tests ): - def make_test_method( td, shed_tool_id=None ): + def make_test_method( td ): def test_tool( self ): - self.do_it( td, shed_tool_id=shed_tool_id ) + self.do_it( td ) return test_tool - if testing_shed_tools: - test_method = make_test_method( testdef, shed_tool_id=tool.id ) - else: - test_method = make_test_method( testdef ) + test_method = make_test_method( testdef ) test_method.__doc__ = "%s ( %s ) > %s" % ( tool.name, tool.id, testdef.name ) namespace[ 'test_tool_%06d' % j ] = test_method + namespace[ 'shed_tool_id' ] = shed_tool_id # The new.classobj function returns a new class object, with name name, derived # from baseclasses (which should be a tuple of classes) and with namespace dict. new_class_obj = new.classobj( name, baseclasses, namespace ) https://bitbucket.org/galaxy/galaxy-central/commits/65c2a41d4a33/ Changeset: 65c2a41d4a33 User: jmchilton Date: 2013-11-22 02:17:12 Summary: Refactor test data input setup logic out of TestToolBox into ToolTestBuilder. Simplifying test_toolbox.py will hopefully make it easier to provide an alternative API driven option. Affected #: 2 files diff -r faf49c10ea033cf7fcde67ebfd426d9e344ea396 -r 65c2a41d4a336d2bf5cf40d8b980ecb2e99cb163 lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -6,6 +6,9 @@ log = logging.getLogger( __name__ ) +DEFAULT_FTYPE = 'auto' +DEFAULT_DBKEY = 'hg17' + class ToolTestBuilder( object ): """ @@ -29,6 +32,31 @@ self.__parse_elem( test_elem, i ) + def test_data( self ): + """ + Iterator over metadata representing the required files for upload. + """ + for fname, extra in self.required_files: + data_dict = dict( + fname=fname, + metadata=extra.get( 'metadata', [] ), + composite_data=extra.get( 'composite_data', [] ), + ftype=extra.get( 'ftype', DEFAULT_FTYPE ), + dbkey=extra.get( 'dbkey', DEFAULT_DBKEY ), + ) + edit_attributes = extra.get( 'edit_attributes', [] ) + + #currently only renaming is supported + for edit_att in edit_attributes: + if edit_att.get( 'type', None ) == 'name': + new_name = edit_att.get( 'value', None ) + assert new_name, 'You must supply the new dataset name as the value tag of the edit_attributes tag' + data_dict['name'] = new_name + else: + raise Exception( 'edit_attributes type (%s) is unimplemented' % edit_att.get( 'type', None ) ) + + yield data_dict + def __parse_elem( self, test_elem, i ): # Composite datasets need a unique name: each test occurs in a fresh # history, but we'll keep it unique per set of tests - use i (test #) diff -r faf49c10ea033cf7fcde67ebfd426d9e344ea396 -r 65c2a41d4a336d2bf5cf40d8b980ecb2e99cb163 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -57,32 +57,20 @@ def __setup_test_data( self, testdef, shed_tool_id ): # Upload any needed files - for fname, extra in testdef.required_files: - metadata = extra.get( 'metadata', [] ) - composite_data = extra.get( 'composite_data', [] ) - self.upload_file( fname, - ftype=extra.get( 'ftype', 'auto' ), - dbkey=extra.get( 'dbkey', 'hg17' ), - metadata=metadata, - composite_data=composite_data, + for test_data in testdef.test_data(): + self.upload_file( test_data['fname'], + ftype=test_data['ftype'], + dbkey=test_data['dbkey'], + metadata=test_data['metadata'], + composite_data=test_data['composite_data'], shed_tool_id=shed_tool_id ) - - print "Uploaded file: ", fname, ", ftype: ", extra.get( 'ftype', 'auto' ), ", extra: ", extra - #Post upload attribute editing - edit_attributes = extra.get( 'edit_attributes', [] ) - - #currently only renaming is supported - for edit_att in edit_attributes: - if edit_att.get( 'type', None ) == 'name': - new_name = edit_att.get( 'value', None ) - assert new_name, 'You must supply the new dataset name as the value tag of the edit_attributes tag' - hda_id = self.get_history_as_data_list()[-1].get( 'id' ) - try: - self.edit_hda_attribute_info( hda_id=str(hda_id), new_name=new_name ) - except: - print "### call to edit_hda failed for hda_id %s, new_name=%s" % (hda_id, new_name) - else: - raise Exception( 'edit_attributes type (%s) is unimplemented' % edit_att.get( 'type', None ) ) + name = test_data.get('name', None) + if name: + hda_id = self.get_history_as_data_list()[-1].get( 'id' ) + try: + self.edit_hda_attribute_info( hda_id=str(hda_id), new_name=name ) + except: + print "### call to edit_hda failed for hda_id %s, new_name=%s" % (hda_id, name) def __run_tool( self, testdef ): # We need to handle the case where we've uploaded a valid compressed file since the upload https://bitbucket.org/galaxy/galaxy-central/commits/3b4ed0725792/ Changeset: 3b4ed0725792 User: jmchilton Date: 2013-11-22 02:17:12 Summary: Refactor data staging and tool running logic out of test/functional/test_toolbox.py. In a new twill specific interactor class, idea here is to proceed by implementing a matching API interactor class. Affected #: 1 file diff -r 65c2a41d4a336d2bf5cf40d8b980ecb2e99cb163 -r 3b4ed072579212764322cbdd8440ecbaad836ca9 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -20,16 +20,20 @@ self.__handle_test_def_errors( testdef ) - latest_history = self.__setup_test_history() + galaxy_interactor = GalaxyInteractorTwill( self ) - self.__setup_test_data( testdef, shed_tool_id ) + test_history = galaxy_interactor.new_history() - data_list = self.__run_tool( testdef ) + # Upload any needed files + for test_data in testdef.test_data(): + galaxy_interactor.stage_data( test_data, shed_tool_id ) + + data_list = galaxy_interactor.run_tool( testdef ) self.assertTrue( data_list ) self.__verify_outputs( testdef, shed_tool_id, data_list ) - self.__delete_history( latest_history ) + galaxy_interactor.delete_history( test_history ) def __handle_test_def_errors(self, testdef): # If the test generation had an error, raise @@ -39,81 +43,6 @@ else: raise Exception( "Test parse failure" ) - def __setup_test_history( self ): - # Start with a new history - self.logout() - self.login( email='test(a)bx.psu.edu' ) - admin_user = sa_session.query( galaxy.model.User ).filter( galaxy.model.User.table.c.email == 'test(a)bx.psu.edu' ).one() - self.new_history() - latest_history = sa_session.query( galaxy.model.History ) \ - .filter( and_( galaxy.model.History.table.c.deleted == False, - galaxy.model.History.table.c.user_id == admin_user.id ) ) \ - .order_by( desc( galaxy.model.History.table.c.create_time ) ) \ - .first() - assert latest_history is not None, "Problem retrieving latest_history from database" - if len( self.get_history_as_data_list() ) > 0: - raise AssertionError("ToolTestCase.do_it failed") - return latest_history - - def __setup_test_data( self, testdef, shed_tool_id ): - # Upload any needed files - for test_data in testdef.test_data(): - self.upload_file( test_data['fname'], - ftype=test_data['ftype'], - dbkey=test_data['dbkey'], - metadata=test_data['metadata'], - composite_data=test_data['composite_data'], - shed_tool_id=shed_tool_id ) - name = test_data.get('name', None) - if name: - hda_id = self.get_history_as_data_list()[-1].get( 'id' ) - try: - self.edit_hda_attribute_info( hda_id=str(hda_id), new_name=name ) - except: - print "### call to edit_hda failed for hda_id %s, new_name=%s" % (hda_id, name) - - def __run_tool( self, testdef ): - # We need to handle the case where we've uploaded a valid compressed file since the upload - # tool will have uncompressed it on the fly. - all_inputs = {} - for name, value, _ in testdef.inputs: - all_inputs[ name ] = value - - # See if we have a grouping.Repeat element - repeat_name = None - for input_name, input_value in testdef.tool.inputs_by_page[0].items(): - if isinstance( input_value, grouping.Repeat ) and all_inputs.get( input_name, 1 ) not in [ 0, "0" ]: # default behavior is to test 1 repeat, for backwards compatibility - if not input_value.min: # If input_value.min == 1, the element is already on the page don't add new element. - repeat_name = input_name - break - - #check if we need to verify number of outputs created dynamically by tool - if testdef.tool.force_history_refresh: - job_finish_by_output_count = len( self.get_history_as_data_list() ) - else: - job_finish_by_output_count = False - - # Do the first page - page_inputs = self.__expand_grouping(testdef.tool.inputs_by_page[0], all_inputs) - - # Run the tool - self.run_tool( testdef.tool.id, repeat_name=repeat_name, **page_inputs ) - print "page_inputs (0)", page_inputs - # Do other pages if they exist - for i in range( 1, testdef.tool.npages ): - page_inputs = self.__expand_grouping(testdef.tool.inputs_by_page[i], all_inputs) - self.submit_form( **page_inputs ) - print "page_inputs (%i)" % i, page_inputs - - # Check the results ( handles single or multiple tool outputs ). Make sure to pass the correct hid. - # The output datasets from the tool should be in the same order as the testdef.outputs. - data_list = None - while data_list is None: - data_list = self.get_history_as_data_list() - if job_finish_by_output_count and len( testdef.outputs ) > ( len( data_list ) - job_finish_by_output_count ): - data_list = None - return data_list - def __verify_outputs( self, testdef, shed_tool_id, data_list ): maxseconds = testdef.maxseconds @@ -135,8 +64,87 @@ print >>sys.stderr, self.get_job_stderr( elem.get( 'id' ), format=True ) raise - def __delete_history( self, latest_history ): - self.delete_history( id=self.security.encode_id( latest_history.id ) ) + +class GalaxyInteractorTwill( object ): + + def __init__( self, twill_test_case ): + self.twill_test_case = twill_test_case + + def stage_data( self, test_data, shed_tool_id ): + self.twill_test_case.upload_file( test_data['fname'], + ftype=test_data['ftype'], + dbkey=test_data['dbkey'], + metadata=test_data['metadata'], + composite_data=test_data['composite_data'], + shed_tool_id=shed_tool_id ) + name = test_data.get('name', None) + if name: + hda_id = self.twill_test_case.get_history_as_data_list()[-1].get( 'id' ) + try: + self.twill_test_case.edit_hda_attribute_info( hda_id=str(hda_id), new_name=name ) + except: + print "### call to edit_hda failed for hda_id %s, new_name=%s" % (hda_id, name) + + def run_tool( self, testdef ): + # We need to handle the case where we've uploaded a valid compressed file since the upload + # tool will have uncompressed it on the fly. + all_inputs = {} + for name, value, _ in testdef.inputs: + all_inputs[ name ] = value + + # See if we have a grouping.Repeat element + repeat_name = None + for input_name, input_value in testdef.tool.inputs_by_page[0].items(): + if isinstance( input_value, grouping.Repeat ) and all_inputs.get( input_name, 1 ) not in [ 0, "0" ]: # default behavior is to test 1 repeat, for backwards compatibility + if not input_value.min: # If input_value.min == 1, the element is already on the page don't add new element. + repeat_name = input_name + break + + #check if we need to verify number of outputs created dynamically by tool + if testdef.tool.force_history_refresh: + job_finish_by_output_count = len( self.twill_test_case.get_history_as_data_list() ) + else: + job_finish_by_output_count = False + + # Do the first page + page_inputs = self.__expand_grouping(testdef.tool.inputs_by_page[0], all_inputs) + + # Run the tool + self.twill_test_case.run_tool( testdef.tool.id, repeat_name=repeat_name, **page_inputs ) + print "page_inputs (0)", page_inputs + # Do other pages if they exist + for i in range( 1, testdef.tool.npages ): + page_inputs = self.__expand_grouping(testdef.tool.inputs_by_page[i], all_inputs) + self.twill_test_case.submit_form( **page_inputs ) + print "page_inputs (%i)" % i, page_inputs + + # Check the results ( handles single or multiple tool outputs ). Make sure to pass the correct hid. + # The output datasets from the tool should be in the same order as the testdef.outputs. + data_list = None + while data_list is None: + data_list = self.twill_test_case.get_history_as_data_list() + if job_finish_by_output_count and len( testdef.outputs ) > ( len( data_list ) - job_finish_by_output_count ): + data_list = None + return data_list + + def new_history( self ): + # Start with a new history + self.twill_test_case.logout() + self.twill_test_case.login( email='test(a)bx.psu.edu' ) + admin_user = sa_session.query( galaxy.model.User ).filter( galaxy.model.User.table.c.email == 'test(a)bx.psu.edu' ).one() + self.twill_test_case.new_history() + latest_history = sa_session.query( galaxy.model.History ) \ + .filter( and_( galaxy.model.History.table.c.deleted == False, + galaxy.model.History.table.c.user_id == admin_user.id ) ) \ + .order_by( desc( galaxy.model.History.table.c.create_time ) ) \ + .first() + assert latest_history is not None, "Problem retrieving latest_history from database" + if len( self.twill_test_case.get_history_as_data_list() ) > 0: + raise AssertionError("ToolTestCase.do_it failed") + return latest_history + + def delete_history( self, latest_history ): + self.twill_test_case.delete_history( id=self.twill_test_case.security.encode_id( latest_history.id ) ) def __expand_grouping( self, tool_inputs, declared_inputs, prefix='' ): expanded_inputs = {} https://bitbucket.org/galaxy/galaxy-central/commits/c31db7ed3f0a/ Changeset: c31db7ed3f0a User: jmchilton Date: 2013-11-22 02:17:12 Summary: Upload a tool test's inputs in parallel. Probably small optimization in most cases since tests generally have few inputs. Affected #: 2 files diff -r 3b4ed072579212764322cbdd8440ecbaad836ca9 -r c31db7ed3f0a1970e45fc5fbe085e07d024cc30a test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -197,7 +197,7 @@ filename = os.path.join( *path ) file(filename, 'wt').write(buffer.getvalue()) - def upload_file( self, filename, ftype='auto', dbkey='unspecified (?)', space_to_tab=False, metadata=None, composite_data=None, shed_tool_id=None ): + def upload_file( self, filename, ftype='auto', dbkey='unspecified (?)', space_to_tab=False, metadata=None, composite_data=None, name=None, shed_tool_id=None, wait=True ): """ Uploads a file. If shed_tool_id has a value, we're testing tools migrated from the distribution to the tool shed, so the tool-data directory of test data files is contained in the installed tool shed repository. @@ -218,12 +218,19 @@ filename = self.get_filename( filename, shed_tool_id=shed_tool_id ) tc.formfile( "tool_form", "file_data", filename ) tc.fv( "tool_form", "space_to_tab", space_to_tab ) + if name: + # NAME is a hidden form element, so the following prop must + # set to use it. + tc.config("readonly_controls_writeable", 1) + tc.fv( "tool_form", "NAME", name ) tc.submit( "runtool_btn" ) self.home() except AssertionError, err: errmsg = "Uploading file resulted in the following exception. Make sure the file (%s) exists. " % filename errmsg += str( err ) raise AssertionError( errmsg ) + if not wait: + return # Make sure every history item has a valid hid hids = self.get_hids_in_history() for hid in hids: diff -r 3b4ed072579212764322cbdd8440ecbaad836ca9 -r c31db7ed3f0a1970e45fc5fbe085e07d024cc30a test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -25,8 +25,11 @@ test_history = galaxy_interactor.new_history() # Upload any needed files + upload_waits = [] for test_data in testdef.test_data(): - galaxy_interactor.stage_data( test_data, shed_tool_id ) + upload_waits.append( galaxy_interactor.stage_data_async( test_data, shed_tool_id ) ) + for upload_wait in upload_waits: + upload_wait() data_list = galaxy_interactor.run_tool( testdef ) self.assertTrue( data_list ) @@ -70,20 +73,24 @@ def __init__( self, twill_test_case ): self.twill_test_case = twill_test_case - def stage_data( self, test_data, shed_tool_id ): + def stage_data_async( self, test_data, shed_tool_id, async=True ): + name = test_data.get( 'name', None ) + if name: + async = False self.twill_test_case.upload_file( test_data['fname'], ftype=test_data['ftype'], dbkey=test_data['dbkey'], metadata=test_data['metadata'], composite_data=test_data['composite_data'], - shed_tool_id=shed_tool_id ) - name = test_data.get('name', None) + shed_tool_id=shed_tool_id, + wait=(not async) ) if name: hda_id = self.twill_test_case.get_history_as_data_list()[-1].get( 'id' ) try: self.twill_test_case.edit_hda_attribute_info( hda_id=str(hda_id), new_name=name ) except: print "### call to edit_hda failed for hda_id %s, new_name=%s" % (hda_id, name) + return lambda: self.twill_test_case.wait() def run_tool( self, testdef ): # We need to handle the case where we've uploaded a valid compressed file since the upload https://bitbucket.org/galaxy/galaxy-central/commits/e2df40d1cc38/ Changeset: e2df40d1cc38 User: jmchilton Date: 2013-11-22 02:17:12 Summary: Inject master API key into functional test Galaxy. Override or set external key with GALAXY_TEST_MASTER_API_KEY. Affected #: 2 files diff -r c31db7ed3f0a1970e45fc5fbe085e07d024cc30a -r e2df40d1cc38c08f6bb229c28f3a044a51013618 scripts/functional_tests.py --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -62,6 +62,7 @@ default_galaxy_test_port_max = 9999 default_galaxy_locales = 'en' default_galaxy_test_file_dir = "test-data" +default_galaxy_master_key = "TEST123" migrated_tool_panel_config = 'migrated_tools_conf.xml' installed_tool_panel_configs = [ 'shed_tool_conf.xml' ] @@ -296,6 +297,7 @@ pass # ---- Build Application -------------------------------------------------- + master_api_key = os.environ.get( "GALAXY_TEST_MASTER_API_KEY", default_galaxy_master_key ) app = None if start_server: kwargs = dict( admin_users='test(a)bx.psu.edu', @@ -322,7 +324,9 @@ tool_parse_help=False, update_integrated_tool_panel=False, use_heartbeat=False, - user_library_import_dir=user_library_import_dir ) + user_library_import_dir=user_library_import_dir, + master_api_key=master_api_key, + ) if psu_production: kwargs[ 'global_conf' ] = None if not database_connection.startswith( 'sqlite://' ): @@ -418,7 +422,10 @@ def _run_functional_test( testing_shed_tools=None ): functional.test_toolbox.toolbox = app.toolbox - functional.test_toolbox.build_tests( testing_shed_tools=testing_shed_tools ) + functional.test_toolbox.build_tests( + testing_shed_tools=testing_shed_tools, + master_api_key=master_api_key, + ) test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() ) test_config.configure( sys.argv ) result = run_tests( test_config ) diff -r c31db7ed3f0a1970e45fc5fbe085e07d024cc30a -r e2df40d1cc38c08f6bb229c28f3a044a51013618 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -191,7 +191,7 @@ return expanded_inputs -def build_tests( testing_shed_tools=False ): +def build_tests( testing_shed_tools=False, master_api_key=None ): """ If the module level variable `toolbox` is set, generate `ToolTestCase` classes for all of its tests and put them into this modules globals() so @@ -226,6 +226,7 @@ test_method.__doc__ = "%s ( %s ) > %s" % ( tool.name, tool.id, testdef.name ) namespace[ 'test_tool_%06d' % j ] = test_method namespace[ 'shed_tool_id' ] = shed_tool_id + namespace[ 'master_api_key' ] = master_api_key # The new.classobj function returns a new class object, with name name, derived # from baseclasses (which should be a tuple of classes) and with namespace dict. new_class_obj = new.classobj( name, baseclasses, namespace ) https://bitbucket.org/galaxy/galaxy-central/commits/3f105f22fc57/ Changeset: 3f105f22fc57 User: jmchilton Date: 2013-11-22 02:17:12 Summary: Tool functional tests - refactor reusable function for verifing output ... from verify_dataset_correctness. Introduce simple, extensible abstraction for fetching outputs for verification in twilltestcase.py. Will want to reuse the core verification of the displayed dataset in the API driven case and will not need or be able to use the extra cruft related to checking UI, job state, etc... Also small twilltestcase code structure updates for Python 2.6+. Affected #: 1 file diff -r e2df40d1cc38c08f6bb229c28f3a044a51013618 -r 3f105f22fc5781d89421f5bcb6d9a0bd5825e9b0 test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -833,70 +833,83 @@ if ext != test_ext: raise AssertionError( errmsg ) else: + # See not in controllers/root.py about encoded_id. hda_id = self.security.encode_id( elem.get( 'id' ) ) - self.home() - # See not in controllers/root.py about encoded_id. - self.visit_page( "display?encoded_id=%s" % hda_id ) + self.verify_hid( filename, hid=hid, hda_id=hda_id, attributes=attributes, shed_tool_id=shed_tool_id) + + def verify_hid( self, filename, hda_id, attributes, shed_tool_id, hid="", dataset_fetcher=None): + dataset_fetcher = dataset_fetcher or self.__default_dataset_fetcher() + data = dataset_fetcher( hda_id ) + if attributes is not None and attributes.get( "assert_list", None ) is not None: + try: + verify_assertions(data, attributes["assert_list"]) + except AssertionError, err: + errmsg = 'History item %s different than expected\n' % (hid) + errmsg += str( err ) + raise AssertionError( errmsg ) + if filename is not None: + local_name = self.get_filename( filename, shed_tool_id=shed_tool_id ) + temp_name = self.makeTfname(fname=filename) + file( temp_name, 'wb' ).write( data ) + + # if the server's env has GALAXY_TEST_SAVE, save the output file to that dir + if self.keepOutdir: + ofn = os.path.join( self.keepOutdir, os.path.basename( local_name ) ) + log.debug( 'keepoutdir: %s, ofn: %s', self.keepOutdir, ofn ) + try: + shutil.copy( temp_name, ofn ) + except Exception, exc: + error_log_msg = ( 'TwillTestCase could not save output file %s to %s: ' % ( temp_name, ofn ) ) + error_log_msg += str( exc ) + log.error( error_log_msg, exc_info=True ) + else: + log.debug('## GALAXY_TEST_SAVE=%s. saved %s' % ( self.keepOutdir, ofn ) ) + try: + if attributes is None: + attributes = {} + compare = attributes.get( 'compare', 'diff' ) + if attributes.get( 'ftype', None ) == 'bam': + local_fh, temp_name = self._bam_to_sam( local_name, temp_name ) + local_name = local_fh.name + extra_files = attributes.get( 'extra_files', None ) + if compare == 'diff': + self.files_diff( local_name, temp_name, attributes=attributes ) + elif compare == 're_match': + self.files_re_match( local_name, temp_name, attributes=attributes ) + elif compare == 're_match_multiline': + self.files_re_match_multiline( local_name, temp_name, attributes=attributes ) + elif compare == 'sim_size': + delta = attributes.get('delta', '100') + s1 = len(data) + s2 = os.path.getsize(local_name) + if abs(s1 - s2) > int(delta): + raise Exception( 'Files %s=%db but %s=%db - compare (delta=%s) failed' % (temp_name, s1, local_name, s2, delta) ) + elif compare == "contains": + self.files_contains( local_name, temp_name, attributes=attributes ) + else: + raise Exception( 'Unimplemented Compare type: %s' % compare ) + if extra_files: + self.verify_extra_files_content( extra_files, hda_id, shed_tool_id=shed_tool_id, dataset_fetcher=dataset_fetcher ) + except AssertionError, err: + errmsg = 'History item %s different than expected, difference (using %s):\n' % ( hid, compare ) + errmsg += "( %s v. %s )\n" % ( local_name, temp_name ) + errmsg += str( err ) + raise AssertionError( errmsg ) + finally: + os.remove( temp_name ) + + def __default_dataset_fetcher( self ): + def fetcher( hda_id, filename=None ): + if filename is None: + page_url = "display?encoded_id=%s" % hda_id + self.home() # I assume this is not needed. + else: + page_url = "datasets/%s/display/%s" % ( hda_id, filename ) + self.visit_page( page_url ) data = self.last_page() - if attributes is not None and attributes.get( "assert_list", None ) is not None: - try: - verify_assertions(data, attributes["assert_list"]) - except AssertionError, err: - errmsg = 'History item %s different than expected\n' % (hid) - errmsg += str( err ) - raise AssertionError( errmsg ) - if filename is not None: - local_name = self.get_filename( filename, shed_tool_id=shed_tool_id ) - temp_name = self.makeTfname(fname=filename) - file( temp_name, 'wb' ).write( data ) + return data - # if the server's env has GALAXY_TEST_SAVE, save the output file to that dir - if self.keepOutdir: - ofn = os.path.join( self.keepOutdir, os.path.basename( local_name ) ) - log.debug( 'keepoutdir: %s, ofn: %s', self.keepOutdir, ofn ) - try: - shutil.copy( temp_name, ofn ) - except Exception, exc: - error_log_msg = ( 'TwillTestCase could not save output file %s to %s: ' % ( temp_name, ofn ) ) - error_log_msg += str( exc ) - log.error( error_log_msg, exc_info=True ) - else: - log.debug('## GALAXY_TEST_SAVE=%s. saved %s' % ( self.keepOutdir, ofn ) ) - try: - # have to nest try-except in try-finally to handle 2.4 - try: - if attributes is None: - attributes = {} - compare = attributes.get( 'compare', 'diff' ) - if attributes.get( 'ftype', None ) == 'bam': - local_fh, temp_name = self._bam_to_sam( local_name, temp_name ) - local_name = local_fh.name - extra_files = attributes.get( 'extra_files', None ) - if compare == 'diff': - self.files_diff( local_name, temp_name, attributes=attributes ) - elif compare == 're_match': - self.files_re_match( local_name, temp_name, attributes=attributes ) - elif compare == 're_match_multiline': - self.files_re_match_multiline( local_name, temp_name, attributes=attributes ) - elif compare == 'sim_size': - delta = attributes.get('delta', '100') - s1 = len(data) - s2 = os.path.getsize(local_name) - if abs(s1 - s2) > int(delta): - raise Exception( 'Files %s=%db but %s=%db - compare (delta=%s) failed' % (temp_name, s1, local_name, s2, delta) ) - elif compare == "contains": - self.files_contains( local_name, temp_name, attributes=attributes ) - else: - raise Exception( 'Unimplemented Compare type: %s' % compare ) - if extra_files: - self.verify_extra_files_content( extra_files, elem.get( 'id' ), shed_tool_id=shed_tool_id ) - except AssertionError, err: - errmsg = 'History item %s different than expected, difference (using %s):\n' % ( hid, compare ) - errmsg += "( %s v. %s )\n" % ( local_name, temp_name ) - errmsg += str( err ) - raise AssertionError( errmsg ) - finally: - os.remove( temp_name ) + return fetcher def _bam_to_sam( self, local_name, temp_name ): temp_local = tempfile.NamedTemporaryFile( suffix='.sam', prefix='local_bam_converted_to_sam_' ) @@ -909,7 +922,7 @@ os.remove( temp_name ) return temp_local, temp_temp - def verify_extra_files_content( self, extra_files, hda_id, shed_tool_id=None ): + def verify_extra_files_content( self, extra_files, hda_id, dataset_fetcher, shed_tool_id=None ): files_list = [] for extra_type, extra_value, extra_name, extra_attributes in extra_files: if extra_type == 'file': @@ -920,44 +933,42 @@ else: raise ValueError( 'unknown extra_files type: %s' % extra_type ) for filename, filepath, attributes in files_list: - self.verify_composite_datatype_file_content( filepath, hda_id, base_name=filename, attributes=attributes, shed_tool_id=shed_tool_id ) + self.verify_composite_datatype_file_content( filepath, hda_id, base_name=filename, attributes=attributes, dataset_fetcher=dataset_fetcher, shed_tool_id=shed_tool_id ) - def verify_composite_datatype_file_content( self, file_name, hda_id, base_name=None, attributes=None, shed_tool_id=None ): + def verify_composite_datatype_file_content( self, file_name, hda_id, base_name=None, attributes=None, dataset_fetcher=None, shed_tool_id=None ): + dataset_fetcher = dataset_fetcher or self.__default_dataset_fetcher() local_name = self.get_filename( file_name, shed_tool_id=shed_tool_id ) if base_name is None: base_name = os.path.split(file_name)[-1] temp_name = self.makeTfname(fname=base_name) - self.visit_url( "%s/datasets/%s/display/%s" % ( self.url, hda_id, base_name ) ) - data = self.last_page() + data = dataset_fetcher( hda_id, base_name ) file( temp_name, 'wb' ).write( data ) if self.keepOutdir > '': ofn = os.path.join(self.keepOutdir, base_name) shutil.copy(temp_name, ofn) log.debug('## GALAXY_TEST_SAVE=%s. saved %s' % (self.keepOutdir, ofn)) try: - # have to nest try-except in try-finally to handle 2.4 - try: - if attributes is None: - attributes = {} - compare = attributes.get( 'compare', 'diff' ) - if compare == 'diff': - self.files_diff( local_name, temp_name, attributes=attributes ) - elif compare == 're_match': - self.files_re_match( local_name, temp_name, attributes=attributes ) - elif compare == 're_match_multiline': - self.files_re_match_multiline( local_name, temp_name, attributes=attributes ) - elif compare == 'sim_size': - delta = attributes.get('delta', '100') - s1 = len(data) - s2 = os.path.getsize(local_name) - if abs(s1 - s2) > int(delta): - raise Exception( 'Files %s=%db but %s=%db - compare (delta=%s) failed' % (temp_name, s1, local_name, s2, delta) ) - else: - raise Exception( 'Unimplemented Compare type: %s' % compare ) - except AssertionError, err: - errmsg = 'Composite file (%s) of History item %s different than expected, difference (using %s):\n' % ( base_name, hda_id, compare ) - errmsg += str( err ) - raise AssertionError( errmsg ) + if attributes is None: + attributes = {} + compare = attributes.get( 'compare', 'diff' ) + if compare == 'diff': + self.files_diff( local_name, temp_name, attributes=attributes ) + elif compare == 're_match': + self.files_re_match( local_name, temp_name, attributes=attributes ) + elif compare == 're_match_multiline': + self.files_re_match_multiline( local_name, temp_name, attributes=attributes ) + elif compare == 'sim_size': + delta = attributes.get('delta', '100') + s1 = len(data) + s2 = os.path.getsize(local_name) + if abs(s1 - s2) > int(delta): + raise Exception( 'Files %s=%db but %s=%db - compare (delta=%s) failed' % (temp_name, s1, local_name, s2, delta) ) + else: + raise Exception( 'Unimplemented Compare type: %s' % compare ) + except AssertionError, err: + errmsg = 'Composite file (%s) of History item %s different than expected, difference (using %s):\n' % ( base_name, hda_id, compare ) + errmsg += str( err ) + raise AssertionError( errmsg ) finally: os.remove( temp_name ) https://bitbucket.org/galaxy/galaxy-central/commits/a35a8198b8fd/ Changeset: a35a8198b8fd User: jmchilton Date: 2013-11-22 02:17:12 Summary: Outline new API based galaxy interactor for functional tests. Add mechanism to tool test parser code to determine which Galaxy interactor should be used ('twill' or 'api' are current options). This maybe a stop gap to force usage of the API interactor until that becomes the default or may prove in the long term to be essential if there are certain tools that will always require Twill-specific functionality or if new browser-based JavaScript (e.g. w/selenium) are implemented. An interactor value can be specified at the tests level and/or at the level of individual test elements in the tool XML (using the 'interactor' attribute on either element). The current default interactor is 'twill'. The default interactor app-wide can be overridden with the GALAXY_TEST_DEFAULT_INTERACTOR environment variable. Affected #: 3 files diff -r 3f105f22fc5781d89421f5bcb6d9a0bd5825e9b0 -r a35a8198b8fd1a0fdea1183ca299fa6ceb41d060 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -52,7 +52,7 @@ from galaxy.tools.parameters.output import ToolOutputActionGroup from galaxy.tools.parameters.validation import LateValidationError from galaxy.tools.filters import FilterFactory -from galaxy.tools.test import ToolTestBuilder +from galaxy.tools.test import parse_tests_elem from galaxy.util import listify, parse_xml, rst_to_html, string_as_bool, string_to_object, xml_text, xml_to_string from galaxy.util.bunch import Bunch from galaxy.util.expressions import ExpressionContext @@ -1222,7 +1222,7 @@ tests_elem = root.find( "tests" ) if tests_elem: try: - self.parse_tests( tests_elem ) + self.tests = parse_tests_elem( self, tests_elem ) except: log.exception( "Failed to parse tool tests" ) else: @@ -1564,16 +1564,6 @@ log.error( "Traceback: %s" % trace_msg ) return return_level - def parse_tests( self, tests_elem ): - """ - Parse any "<test>" elements, create a `ToolTestBuilder` for each and - store in `self.tests`. - """ - self.tests = [] - for i, test_elem in enumerate( tests_elem.findall( 'test' ) ): - test = ToolTestBuilder( self, test_elem, i ) - self.tests.append( test ) - def parse_input_page( self, input_elem, enctypes ): """ Parse a page of inputs. This basically just calls 'parse_input_elem', diff -r 3f105f22fc5781d89421f5bcb6d9a0bd5825e9b0 -r a35a8198b8fd1a0fdea1183ca299fa6ceb41d060 lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -1,3 +1,4 @@ +import os import os.path from parameters import basic from parameters import grouping @@ -8,6 +9,21 @@ DEFAULT_FTYPE = 'auto' DEFAULT_DBKEY = 'hg17' +DEFAULT_INTERACTOR = "twill" # Default mechanism test code uses for interacting with Galaxy instance. + + +def parse_tests_elem(tool, tests_elem): + """ + Build ToolTestBuilder objects for each "<test>" elements and + return default interactor (if any). + """ + default_interactor = os.environ.get( 'GALAXY_TEST_DEFAULT_INTERACTOR', DEFAULT_INTERACTOR ) + tests_default_interactor = tests_elem.get( 'interactor', default_interactor ) + tests = [] + for i, test_elem in enumerate( tests_elem.findall( 'test' ) ): + test = ToolTestBuilder( tool, test_elem, i, default_interactor=tests_default_interactor ) + tests.append( test ) + return tests class ToolTestBuilder( object ): @@ -17,7 +33,7 @@ doing dynamic tests in this way allows better integration) """ - def __init__( self, tool, test_elem, i ): + def __init__( self, tool, test_elem, i, default_interactor ): name = test_elem.get( 'name', 'Test-%d' % (i + 1) ) maxseconds = int( test_elem.get( 'maxseconds', '120' ) ) @@ -30,7 +46,7 @@ self.error = False self.exception = None - self.__parse_elem( test_elem, i ) + self.__parse_elem( test_elem, i, default_interactor ) def test_data( self ): """ @@ -57,12 +73,18 @@ yield data_dict - def __parse_elem( self, test_elem, i ): + def __parse_elem( self, test_elem, i, default_interactor ): # Composite datasets need a unique name: each test occurs in a fresh # history, but we'll keep it unique per set of tests - use i (test #) # and composite_data_names_counter (instance per test #) composite_data_names_counter = 0 try: + # Mechanism test code uses for interacting with Galaxy instance, + # until 'api' is the default switch this to API to use its new + # features. Once 'api' is the default set to 'twill' to use legacy + # features or workarounds. + self.interactor = test_elem.get( 'interactor', default_interactor ) + for param_elem in test_elem.findall( "param" ): attrib = dict( param_elem.attrib ) if 'values' in attrib: diff -r 3f105f22fc5781d89421f5bcb6d9a0bd5825e9b0 -r a35a8198b8fd1a0fdea1183ca299fa6ceb41d060 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -1,6 +1,8 @@ import sys import new +import os from galaxy.tools.parameters import grouping +from galaxy.util import string_as_bool from base.twilltestcase import TwillTestCase import galaxy.model from galaxy.model.orm import and_, desc @@ -20,7 +22,7 @@ self.__handle_test_def_errors( testdef ) - galaxy_interactor = GalaxyInteractorTwill( self ) + galaxy_interactor = self.__galaxy_interactor( testdef ) test_history = galaxy_interactor.new_history() @@ -38,6 +40,11 @@ galaxy_interactor.delete_history( test_history ) + def __galaxy_interactor( self, testdef ): + interactor_key = testdef.interactor + interactor_class = GALAXY_INTERACTORS[ interactor_key ] + return interactor_class( self ) + def __handle_test_def_errors(self, testdef): # If the test generation had an error, raise if testdef.error: @@ -68,6 +75,25 @@ raise +class GalaxyInteractorApi( object ): + + def __init__( self, twill_test_case ): + self.twill_test_case = twill_test_case + self.master_api_key = twill_test_case.master_api_key + + def new_history( self ): + return None + + def stage_data_async( self, test_data, shed_tool_id, async=True ): + return lambda: True + + def run_tool( self, testdef ): + return [] + + def delete_history( self, history ): + return None + + class GalaxyInteractorTwill( object ): def __init__( self, twill_test_case ): @@ -231,3 +257,9 @@ # from baseclasses (which should be a tuple of classes) and with namespace dict. new_class_obj = new.classobj( name, baseclasses, namespace ) G[ name ] = new_class_obj + + +GALAXY_INTERACTORS = { + 'api': GalaxyInteractorApi, + 'twill': GalaxyInteractorTwill, +} https://bitbucket.org/galaxy/galaxy-central/commits/15c6170f58c8/ Changeset: 15c6170f58c8 User: jmchilton Date: 2013-11-22 02:17:12 Summary: Twill-less tool tests initial implementation. Implement GalaxyInteractorApi, is working for simple tests. Lots left to do likely - conditionals, repeats, composite uploads, pages(?). Introduces dependency on Python requests package - subsequent changeset will provide urllib2 based fallback if requests unavailable. Affected #: 1 file diff -r a35a8198b8fd1a0fdea1183ca299fa6ceb41d060 -r 15c6170f58c87ed719efb37b62c9eab1c2956102 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -7,6 +7,8 @@ import galaxy.model from galaxy.model.orm import and_, desc from galaxy.model.mapping import context as sa_session +from simplejson import dumps +import requests toolbox = None @@ -29,14 +31,14 @@ # Upload any needed files upload_waits = [] for test_data in testdef.test_data(): - upload_waits.append( galaxy_interactor.stage_data_async( test_data, shed_tool_id ) ) + upload_waits.append( galaxy_interactor.stage_data_async( test_data, test_history, shed_tool_id ) ) for upload_wait in upload_waits: upload_wait() - data_list = galaxy_interactor.run_tool( testdef ) + data_list = galaxy_interactor.run_tool( testdef, test_history ) self.assertTrue( data_list ) - self.__verify_outputs( testdef, shed_tool_id, data_list ) + self.__verify_outputs( testdef, test_history, shed_tool_id, data_list, galaxy_interactor ) galaxy_interactor.delete_history( test_history ) @@ -53,53 +55,167 @@ else: raise Exception( "Test parse failure" ) - def __verify_outputs( self, testdef, shed_tool_id, data_list ): + def __verify_outputs( self, testdef, history, shed_tool_id, data_list, galaxy_interactor ): maxseconds = testdef.maxseconds - elem_index = 0 - len( testdef.outputs ) + output_index = 0 - len( testdef.outputs ) for output_tuple in testdef.outputs: # Get the correct hid - elem = data_list[ elem_index ] - self.assertTrue( elem is not None ) - self.__verify_output( output_tuple, shed_tool_id, elem, maxseconds=maxseconds ) - elem_index += 1 - - def __verify_output( self, output_tuple, shed_tool_id, elem, maxseconds ): + output_data = data_list[ output_index ] + self.assertTrue( output_data is not None ) name, outfile, attributes = output_tuple - elem_hid = elem.get( 'hid' ) - try: - self.verify_dataset_correctness( outfile, hid=elem_hid, attributes=attributes, shed_tool_id=shed_tool_id ) - except Exception: - print >>sys.stderr, self.get_job_stdout( elem.get( 'id' ), format=True ) - print >>sys.stderr, self.get_job_stderr( elem.get( 'id' ), format=True ) - raise + galaxy_interactor.verify_output( history, output_data, outfile, attributes=attributes, shed_tool_id=shed_tool_id, maxseconds=maxseconds ) + output_index += 1 class GalaxyInteractorApi( object ): def __init__( self, twill_test_case ): self.twill_test_case = twill_test_case - self.master_api_key = twill_test_case.master_api_key + self.api_url = "%s/api" % twill_test_case.url.rstrip("/") + self.api_key = self.__get_user_key( twill_test_case.master_api_key ) + self.uploads = {} + + def verify_output( self, history_id, output_data, outfile, attributes, shed_tool_id, maxseconds ): + hid = output_data.get( 'id' ) + try: + fetcher = self.__dataset_fetcher( history_id ) + self.twill_test_case.verify_hid( outfile, hda_id=hid, attributes=attributes, dataset_fetcher=fetcher, shed_tool_id=shed_tool_id ) + except Exception: + ## TODO: Print this! + # print >>sys.stderr, self.twill_test_case.get_job_stdout( output_data.get( 'id' ), format=True ) + ## TODO: Print this! + # print >>sys.stderr, self.twill_test_case.get_job_stderr( output_data.get( 'id' ), format=True ) + raise def new_history( self ): - return None + history_json = self.__post( "histories", {"name": "test_history"} ).json() + return history_json[ 'id' ] - def stage_data_async( self, test_data, shed_tool_id, async=True ): - return lambda: True + def stage_data_async( self, test_data, history_id, shed_tool_id, async=True ): + fname = test_data[ 'fname' ] + file_name = self.twill_test_case.get_filename( fname, shed_tool_id=shed_tool_id ) + name = test_data.get( 'name', None ) + if not name: + name = os.path.basename( file_name ) + tool_input = { + "file_type": test_data[ 'ftype' ], + "dbkey": test_data[ 'dbkey' ], # TODO: Handle it! Doesn't work if undefined, does seem to in Twill. + "files_0|NAME": name, + "files_0|type": "upload_dataset", + } + files = { + "files_0|file_data": open( file_name, 'rb') + } + submit_response = self.__submit_tool( history_id, "upload1", tool_input, extra_data={"type": "upload_dataset"}, files=files ).json() + dataset = submit_response["outputs"][0] + #raise Exception(str(dataset)) + hid = dataset['id'] + self.uploads[ fname ] = {"src": "hda", "id": hid} + return self.__wait_for_history( history_id ) - def run_tool( self, testdef ): - return [] + def run_tool( self, testdef, history_id ): + # We need to handle the case where we've uploaded a valid compressed file since the upload + # tool will have uncompressed it on the fly. + all_inputs = {} + for name, value, _ in testdef.inputs: + all_inputs[ name ] = value + + for key, value in all_inputs.iteritems(): + # TODO: Restrict this to param inputs. + if value in self.uploads: + all_inputs[key] = self.uploads[ value ] + + # TODO: Handle repeats. + # TODO: Handle pages. + # TODO: Handle force_history_refresh + datasets = self.__submit_tool( history_id, tool_id=testdef.tool.id, tool_input=all_inputs ) + self.__wait_for_history( history_id )() # TODO: Remove and respect maxseconds! + return datasets.json()[ 'outputs' ] + + def output_hid( self, output_data ): + return output_data[ 'id' ] def delete_history( self, history ): return None + def __wait_for_history( self, history_id ): + def wait(): + while True: + history_json = self.__get( "histories/%s" % history_id ).json() + state = history_json[ 'state' ] + if state == 'ok': + #raise Exception(str(self.__get( self.__get( "histories/%s/contents" % history_id ).json()[0]['url'] ).json() ) ) + #raise Exception(str(self.__get( self.__get( "histories/%s/contents" % history_id ).json()[0]['url'] ).json() ) ) + break + elif state == 'error': + raise Exception("History in error state.") + return wait + + def __submit_tool( self, history_id, tool_id, tool_input, extra_data={}, files=None ): + data = dict( + history_id=history_id, + tool_id=tool_id, + inputs=dumps( tool_input ), + **extra_data + ) + return self.__post( "tools", files=files, data=data ) + + def __get_user_key( self, admin_key ): + all_users = self.__get( 'users', key=admin_key ).json() + try: + test_user = [ user for user in all_users if user["email"] == 'test(a)bx.psu.edu' ][0] + except IndexError: + data = dict( + email='test(a)bx.psu.edu', + password='testuser', + username='admin-user', + ) + test_user = self.__post( 'users', data, key=admin_key ).json() + return self.__post( "users/%s/api_key" % test_user['id'], key=admin_key ).json() + + def __dataset_fetcher( self, history_id ): + def fetcher( hda_id, base_name=None ): + url = "histories/%s/contents/%s/display" % (history_id, hda_id) + if base_name: + url += "&filename=%s" % base_name + return self.__get( url ).text + + return fetcher + + def __post( self, path, data={}, files=None, key=None): + if not key: + key = self.api_key + data = data.copy() + data['key'] = key + return requests.post( "%s/%s" % (self.api_url, path), data=data, files=files ) + + def __get( self, path, data={}, key=None ): + if not key: + key = self.api_key + data = data.copy() + data['key'] = key + if path.startswith("/api"): + path = path[ len("/api"): ] + url = "%s/%s" % (self.api_url, path) + return requests.get( url, params=data ) + class GalaxyInteractorTwill( object ): def __init__( self, twill_test_case ): self.twill_test_case = twill_test_case - def stage_data_async( self, test_data, shed_tool_id, async=True ): + def verify_output( self, history, output_data, outfile, attributes, shed_tool_id, maxseconds ): + hid = output_data.get( 'hid' ) + try: + self.twill_test_case.verify_dataset_correctness( outfile, hid=hid, attributes=attributes, shed_tool_id=shed_tool_id ) + except Exception: + print >>sys.stderr, self.twill_test_case.get_job_stdout( output_data.get( 'id' ), format=True ) + print >>sys.stderr, self.twill_test_case.get_job_stderr( output_data.get( 'id' ), format=True ) + raise + + def stage_data_async( self, test_data, history, shed_tool_id, async=True ): name = test_data.get( 'name', None ) if name: async = False @@ -118,7 +234,7 @@ print "### call to edit_hda failed for hda_id %s, new_name=%s" % (hda_id, name) return lambda: self.twill_test_case.wait() - def run_tool( self, testdef ): + def run_tool( self, testdef, test_history ): # We need to handle the case where we've uploaded a valid compressed file since the upload # tool will have uncompressed it on the fly. all_inputs = {} @@ -179,6 +295,9 @@ def delete_history( self, latest_history ): self.twill_test_case.delete_history( id=self.twill_test_case.security.encode_id( latest_history.id ) ) + def output_hid( self, output_data ): + return output_data.get( 'hid' ) + def __expand_grouping( self, tool_inputs, declared_inputs, prefix='' ): expanded_inputs = {} for key, value in tool_inputs.items(): https://bitbucket.org/galaxy/galaxy-central/commits/1411df5a5fcc/ Changeset: 1411df5a5fcc User: jmchilton Date: 2013-11-22 02:17:12 Summary: Twill-less tool tests - allow tests with GALAXY_TEST_EXTERNAL... ... if GALAXY_TEST_EXTERNAL is ever fixed. Affected #: 2 files diff -r 15c6170f58c87ed719efb37b62c9eab1c2956102 -r 1411df5a5fcc2686512b5560435279f305fc755a scripts/functional_tests.py --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -63,6 +63,7 @@ default_galaxy_locales = 'en' default_galaxy_test_file_dir = "test-data" default_galaxy_master_key = "TEST123" +default_galaxy_user_key = None migrated_tool_panel_config = 'migrated_tools_conf.xml' installed_tool_panel_configs = [ 'shed_tool_conf.xml' ] @@ -425,6 +426,7 @@ functional.test_toolbox.build_tests( testing_shed_tools=testing_shed_tools, master_api_key=master_api_key, + user_api_key=os.environ.get( "GALAXY_TEST_USER_API_KEY", default_galaxy_user_key ), ) test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() ) test_config.configure( sys.argv ) diff -r 15c6170f58c87ed719efb37b62c9eab1c2956102 -r 1411df5a5fcc2686512b5560435279f305fc755a test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -73,7 +73,7 @@ def __init__( self, twill_test_case ): self.twill_test_case = twill_test_case self.api_url = "%s/api" % twill_test_case.url.rstrip("/") - self.api_key = self.__get_user_key( twill_test_case.master_api_key ) + self.api_key = self.__get_user_key( twill_test_case.user_api_key, twill_test_case.master_api_key ) self.uploads = {} def verify_output( self, history_id, output_data, outfile, attributes, shed_tool_id, maxseconds ): @@ -161,7 +161,9 @@ ) return self.__post( "tools", files=files, data=data ) - def __get_user_key( self, admin_key ): + def __get_user_key( self, user_key, admin_key ): + if user_key: + return user_key all_users = self.__get( 'users', key=admin_key ).json() try: test_user = [ user for user in all_users if user["email"] == 'test(a)bx.psu.edu' ][0] @@ -336,7 +338,7 @@ return expanded_inputs -def build_tests( testing_shed_tools=False, master_api_key=None ): +def build_tests( testing_shed_tools=False, master_api_key=None, user_api_key=None ): """ If the module level variable `toolbox` is set, generate `ToolTestCase` classes for all of its tests and put them into this modules globals() so @@ -372,6 +374,7 @@ namespace[ 'test_tool_%06d' % j ] = test_method namespace[ 'shed_tool_id' ] = shed_tool_id namespace[ 'master_api_key' ] = master_api_key + namespace[ 'user_api_key' ] = user_api_key # The new.classobj function returns a new class object, with name name, derived # from baseclasses (which should be a tuple of classes) and with namespace dict. new_class_obj = new.classobj( name, baseclasses, namespace ) https://bitbucket.org/galaxy/galaxy-central/commits/2dcc0a0e3732/ Changeset: 2dcc0a0e3732 User: jmchilton Date: 2013-11-22 02:17:12 Summary: Move test_toolbox's expand_grouping out into tool/test.py. Will be merged with related tree code already in test.py in a subsequent changeset. Affected #: 2 files diff -r 1411df5a5fcc2686512b5560435279f305fc755a -r 2dcc0a0e3732cf32b603279fdc443c6a2a9d9ae8 lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -73,6 +73,43 @@ yield data_dict + def expand_grouping( self, tool_inputs, declared_inputs, prefix='' ): + expanded_inputs = {} + for key, value in tool_inputs.items(): + if isinstance( value, grouping.Conditional ): + if prefix: + new_prefix = "%s|%s" % ( prefix, value.name ) + else: + new_prefix = value.name + for i, case in enumerate( value.cases ): + if declared_inputs[ value.test_param.name ] == case.value: + if isinstance(case.value, str): + expanded_inputs[ "%s|%s" % ( new_prefix, value.test_param.name ) ] = case.value.split( "," ) + else: + expanded_inputs[ "%s|%s" % ( new_prefix, value.test_param.name ) ] = case.value + for input_name, input_value in case.inputs.items(): + expanded_inputs.update( self.expand_grouping( { input_name: input_value }, declared_inputs, prefix=new_prefix ) ) + elif isinstance( value, grouping.Repeat ): + for repeat_index in xrange( 0, 1 ): # need to allow for and figure out how many repeats we have + for r_name, r_value in value.inputs.iteritems(): + new_prefix = "%s_%d" % ( value.name, repeat_index ) + if prefix: + new_prefix = "%s|%s" % ( prefix, new_prefix ) + expanded_inputs.update( self.expand_grouping( { new_prefix : r_value }, declared_inputs, prefix=new_prefix ) ) + elif value.name not in declared_inputs: + print "%s not declared in tool test, will not change default value." % value.name + elif isinstance(declared_inputs[value.name], str): + if prefix: + expanded_inputs["%s|%s" % ( prefix, value.name ) ] = declared_inputs[value.name].split(",") + else: + expanded_inputs[value.name] = declared_inputs[value.name].split(",") + else: + if prefix: + expanded_inputs["%s|%s" % ( prefix, value.name ) ] = declared_inputs[value.name] + else: + expanded_inputs[value.name] = declared_inputs[value.name] + return expanded_inputs + def __parse_elem( self, test_elem, i, default_interactor ): # Composite datasets need a unique name: each test occurs in a fresh # history, but we'll keep it unique per set of tests - use i (test #) diff -r 1411df5a5fcc2686512b5560435279f305fc755a -r 2dcc0a0e3732cf32b603279fdc443c6a2a9d9ae8 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -258,14 +258,14 @@ job_finish_by_output_count = False # Do the first page - page_inputs = self.__expand_grouping(testdef.tool.inputs_by_page[0], all_inputs) + page_inputs = testdef.expand_grouping(testdef.tool.inputs_by_page[0], all_inputs) # Run the tool self.twill_test_case.run_tool( testdef.tool.id, repeat_name=repeat_name, **page_inputs ) print "page_inputs (0)", page_inputs # Do other pages if they exist for i in range( 1, testdef.tool.npages ): - page_inputs = self.__expand_grouping(testdef.tool.inputs_by_page[i], all_inputs) + page_inputs = testdef.expand_grouping(testdef.tool.inputs_by_page[i], all_inputs) self.twill_test_case.submit_form( **page_inputs ) print "page_inputs (%i)" % i, page_inputs @@ -300,43 +300,6 @@ def output_hid( self, output_data ): return output_data.get( 'hid' ) - def __expand_grouping( self, tool_inputs, declared_inputs, prefix='' ): - expanded_inputs = {} - for key, value in tool_inputs.items(): - if isinstance( value, grouping.Conditional ): - if prefix: - new_prefix = "%s|%s" % ( prefix, value.name ) - else: - new_prefix = value.name - for i, case in enumerate( value.cases ): - if declared_inputs[ value.test_param.name ] == case.value: - if isinstance(case.value, str): - expanded_inputs[ "%s|%s" % ( new_prefix, value.test_param.name ) ] = case.value.split( "," ) - else: - expanded_inputs[ "%s|%s" % ( new_prefix, value.test_param.name ) ] = case.value - for input_name, input_value in case.inputs.items(): - expanded_inputs.update( self.__expand_grouping( { input_name: input_value }, declared_inputs, prefix=new_prefix ) ) - elif isinstance( value, grouping.Repeat ): - for repeat_index in xrange( 0, 1 ): # need to allow for and figure out how many repeats we have - for r_name, r_value in value.inputs.iteritems(): - new_prefix = "%s_%d" % ( value.name, repeat_index ) - if prefix: - new_prefix = "%s|%s" % ( prefix, new_prefix ) - expanded_inputs.update( self.__expand_grouping( { new_prefix : r_value }, declared_inputs, prefix=new_prefix ) ) - elif value.name not in declared_inputs: - print "%s not declared in tool test, will not change default value." % value.name - elif isinstance(declared_inputs[value.name], str): - if prefix: - expanded_inputs["%s|%s" % ( prefix, value.name ) ] = declared_inputs[value.name].split(",") - else: - expanded_inputs[value.name] = declared_inputs[value.name].split(",") - else: - if prefix: - expanded_inputs["%s|%s" % ( prefix, value.name ) ] = declared_inputs[value.name] - else: - expanded_inputs[value.name] = declared_inputs[value.name] - return expanded_inputs - def build_tests( testing_shed_tools=False, master_api_key=None, user_api_key=None ): """ https://bitbucket.org/galaxy/galaxy-central/commits/0435cbff42d1/ Changeset: 0435cbff42d1 User: jmchilton Date: 2013-11-22 02:17:12 Summary: Simplify logic in tool test expand_grouping. Affected #: 1 file diff -r 2dcc0a0e3732cf32b603279fdc443c6a2a9d9ae8 -r 0435cbff42d144e90cf750851f3998b3217a4891 lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -76,17 +76,13 @@ def expand_grouping( self, tool_inputs, declared_inputs, prefix='' ): expanded_inputs = {} for key, value in tool_inputs.items(): + expanded_key = value.name if not prefix else "%s|%s" % (prefix, value.name) if isinstance( value, grouping.Conditional ): - if prefix: - new_prefix = "%s|%s" % ( prefix, value.name ) - else: - new_prefix = value.name + new_prefix = expanded_key for i, case in enumerate( value.cases ): if declared_inputs[ value.test_param.name ] == case.value: - if isinstance(case.value, str): - expanded_inputs[ "%s|%s" % ( new_prefix, value.test_param.name ) ] = case.value.split( "," ) - else: - expanded_inputs[ "%s|%s" % ( new_prefix, value.test_param.name ) ] = case.value + expanded_value = self.__split_if_str(case.value) + expanded_inputs[ "%s|%s" % ( new_prefix, value.test_param.name ) ] = expanded_value for input_name, input_value in case.inputs.items(): expanded_inputs.update( self.expand_grouping( { input_name: input_value }, declared_inputs, prefix=new_prefix ) ) elif isinstance( value, grouping.Repeat ): @@ -98,18 +94,17 @@ expanded_inputs.update( self.expand_grouping( { new_prefix : r_value }, declared_inputs, prefix=new_prefix ) ) elif value.name not in declared_inputs: print "%s not declared in tool test, will not change default value." % value.name - elif isinstance(declared_inputs[value.name], str): - if prefix: - expanded_inputs["%s|%s" % ( prefix, value.name ) ] = declared_inputs[value.name].split(",") - else: - expanded_inputs[value.name] = declared_inputs[value.name].split(",") else: - if prefix: - expanded_inputs["%s|%s" % ( prefix, value.name ) ] = declared_inputs[value.name] - else: - expanded_inputs[value.name] = declared_inputs[value.name] + value = self.__split_if_str(declared_inputs[value.name]) + expanded_inputs[expanded_key] = value return expanded_inputs + def __split_if_str( self, value ): + split = isinstance(value, str) + if split: + value = value.split(",") + return value + def __parse_elem( self, test_elem, i, default_interactor ): # Composite datasets need a unique name: each test occurs in a fresh # history, but we'll keep it unique per set of tests - use i (test #) https://bitbucket.org/galaxy/galaxy-central/commits/8b99c7bc5e11/ Changeset: 8b99c7bc5e11 User: jmchilton Date: 2013-11-22 02:17:12 Summary: Tool functional tests - all unspecified default conditional values. Fix should apply to both twill and API variants. For now just grabbing first option in select list, a subsequent changeset adds much richer options including handling of booleans and options labeled as default. Affected #: 1 file diff -r 0435cbff42d144e90cf750851f3998b3217a4891 -r 8b99c7bc5e11cd7c452c2ad86f951109b495271c lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -73,18 +73,29 @@ yield data_dict + def __matching_case( self, cond, declared_inputs ): + param = cond.test_param + declared_value = declared_inputs.get( param.name, None ) + for i, case in enumerate( cond.cases ): + if declared_value and (case.value == declared_value): + return case + if not declared_value: + # TODO: Default might not be top value, fix this. + return case + print "Not matching case found for %s value %s. Test may fail in unexpected ways." % ( param.name, declared_value ) + def expand_grouping( self, tool_inputs, declared_inputs, prefix='' ): expanded_inputs = {} for key, value in tool_inputs.items(): expanded_key = value.name if not prefix else "%s|%s" % (prefix, value.name) if isinstance( value, grouping.Conditional ): new_prefix = expanded_key - for i, case in enumerate( value.cases ): - if declared_inputs[ value.test_param.name ] == case.value: - expanded_value = self.__split_if_str(case.value) - expanded_inputs[ "%s|%s" % ( new_prefix, value.test_param.name ) ] = expanded_value - for input_name, input_value in case.inputs.items(): - expanded_inputs.update( self.expand_grouping( { input_name: input_value }, declared_inputs, prefix=new_prefix ) ) + case = self.__matching_case( value, declared_inputs ) + if case: + expanded_value = self.__split_if_str(case.value) + expanded_inputs[ "%s|%s" % ( new_prefix, value.test_param.name ) ] = expanded_value + for input_name, input_value in case.inputs.items(): + expanded_inputs.update( self.expand_grouping( { input_name: input_value }, declared_inputs, prefix=new_prefix ) ) elif isinstance( value, grouping.Repeat ): for repeat_index in xrange( 0, 1 ): # need to allow for and figure out how many repeats we have for r_name, r_value in value.inputs.iteritems(): https://bitbucket.org/galaxy/galaxy-central/commits/3360087fcac9/ Changeset: 3360087fcac9 User: jmchilton Date: 2013-11-22 02:17:12 Summary: Twill-less tool tests - handle repeats and conditionals. Expand 'inputs' tree when building tool inputs for API functional tests. Code is in there for both flat inputs (the way the UI would submit them), and a nested tree structure. The nested tree structure will not work with the Galaxy API currently - but it is the way the API should work :). Affected #: 2 files diff -r 8b99c7bc5e11cd7c452c2ad86f951109b495271c -r 3360087fcac9cfecb7848c0b60a26b91c66d1c8b lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -73,6 +73,24 @@ yield data_dict + def to_dict( self, tool_inputs, declared_inputs ): + expanded_inputs = {} + for key, value in tool_inputs.items(): + if isinstance( value, grouping.Conditional ): + for i, case in enumerate( value.cases ): + if declared_inputs[ value.test_param.name ] == case.value: + pass # TODO + elif isinstance( value, grouping.Repeat ): + values = [] + for r_name, r_value in value.inputs.iteritems(): + values.append( self.to_dict( {r_name: r_value} , declared_inputs ) ) + expanded_inputs[ value.name ] = values + elif value.name not in declared_inputs: + print "%s not declared in tool test, will not change default value." % value.name + else: + expanded_inputs[ value.name ] = declared_inputs[value.name] + return expanded_inputs + def __matching_case( self, cond, declared_inputs ): param = cond.test_param declared_value = declared_inputs.get( param.name, None ) diff -r 8b99c7bc5e11cd7c452c2ad86f951109b495271c -r 3360087fcac9cfecb7848c0b60a26b91c66d1c8b test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -126,10 +126,30 @@ if value in self.uploads: all_inputs[key] = self.uploads[ value ] - # TODO: Handle repeats. - # TODO: Handle pages. - # TODO: Handle force_history_refresh - datasets = self.__submit_tool( history_id, tool_id=testdef.tool.id, tool_input=all_inputs ) + # TODO: Handle pages? + # TODO: Handle force_history_refresh? + flat_inputs = True + if flat_inputs: + # Build up tool_input flately (e.g {"a_repeat_0|a_repeat_param" : "value1"}) + expanded_inputs = {} + expanded_inputs.update(testdef.expand_grouping(testdef.tool.inputs_by_page[0], all_inputs)) + for i in range( 1, testdef.tool.npages ): + expanded_inputs.update(testdef.expand_grouping(testdef.tool.inputs_by_page[i], all_inputs)) + + # # HACK: Flatten single-value lists. Required when using expand_grouping + for key, value in expanded_inputs.iteritems(): + if isinstance(value, list) and len(value) == 1: + expanded_inputs[key] = value[0] + tool_input = expanded_inputs + else: + # Build up tool_input as nested dictionary (e.g. {"a_repeat": [{"a_repeat_param" : "value1"}]}) + # Doesn't work with the tool API at this time. + tool_input = {} + tool_input.update(testdef.to_dict(testdef.tool.inputs_by_page[0], all_inputs)) + for i in range( 1, testdef.tool.npages ): + tool_input.update(testdef.to_dict(testdef.tool.inputs_by_page[i], all_inputs)) + + datasets = self.__submit_tool( history_id, tool_id=testdef.tool.id, tool_input=tool_input ) self.__wait_for_history( history_id )() # TODO: Remove and respect maxseconds! return datasets.json()[ 'outputs' ] https://bitbucket.org/galaxy/galaxy-central/commits/47eeac9693d3/ Changeset: 47eeac9693d3 User: jmchilton Date: 2013-11-22 02:17:12 Summary: Twill-less tool tests fix for nested input filenames. Affected #: 1 file diff -r 3360087fcac9cfecb7848c0b60a26b91c66d1c8b -r 47eeac9693d38593a60bf43f1920a6417fca2e6a test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -111,7 +111,7 @@ dataset = submit_response["outputs"][0] #raise Exception(str(dataset)) hid = dataset['id'] - self.uploads[ fname ] = {"src": "hda", "id": hid} + self.uploads[ os.path.basename(fname) ] = self.uploads[ fname ] = {"src": "hda", "id": hid} return self.__wait_for_history( history_id ) def run_tool( self, testdef, history_id ): https://bitbucket.org/galaxy/galaxy-central/commits/b21e913e21e9/ Changeset: b21e913e21e9 User: jmchilton Date: 2013-11-22 02:17:12 Summary: Twill-less tool tests fix for tests with html outputs. For example gatk_analyze_covariates. Affected #: 1 file diff -r 47eeac9693d38593a60bf43f1920a6417fca2e6a -r b21e913e21e951a9d54de4b8e480334dabb70e2d test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -198,7 +198,7 @@ def __dataset_fetcher( self, history_id ): def fetcher( hda_id, base_name=None ): - url = "histories/%s/contents/%s/display" % (history_id, hda_id) + url = "histories/%s/contents/%s/display?raw=true" % (history_id, hda_id) if base_name: url += "&filename=%s" % base_name return self.__get( url ).text https://bitbucket.org/galaxy/galaxy-central/commits/ff31838afb70/ Changeset: ff31838afb70 User: jmchilton Date: 2013-11-22 02:17:12 Summary: Twill-less tool tests fix for binary data in results. Use requests' response.content instead of response.text for reading results. Fixes the following error 'UnicodeEncodeError: 'ascii' codec can't encode characters in position 1-3: ordinal not in range(128)' when dealing with ASN.1 binary files. Affected #: 1 file diff -r b21e913e21e951a9d54de4b8e480334dabb70e2d -r ff31838afb704f86e10ce4798f8d41e6dfcecb59 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -201,7 +201,7 @@ url = "histories/%s/contents/%s/display?raw=true" % (history_id, hda_id) if base_name: url += "&filename=%s" % base_name - return self.__get( url ).text + return self.__get( url ).content return fetcher https://bitbucket.org/galaxy/galaxy-central/commits/deaa67411c92/ Changeset: deaa67411c92 User: jmchilton Date: 2013-11-22 02:17:12 Summary: Twill-less tool tests - enhancements related to errors. - Report formatted standard error and standard output of tools like Twill interactor. - Respect testdef.maxseconds to timeout test cases. - Check history item - like Twill interactor do not verify 'error' datasets. - Properly raise exceptions with reported API error messages when running tools. Affected #: 3 files diff -r ff31838afb704f86e10ce4798f8d41e6dfcecb59 -r deaa67411c9212f8c24f3062e8c7710466118f34 lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -10,6 +10,7 @@ DEFAULT_FTYPE = 'auto' DEFAULT_DBKEY = 'hg17' DEFAULT_INTERACTOR = "twill" # Default mechanism test code uses for interacting with Galaxy instance. +DEFAULT_MAX_SECS = 120 def parse_tests_elem(tool, tests_elem): @@ -35,7 +36,7 @@ def __init__( self, tool, test_elem, i, default_interactor ): name = test_elem.get( 'name', 'Test-%d' % (i + 1) ) - maxseconds = int( test_elem.get( 'maxseconds', '120' ) ) + maxseconds = int( test_elem.get( 'maxseconds', DEFAULT_MAX_SECS ) ) self.tool = tool self.name = name diff -r ff31838afb704f86e10ce4798f8d41e6dfcecb59 -r deaa67411c9212f8c24f3062e8c7710466118f34 test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -616,6 +616,9 @@ self.visit_page( "datasets/%s/%s" % ( self.security.encode_id( hda_id ), stream ) ) output = self.last_page() + return self._format_stream( output, stream, format ) + + def _format_stream( self, output, stream, format ): if format: msg = "---------------------- >> begin tool %s << -----------------------\n" % stream msg += output + "\n" @@ -1409,13 +1412,16 @@ return True return False - def wait( self, maxseconds=120 ): + def wait( self, **kwds ): """Waits for the tools to finish""" + return self.wait_for(lambda: self.get_running_datasets(), **kwds) + + def wait_for(self, func, maxseconds=120): sleep_amount = 0.1 slept = 0 - self.home() while slept <= maxseconds: - if self.get_running_datasets(): + result = func() + if result: time.sleep( sleep_amount ) slept += sleep_amount sleep_amount *= 2 diff -r ff31838afb704f86e10ce4798f8d41e6dfcecb59 -r deaa67411c9212f8c24f3062e8c7710466118f34 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -64,7 +64,14 @@ output_data = data_list[ output_index ] self.assertTrue( output_data is not None ) name, outfile, attributes = output_tuple - galaxy_interactor.verify_output( history, output_data, outfile, attributes=attributes, shed_tool_id=shed_tool_id, maxseconds=maxseconds ) + try: + galaxy_interactor.verify_output( history, output_data, outfile, attributes=attributes, shed_tool_id=shed_tool_id, maxseconds=maxseconds ) + except Exception: + for stream in ['stdout', 'stderr']: + stream_output = galaxy_interactor.get_job_stream( history, output_data, stream=stream ) + print >>sys.stderr, self._format_stream( stream_output, stream=stream, format=True ) + raise + output_index += 1 @@ -77,16 +84,16 @@ self.uploads = {} def verify_output( self, history_id, output_data, outfile, attributes, shed_tool_id, maxseconds ): + self.twill_test_case.wait_for( lambda: not self.__history_ready( history_id ), maxseconds=maxseconds) hid = output_data.get( 'id' ) - try: - fetcher = self.__dataset_fetcher( history_id ) - self.twill_test_case.verify_hid( outfile, hda_id=hid, attributes=attributes, dataset_fetcher=fetcher, shed_tool_id=shed_tool_id ) - except Exception: - ## TODO: Print this! - # print >>sys.stderr, self.twill_test_case.get_job_stdout( output_data.get( 'id' ), format=True ) - ## TODO: Print this! - # print >>sys.stderr, self.twill_test_case.get_job_stderr( output_data.get( 'id' ), format=True ) - raise + fetcher = self.__dataset_fetcher( history_id ) + ## TODO: Twill version verifys dataset is 'ok' in here. + self.twill_test_case.verify_hid( outfile, hda_id=hid, attributes=attributes, dataset_fetcher=fetcher, shed_tool_id=shed_tool_id ) + + def get_job_stream( self, history_id, output_data, stream ): + hid = output_data.get( 'id' ) + data = self.__get( "histories/%s/contents/%s/provenance" % (history_id, hid) ).json() + return data.get( stream, '' ) def new_history( self ): history_json = self.__post( "histories", {"name": "test_history"} ).json() @@ -150,8 +157,11 @@ tool_input.update(testdef.to_dict(testdef.tool.inputs_by_page[i], all_inputs)) datasets = self.__submit_tool( history_id, tool_id=testdef.tool.id, tool_input=tool_input ) - self.__wait_for_history( history_id )() # TODO: Remove and respect maxseconds! - return datasets.json()[ 'outputs' ] + datasets_object = datasets.json() + try: + return datasets_object[ 'outputs' ] + except KeyError: + raise Exception( datasets_object[ 'message' ] ) def output_hid( self, output_data ): return output_data[ 'id' ] @@ -161,17 +171,19 @@ def __wait_for_history( self, history_id ): def wait(): - while True: - history_json = self.__get( "histories/%s" % history_id ).json() - state = history_json[ 'state' ] - if state == 'ok': - #raise Exception(str(self.__get( self.__get( "histories/%s/contents" % history_id ).json()[0]['url'] ).json() ) ) - #raise Exception(str(self.__get( self.__get( "histories/%s/contents" % history_id ).json()[0]['url'] ).json() ) ) - break - elif state == 'error': - raise Exception("History in error state.") + while not self.__history_ready( history_id ): + pass return wait + def __history_ready( self, history_id ): + history_json = self.__get( "histories/%s" % history_id ).json() + state = history_json[ 'state' ] + if state == 'ok': + return True + elif state == 'error': + raise Exception("History in error state.") + return False + def __submit_tool( self, history_id, tool_id, tool_input, extra_data={}, files=None ): data = dict( history_id=history_id, @@ -230,12 +242,10 @@ def verify_output( self, history, output_data, outfile, attributes, shed_tool_id, maxseconds ): hid = output_data.get( 'hid' ) - try: - self.twill_test_case.verify_dataset_correctness( outfile, hid=hid, attributes=attributes, shed_tool_id=shed_tool_id ) - except Exception: - print >>sys.stderr, self.twill_test_case.get_job_stdout( output_data.get( 'id' ), format=True ) - print >>sys.stderr, self.twill_test_case.get_job_stderr( output_data.get( 'id' ), format=True ) - raise + self.twill_test_case.verify_dataset_correctness( outfile, hid=hid, attributes=attributes, shed_tool_id=shed_tool_id, maxseconds=maxseconds ) + + def get_job_stream( self, history_id, output_data, stream ): + return self.twill_test_case._get_job_stream_output( output_data.get( 'id' ), stream=stream, format=False ) def stage_data_async( self, test_data, history, shed_tool_id, async=True ): name = test_data.get( 'name', None ) https://bitbucket.org/galaxy/galaxy-central/commits/f5add174debe/ Changeset: f5add174debe User: jmchilton Date: 2013-11-22 02:17:12 Summary: Add new option, tools, and data to functional test framework to test/demonstrate tool framework itself. Add smaple repeat tool demonstrating some weaknesses of twill-based current framework. Affected #: 12 files diff -r deaa67411c9212f8c24f3062e8c7710466118f34 -r f5add174debe1ca23aa21a42530c650ad6b1ad42 run_functional_tests.sh --- a/run_functional_tests.sh +++ b/run_functional_tests.sh @@ -13,6 +13,8 @@ echo "'run_functional_tests.sh -list' for listing all the tool ids" echo "'run_functional_tests.sh -toolshed' for running all the test scripts in the ./test/tool_shed/functional directory" echo "'run_functional_tests.sh -toolshed testscriptname' for running one test script named testscriptname in the .test/tool_shed/functional directory" + echo "'run_functional_tests.sh -framework' for running through example tool tests testing framework features in test/functional/tools" + echo "'run_functional_tests.sh -framework -id toolid' for testing one framework tool (in test/functional/tools/) with id 'toolid'" elif [ $1 = '-id' ]; then python ./scripts/functional_tests.py -v functional.test_toolbox:TestForTool_$2 --with-nosehtml --html-report-file run_functional_tests.html elif [ $1 = '-sid' ]; then @@ -46,6 +48,14 @@ else python ./test/tool_shed/functional_tests.py -v --with-nosehtml --html-report-file ./test/tool_shed/run_functional_tests.html $2 fi +elif [ $1 = '-framework' ]; then + if [ ! $2 ]; then + python ./scripts/functional_tests.py -v functional.test_toolbox --with-nosehtml --html-report-file run_functional_tests.html -framework + elif [ $2 = '-id' ]; then + python ./scripts/functional_tests.py -v functional.test_toolbox:TestForTool_$3 --with-nosehtml --html-report-file run_functional_tests.html -framework + else + echo "Invalid test option selected, if -framework first argument to $0, optional second argument must be -id followed a tool id." + fi else python ./scripts/functional_tests.py -v --with-nosehtml --html-report-file run_functional_tests.html $1 fi diff -r deaa67411c9212f8c24f3062e8c7710466118f34 -r f5add174debe1ca23aa21a42530c650ad6b1ad42 scripts/functional_tests.py --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -178,6 +178,7 @@ os.environ[ 'HTTP_ACCEPT_LANGUAGE' ] = default_galaxy_locales testing_migrated_tools = '-migrated' in sys.argv testing_installed_tools = '-installed' in sys.argv + datatypes_conf_override = None if testing_migrated_tools or testing_installed_tools: sys.argv.pop() @@ -197,8 +198,21 @@ # Exclude all files except test_toolbox.py. ignore_files = ( re.compile( r'^test_[adghlmsu]*' ), re.compile( r'^test_ta*' ) ) else: - tool_config_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', 'tool_conf.xml' ) - galaxy_test_file_dir = os.environ.get( 'GALAXY_TEST_FILE_DIR', default_galaxy_test_file_dir ) + framework_test = '-framework' in sys.argv # Run through suite of tests testing framework. + if framework_test: + sys.argv.pop() + framework_tool_dir = os.path.join('test', 'functional', 'tools') + tool_conf = os.path.join( framework_tool_dir, 'samples_tool_conf.xml' ) + datatypes_conf_override = os.path.join( framework_tool_dir, 'sample_datatypes_conf.xml' ) + test_dir = os.path.join( framework_tool_dir, 'test-data') + + tool_path = framework_tool_dir + else: + # Use tool_conf.xml toolbox. + tool_conf = 'tool_conf.xml' + test_dir = default_galaxy_test_file_dir + tool_config_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', tool_conf ) + galaxy_test_file_dir = os.environ.get( 'GALAXY_TEST_FILE_DIR', test_dir ) if not os.path.isabs( galaxy_test_file_dir ): galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) library_import_dir = galaxy_test_file_dir @@ -338,6 +352,8 @@ if use_distributed_object_store: kwargs[ 'object_store' ] = 'distributed' kwargs[ 'distributed_object_store_config_file' ] = 'distributed_object_store_conf.xml.sample' + if datatypes_conf_override: + kwargs[ 'datatypes_config_file' ] = datatypes_conf_override # If the user has passed in a path for the .ini file, do not overwrite it. galaxy_config_file = os.environ.get( 'GALAXY_TEST_INI_FILE', None ) if not galaxy_config_file: diff -r deaa67411c9212f8c24f3062e8c7710466118f34 -r f5add174debe1ca23aa21a42530c650ad6b1ad42 test/functional/tools/README.txt --- /dev/null +++ b/test/functional/tools/README.txt @@ -0,0 +1,3 @@ +This directory contains tools only useful for testing the tool test framework +and demonstrating it features. Run the driver script 'run_functional_tests.sh' +with '-installed' as first argument to run through these tests. diff -r deaa67411c9212f8c24f3062e8c7710466118f34 -r f5add174debe1ca23aa21a42530c650ad6b1ad42 test/functional/tools/multi_repeats.xml --- /dev/null +++ b/test/functional/tools/multi_repeats.xml @@ -0,0 +1,39 @@ +<tool id="multirepeat" name="multirepeat"> + <description>tail-to-head</description> + <command> + cat $input1 #for $q in $queries# ${q.input2} #end for# #for $q in $more_queries# ${q.more_queries_input} #end for# > $out_file1 + </command> + <inputs> + <param name="input1" type="data" label="Concatenate Dataset"/> + <repeat name="queries" title="Dataset"> + <param name="input2" type="data" label="Select" /> + </repeat> + <repeat name="more_queries" title="Dataset"> + <param name="more_queries_input" type="data" label="Select" /> + </repeat> + </inputs> + <outputs> + <data name="out_file1" format="input" metadata_source="input1"/> + </outputs> + <tests> + <test> + <param name="input1" value="simple_line.txt"/> + <param name="input2" value="simple_line.txt"/> + <output name="out_file1" file="simple_line_x2.txt"/> + </test> + <test> + <param name="input1" value="simple_line.txt"/> + <param name="input2" value="simple_line.txt"/> + <param name="input2" value="simple_line.txt"/> + <output name="out_file1" file="simple_line_x3.txt"/> + </test> + <test> + <param name="input1" value="simple_line.txt"/> + <param name="input2" value="simple_line.txt"/> + <param name="input2" value="simple_line.txt"/> + <param name="more_queries_input" value="simple_line.txt"/> + <param name="more_queries_input" value="simple_line.txt"/> + <output name="out_file1" file="simple_line_x5.txt"/> + </test> + </tests> +</tool> diff -r deaa67411c9212f8c24f3062e8c7710466118f34 -r f5add174debe1ca23aa21a42530c650ad6b1ad42 test/functional/tools/sample_datatypes_conf.xml --- /dev/null +++ b/test/functional/tools/sample_datatypes_conf.xml @@ -0,0 +1,8 @@ +<?xml version="1.0"?> +<datatypes> + <registration converters_path="lib/galaxy/datatypes/converters" display_path="display_applications"> + <datatype extension="velvet" type="galaxy.datatypes.assembly:Velvet" display_in_upload="true"/> + <datatype extension="txt" type="galaxy.datatypes.data:Text" display_in_upload="true"/> + <datatype extension="tabular" type="galaxy.datatypes.tabular:Tabular" display_in_upload="true"/> + </registration> +</datatypes> \ No newline at end of file diff -r deaa67411c9212f8c24f3062e8c7710466118f34 -r f5add174debe1ca23aa21a42530c650ad6b1ad42 test/functional/tools/samples_tool_conf.xml --- /dev/null +++ b/test/functional/tools/samples_tool_conf.xml @@ -0,0 +1,5 @@ +<?xml version="1.0"?> +<toolbox> + <tool file="upload.xml"/> + <tool file="multi_repeats.xml"/> +</toolbox> \ No newline at end of file diff -r deaa67411c9212f8c24f3062e8c7710466118f34 -r f5add174debe1ca23aa21a42530c650ad6b1ad42 test/functional/tools/test-data/simple_line.txt --- /dev/null +++ b/test/functional/tools/test-data/simple_line.txt @@ -0,0 +1,1 @@ +This is a line of text. diff -r deaa67411c9212f8c24f3062e8c7710466118f34 -r f5add174debe1ca23aa21a42530c650ad6b1ad42 test/functional/tools/test-data/simple_line_x2.txt --- /dev/null +++ b/test/functional/tools/test-data/simple_line_x2.txt @@ -0,0 +1,2 @@ +This is a line of text. +This is a line of text. diff -r deaa67411c9212f8c24f3062e8c7710466118f34 -r f5add174debe1ca23aa21a42530c650ad6b1ad42 test/functional/tools/test-data/simple_line_x3.txt --- /dev/null +++ b/test/functional/tools/test-data/simple_line_x3.txt @@ -0,0 +1,3 @@ +This is a line of text. +This is a line of text. +This is a line of text. diff -r deaa67411c9212f8c24f3062e8c7710466118f34 -r f5add174debe1ca23aa21a42530c650ad6b1ad42 test/functional/tools/test-data/simple_line_x5.txt --- /dev/null +++ b/test/functional/tools/test-data/simple_line_x5.txt @@ -0,0 +1,5 @@ +This is a line of text. +This is a line of text. +This is a line of text. +This is a line of text. +This is a line of text. diff -r deaa67411c9212f8c24f3062e8c7710466118f34 -r f5add174debe1ca23aa21a42530c650ad6b1ad42 test/functional/tools/upload.py --- /dev/null +++ b/test/functional/tools/upload.py @@ -0,0 +1,1 @@ +../../../tools/data_source/upload.py \ No newline at end of file diff -r deaa67411c9212f8c24f3062e8c7710466118f34 -r f5add174debe1ca23aa21a42530c650ad6b1ad42 test/functional/tools/upload.xml --- /dev/null +++ b/test/functional/tools/upload.xml @@ -0,0 +1,1 @@ +../../../tools/data_source/upload.xml \ No newline at end of file https://bitbucket.org/galaxy/galaxy-central/commits/81b6aca75182/ Changeset: 81b6aca75182 User: jmchilton Date: 2013-11-22 02:17:12 Summary: Twill-less tool tests - allow multiple param instances with same name. Allows specifing multiple repeat instances. Affected #: 2 files diff -r f5add174debe1ca23aa21a42530c650ad6b1ad42 -r 81b6aca751826e85bbf58fadb21f96d370dd9f6e lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -103,6 +103,41 @@ return case print "Not matching case found for %s value %s. Test may fail in unexpected ways." % ( param.name, declared_value ) + def expand_multi_grouping( self, tool_inputs, declared_inputs, prefix='', index=0 ): + """ + Used by API, slight generalization of expand_grouping used by Twill based interactor. Still + not quite the context/tree based specification that should exist! + """ + expanded_inputs = {} + for key, value in tool_inputs.items(): + expanded_key = value.name if not prefix else "%s|%s" % (prefix, value.name) + if isinstance( value, grouping.Conditional ): + new_prefix = expanded_key + case = self.__matching_case( value, declared_inputs ) + if case: + expanded_value = self.__split_if_str(case.value) + expanded_inputs[ "%s|%s" % ( new_prefix, value.test_param.name ) ] = expanded_value + for input_name, input_value in case.inputs.items(): + expanded_inputs.update( self.expand_multi_grouping( { input_name: input_value }, declared_inputs, prefix=new_prefix ), index=index ) + elif isinstance( value, grouping.Repeat ): + repeat_index = 0 + any_children_matched = True + while any_children_matched: + any_children_matched = False + for r_name, r_value in value.inputs.iteritems(): + new_prefix = "%s_%d" % ( value.name, repeat_index ) + if prefix: + new_prefix = "%s|%s" % ( prefix, new_prefix ) + expanded_input = self.expand_multi_grouping( { new_prefix : r_value }, declared_inputs, prefix=new_prefix, index=repeat_index ) + if expanded_input: + any_children_matched = True + expanded_inputs.update( expanded_input ) + repeat_index += 1 + elif value.name in declared_inputs and len(declared_inputs[ value.name ]) > index: + value = self.__split_if_str( declared_inputs[ value.name ][ index ] ) + expanded_inputs[ expanded_key ] = value + return expanded_inputs + def expand_grouping( self, tool_inputs, declared_inputs, prefix='' ): expanded_inputs = {} for key, value in tool_inputs.items(): diff -r f5add174debe1ca23aa21a42530c650ad6b1ad42 -r 81b6aca751826e85bbf58fadb21f96d370dd9f6e test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -126,12 +126,14 @@ # tool will have uncompressed it on the fly. all_inputs = {} for name, value, _ in testdef.inputs: - all_inputs[ name ] = value - - for key, value in all_inputs.iteritems(): # TODO: Restrict this to param inputs. if value in self.uploads: - all_inputs[key] = self.uploads[ value ] + value = self.uploads[ value ] + + if name in all_inputs: + all_inputs[name].append( value ) + else: + all_inputs[name] = [ value ] # TODO: Handle pages? # TODO: Handle force_history_refresh? @@ -139,9 +141,9 @@ if flat_inputs: # Build up tool_input flately (e.g {"a_repeat_0|a_repeat_param" : "value1"}) expanded_inputs = {} - expanded_inputs.update(testdef.expand_grouping(testdef.tool.inputs_by_page[0], all_inputs)) + expanded_inputs.update(testdef.expand_multi_grouping(testdef.tool.inputs_by_page[0], all_inputs)) for i in range( 1, testdef.tool.npages ): - expanded_inputs.update(testdef.expand_grouping(testdef.tool.inputs_by_page[i], all_inputs)) + expanded_inputs.update(testdef.expand_multi_grouping(testdef.tool.inputs_by_page[i], all_inputs)) # # HACK: Flatten single-value lists. Required when using expand_grouping for key, value in expanded_inputs.iteritems(): https://bitbucket.org/galaxy/galaxy-central/commits/72c22a117bdd/ Changeset: 72c22a117bdd User: jmchilton Date: 2013-11-22 02:17:12 Summary: Tool functional tests - allow disambiguation of params in conditionals. Disambiguate by add prefix for parent (or any number of direct ancestors) with pipe (|). See included test case for an example. Should fix twill and API driven functional tests. Affected #: 3 files diff -r 81b6aca751826e85bbf58fadb21f96d370dd9f6e -r 72c22a117bdd7fe83b05dd616114bbb7b6a2dae8 lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -3,6 +3,7 @@ from parameters import basic from parameters import grouping from galaxy.util import string_as_bool +from galaxy.util.bunch import Bunch import logging log = logging.getLogger( __name__ ) @@ -92,15 +93,20 @@ expanded_inputs[ value.name ] = declared_inputs[value.name] return expanded_inputs - def __matching_case( self, cond, declared_inputs ): + def __matching_case( self, cond, declared_inputs, prefix, index=None ): param = cond.test_param - declared_value = declared_inputs.get( param.name, None ) + declared_value = self.__declared_match( declared_inputs, param.name, prefix) + if index is not None: + declared_value = declared_value[index] for i, case in enumerate( cond.cases ): - if declared_value and (case.value == declared_value): + if declared_value is not None and (case.value == declared_value): return case - if not declared_value: + if declared_value is None: # TODO: Default might not be top value, fix this. + # TODO: Also may be boolean, got to look at checked. return case + else: + return Bunch(value=declared_value, inputs=Bunch(items=lambda: [])) print "Not matching case found for %s value %s. Test may fail in unexpected ways." % ( param.name, declared_value ) def expand_multi_grouping( self, tool_inputs, declared_inputs, prefix='', index=0 ): @@ -113,9 +119,9 @@ expanded_key = value.name if not prefix else "%s|%s" % (prefix, value.name) if isinstance( value, grouping.Conditional ): new_prefix = expanded_key - case = self.__matching_case( value, declared_inputs ) + case = self.__matching_case( value, declared_inputs, new_prefix, index=index ) if case: - expanded_value = self.__split_if_str(case.value) + expanded_value = self.__split_if_str( case.value ) expanded_inputs[ "%s|%s" % ( new_prefix, value.test_param.name ) ] = expanded_value for input_name, input_value in case.inputs.items(): expanded_inputs.update( self.expand_multi_grouping( { input_name: input_value }, declared_inputs, prefix=new_prefix ), index=index ) @@ -133,9 +139,11 @@ any_children_matched = True expanded_inputs.update( expanded_input ) repeat_index += 1 - elif value.name in declared_inputs and len(declared_inputs[ value.name ]) > index: - value = self.__split_if_str( declared_inputs[ value.name ][ index ] ) - expanded_inputs[ expanded_key ] = value + else: + declared_value = self.__declared_match( declared_inputs, value.name, prefix ) + if declared_value and len(declared_value) > index: + value = self.__split_if_str( declared_value[ index ] ) + expanded_inputs[ expanded_key ] = value return expanded_inputs def expand_grouping( self, tool_inputs, declared_inputs, prefix='' ): @@ -144,7 +152,7 @@ expanded_key = value.name if not prefix else "%s|%s" % (prefix, value.name) if isinstance( value, grouping.Conditional ): new_prefix = expanded_key - case = self.__matching_case( value, declared_inputs ) + case = self.__matching_case( value, declared_inputs, new_prefix ) if case: expanded_value = self.__split_if_str(case.value) expanded_inputs[ "%s|%s" % ( new_prefix, value.test_param.name ) ] = expanded_value @@ -157,13 +165,26 @@ if prefix: new_prefix = "%s|%s" % ( prefix, new_prefix ) expanded_inputs.update( self.expand_grouping( { new_prefix : r_value }, declared_inputs, prefix=new_prefix ) ) - elif value.name not in declared_inputs: - print "%s not declared in tool test, will not change default value." % value.name else: - value = self.__split_if_str(declared_inputs[value.name]) - expanded_inputs[expanded_key] = value + declared_value = self.__declared_match( declared_inputs, value.name, prefix ) + if not declared_value: + print "%s not declared in tool test, will not change default value." % value.name + else: + value = self.__split_if_str(declared_value) + expanded_inputs[expanded_key] = value return expanded_inputs + def __declared_match( self, declared_inputs, name, prefix ): + prefix_suffixes = [ "%s|" % part for part in prefix.split( "|" ) ] if prefix else [] + prefix_suffixes.append( name ) + prefix_suffixes.reverse() + prefixed_name = "" + for prefix_suffix in prefix_suffixes: + prefixed_name = "%s%s" % ( prefix_suffix, prefixed_name ) + if prefixed_name in declared_inputs: + return declared_inputs[prefixed_name] + return None + def __split_if_str( self, value ): split = isinstance(value, str) if split: diff -r 81b6aca751826e85bbf58fadb21f96d370dd9f6e -r 72c22a117bdd7fe83b05dd616114bbb7b6a2dae8 test/functional/tools/disambiguate_cond.xml --- /dev/null +++ b/test/functional/tools/disambiguate_cond.xml @@ -0,0 +1,50 @@ +<tool id="handle_cond" name="handle_cond"> + <description>tail-to-head</description> + <command> + echo "$p1.p1v $p2.p2v $p3.p3v" > $out_file1 + </command> + <inputs> + <conditional name="p1"> + <param type="boolean" name="use" /> + <when value="true"> + <param name="p1v" value="4" type="integer" /> + </when> + <when value="false"> + <param name="p1v" value="7" type="integer" /> + </when> + </conditional> + <conditional name="p2"> + <param type="boolean" name="use" /> + <when value="true"> + <param name="p2v" value="4" type="integer" /> + </when> + <when value="false"> + <param name="p2v" value="7" type="integer" /> + </when> + </conditional> + <conditional name="p3"> + <param type="boolean" name="use" /> + <when value="true"> + <param name="p3v" value="4" type="integer" /> + </when> + <when value="false"> + <param name="p3v" value="7" type="integer" /> + </when> + </conditional> + </inputs> + <outputs> + <data name="out_file1" format="txt" /> + </outputs> + <tests> + <test> + <param name="p1|use" value="True"/> + <param name="p2|use" value="False"/> + <param name="p3|use" value="True"/> + <output name="out_file1"> + <assert_contents> + <has_line line="4 7 4" /> + </assert_contents> + </output> + </test> + </tests> +</tool> diff -r 81b6aca751826e85bbf58fadb21f96d370dd9f6e -r 72c22a117bdd7fe83b05dd616114bbb7b6a2dae8 test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -1,5 +1,6 @@ <?xml version="1.0"?><toolbox><tool file="upload.xml"/> + <tool file="disambiguate_cond.xml" /><tool file="multi_repeats.xml"/></toolbox> \ No newline at end of file https://bitbucket.org/galaxy/galaxy-central/commits/61e4e10e03ce/ Changeset: 61e4e10e03ce User: jmchilton Date: 2013-11-22 02:17:12 Summary: Twill-less tool tests - handle composite inputs. Add example tool test for composite data (worked with Twill based test framework prior to this commit, works with API based framework as a result of this commit). Affected #: 4 files diff -r 72c22a117bdd7fe83b05dd616114bbb7b6a2dae8 -r 61e4e10e03ce8de315a1ae81d12529d80dedf594 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -101,24 +101,42 @@ def stage_data_async( self, test_data, history_id, shed_tool_id, async=True ): fname = test_data[ 'fname' ] - file_name = self.twill_test_case.get_filename( fname, shed_tool_id=shed_tool_id ) - name = test_data.get( 'name', None ) - if not name: - name = os.path.basename( file_name ) tool_input = { "file_type": test_data[ 'ftype' ], - "dbkey": test_data[ 'dbkey' ], # TODO: Handle it! Doesn't work if undefined, does seem to in Twill. - "files_0|NAME": name, - "files_0|type": "upload_dataset", + "dbkey": test_data[ 'dbkey' ], } - files = { - "files_0|file_data": open( file_name, 'rb') - } - submit_response = self.__submit_tool( history_id, "upload1", tool_input, extra_data={"type": "upload_dataset"}, files=files ).json() + composite_data = test_data[ 'composite_data' ] + if composite_data: + files = {} + for i, composite_file in enumerate( composite_data ): + file_name = self.twill_test_case.get_filename( composite_file.get( 'value' ), shed_tool_id=shed_tool_id ) + files["files_%s|file_data" % i] = open( file_name, 'rb' ) + tool_input.update({ + #"files_%d|NAME" % i: name, + "files_%d|type" % i: "upload_dataset", + ## TODO: + #"files_%d|space_to_tab" % i: composite_file.get( 'space_to_tab', False ) + }) + name = test_data[ 'name' ] + else: + file_name = self.twill_test_case.get_filename( fname, shed_tool_id=shed_tool_id ) + name = test_data.get( 'name', None ) + if not name: + name = os.path.basename( file_name ) + + tool_input.update({ + "files_0|NAME": name, + "files_0|type": "upload_dataset", + }) + files = { + "files_0|file_data": open( file_name, 'rb') + } + submit_response_object = self.__submit_tool( history_id, "upload1", tool_input, extra_data={"type": "upload_dataset"}, files=files ) + submit_response = submit_response_object.json() dataset = submit_response["outputs"][0] #raise Exception(str(dataset)) hid = dataset['id'] - self.uploads[ os.path.basename(fname) ] = self.uploads[ fname ] = {"src": "hda", "id": hid} + self.uploads[ os.path.basename(fname) ] = self.uploads[ fname ] = self.uploads[ name ] = {"src": "hda", "id": hid} return self.__wait_for_history( history_id ) def run_tool( self, testdef, history_id ): diff -r 72c22a117bdd7fe83b05dd616114bbb7b6a2dae8 -r 61e4e10e03ce8de315a1ae81d12529d80dedf594 test/functional/tools/composite.xml --- /dev/null +++ b/test/functional/tools/composite.xml @@ -0,0 +1,21 @@ +<tool id="velvet_inspired" name="velvet_inspired" version="1.0.0"> + <description>Velvet sequence assembler for very short reads</description> + <command>cat '$input.extra_files_path/Sequences' > $output</command> + <inputs> + <param name="input" type="data" format="velvet" label="Velvet Dataset" help="Prepared by velveth."/> + </inputs> + <outputs> + <data format="txt" name="output" label="${tool.name} on ${on_string}: LastGraph"> + </data> + </outputs> + <tests> + <test> + <param name="input" value="velveth_test1/output.html" ftype="velvet" > + <composite_data value='velveth_test1/Sequences' ftype="Sequences"/> + <composite_data value='velveth_test1/Roadmaps' ftype="Roadmaps"/> + <composite_data value='velveth_test1/Log'/> + </param> + <output name="unused_reads_fasta" file="velveth_test1/Sequences" compare="diff"/> + </test> + </tests> +</tool> diff -r 72c22a117bdd7fe83b05dd616114bbb7b6a2dae8 -r 61e4e10e03ce8de315a1ae81d12529d80dedf594 test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -1,6 +1,7 @@ <?xml version="1.0"?><toolbox><tool file="upload.xml"/> + <tool file="composite.xml" /><tool file="disambiguate_cond.xml" /><tool file="multi_repeats.xml"/></toolbox> \ No newline at end of file diff -r 72c22a117bdd7fe83b05dd616114bbb7b6a2dae8 -r 61e4e10e03ce8de315a1ae81d12529d80dedf594 test/functional/tools/test-data/velveth_test1 --- /dev/null +++ b/test/functional/tools/test-data/velveth_test1 @@ -0,0 +1,1 @@ +../../../../test-data/velveth_test1 \ No newline at end of file https://bitbucket.org/galaxy/galaxy-central/commits/df80192bfdcc/ Changeset: df80192bfdcc User: jmchilton Date: 2013-11-22 02:17:12 Summary: Add more sample tools for tool testing framework. Adding multi_select.xml, which demonstrates twill cannot deal with - at the beginning of select param values - this test works immediately with API interactor. Also adding a multi_output.xml, this is a tool using variable number of outputs (force_refresh=True), both interactors pass this but it is a good test to verify that. Also adding example to test multi-page tools (multi_page.xml) - both interactors pass this test. Though the API needed to be adjusted to allow its use (in a previous changeset). Also adding example tool demonstrating extra_files output. Finally, a tool simple_constructs.xml that just tests the basics, various parameter types, simple conditional, and simple repeat. Affected #: 6 files diff -r 61e4e10e03ce8de315a1ae81d12529d80dedf594 -r df80192bfdcc3b8ed37b104c757e0dcc21535420 test/functional/tools/composite_output.xml --- /dev/null +++ b/test/functional/tools/composite_output.xml @@ -0,0 +1,24 @@ +<tool id="composite_output" name="composite_output" version="1.0.0"> + <command>mkdir $output.extra_files_path; cp $input.extra_files_path/* $output.extra_files_path</command> + <inputs> + <param name="input" type="data" format="velvet" label="Velvet Dataset" help="Prepared by velveth."/> + </inputs> + <outputs> + <data format="velvet" name="output" label=""> + </data> + </outputs> + <tests> + <test> + <param name="input" value="velveth_test1/output.html" ftype="velvet" > + <composite_data value='velveth_test1/Sequences' ftype="Sequences"/> + <composite_data value='velveth_test1/Roadmaps' ftype="Roadmaps"/> + <composite_data value='velveth_test1/Log'/> + </param> + <output name="output" file="velveth_test1/output.html"> + <extra_files type="file" name="Sequences" value="velveth_test1/Sequences" /> + <extra_files type="file" name="Roadmaps" value="velveth_test1/Roadmaps" /> + <extra_files type="file" name="Log" value="velveth_test1/Log" /> + </output> + </test> + </tests> +</tool> diff -r 61e4e10e03ce8de315a1ae81d12529d80dedf594 -r df80192bfdcc3b8ed37b104c757e0dcc21535420 test/functional/tools/multi_output.xml --- /dev/null +++ b/test/functional/tools/multi_output.xml @@ -0,0 +1,22 @@ +<tool id="multi_output" name="Multi_Output" description="multi_output" force_history_refresh="True" version="0.1.0"> + <command> + echo "Hello" > $report; + echo "World" > '${__new_file_path__}/primary_${report.id}_moo_visible_?' + </command> + <inputs> + <param name="input" type="integer" value="7" /> + </inputs> + <outputs> + <data format="txt" name="report" /> + </outputs> + <tests> + <test> + <param name="input" value="7" /> + <output name="report"> + <assert_contents> + <has_line line="Hello" /> + </assert_contents> + </output> + </test> + </tests> +</tool> diff -r 61e4e10e03ce8de315a1ae81d12529d80dedf594 -r df80192bfdcc3b8ed37b104c757e0dcc21535420 test/functional/tools/multi_page.xml --- /dev/null +++ b/test/functional/tools/multi_page.xml @@ -0,0 +1,29 @@ +<tool id="multi_page" name="Multi_Page" description="multi_page" version="0.1.0"> + <configfiles> + <configfile name="config">${p1} ${p2}</configfile> + </configfiles> + <command>cat $config > $output</command> + <inputs> + <page> + <param name="p1" type="integer" value="1" /> + </page> + <page> + <param name="p2" type="integer" value="2" /> + </page> + </inputs> + <outputs> + <data format="txt" name="output" /> + </outputs> + <tests> + <test> + <param name="p1" value="3" /> + <param name="p2" value="4" /> + + <output name="output"> + <assert_contents> + <has_line line="3 4" /> + </assert_contents> + </output> + </test> + </tests> +</tool> diff -r 61e4e10e03ce8de315a1ae81d12529d80dedf594 -r df80192bfdcc3b8ed37b104c757e0dcc21535420 test/functional/tools/multi_select.xml --- /dev/null +++ b/test/functional/tools/multi_select.xml @@ -0,0 +1,29 @@ +<tool id="multi_select" name="multi_select" version="1.0.0"> + <description>multi_select</description> + <configfiles> + <configfile name="config">${select_ex}</configfile> + </configfiles> + <command>cat $config > $output</command> + <inputs> + <param name="select_ex" type="select" display="checkboxes" multiple="true"> + <option value="--ex1">Ex1</option> + <option value="ex2">Ex2</option> + <option value="--ex3">Ex3</option> + <option value="--ex4">Ex4</option> + <option value="ex5">Ex5</option> + </param> + </inputs> + <outputs> + <data format="txt" name="output" /> + </outputs> + <tests> + <test> + <param name="select_ex" value="--ex1,ex2,--ex3" /> + <output name="output"> + <assert_contents> + <has_line line="--ex1,ex2,--ex3" /> + </assert_contents> + </output> + </test> + </tests> +</tool> diff -r 61e4e10e03ce8de315a1ae81d12529d80dedf594 -r df80192bfdcc3b8ed37b104c757e0dcc21535420 test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -1,7 +1,12 @@ <?xml version="1.0"?><toolbox><tool file="upload.xml"/> + <tool file="simple_constructs.xml" /><tool file="composite.xml" /><tool file="disambiguate_cond.xml" /><tool file="multi_repeats.xml"/> + <tool file="multi_page.xml"/> + <tool file="multi_select.xml" /> + <tool file="multi_output.xml" /> + <tool file="composite_output.xml" /></toolbox> \ No newline at end of file diff -r 61e4e10e03ce8de315a1ae81d12529d80dedf594 -r df80192bfdcc3b8ed37b104c757e0dcc21535420 test/functional/tools/simple_constructs.xml --- /dev/null +++ b/test/functional/tools/simple_constructs.xml @@ -0,0 +1,47 @@ +<tool id="simple_constructs" name="simple_constructs"> + <command> + echo "$p1.p1val" >> $out_file1; + echo "$booltest" >> $out_file1; + echo "$inttest" >> $out_file1; + echo "$floattest" >> $out_file1; + cat "$files[0].file" >> $out_file1; + </command> + <inputs> + <conditional name="p1"> + <param type="boolean" name="p1use" /> + <when value="true"> + <param name="p1val" value="p1used" type="text" /> + </when> + <when value="false"> + <param name="p1val" value="p1notused" type="text" /> + </when> + </conditional> + <param name="booltest" truevalue="booltrue" falsevalue="boolfalse" checked="false" type="boolean" /> + <param name="inttest" value="1" type="integer" /> + <param name="floattest" value="1.0" type="float" /> + <repeat name="files" title="Files"> + <param name="file" type="data" format="txt" /> + </repeat> + </inputs> + <outputs> + <data name="out_file1" format="txt" /> + </outputs> + <tests> + <test> + <param name="p1use" value="true" /> + <param name="booltest" value="true" /> + <param name="inttest" value="12456" /> + <param name="floattest" value="6.789" /> + <param name="file" value="simple_line.txt" /> + <output name="out_file1"> + <assert_contents> + <has_line line="p1used" /> + <has_line line="booltrue" /> + <has_line line="12456" /> + <has_line line="6.789" /> + <has_line line="This is a line of text." /> + </assert_contents> + </output> + </test> + </tests> +</tool> https://bitbucket.org/galaxy/galaxy-central/commits/e3731610302a/ Changeset: e3731610302a User: jmchilton Date: 2013-11-22 02:17:12 Summary: Provide fallback implementation of requests functionality if lib unavailable. Just limited post/get functionality needed for tests. Affected #: 1 file diff -r df80192bfdcc3b8ed37b104c757e0dcc21535420 -r e3731610302a9de11bc15c3b039fac7a17a17558 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -7,8 +7,7 @@ import galaxy.model from galaxy.model.orm import and_, desc from galaxy.model.mapping import context as sa_session -from simplejson import dumps -import requests +from simplejson import dumps, loads toolbox = None @@ -133,7 +132,10 @@ } submit_response_object = self.__submit_tool( history_id, "upload1", tool_input, extra_data={"type": "upload_dataset"}, files=files ) submit_response = submit_response_object.json() - dataset = submit_response["outputs"][0] + try: + dataset = submit_response["outputs"][0] + except KeyError: + raise Exception(submit_response) #raise Exception(str(dataset)) hid = dataset['id'] self.uploads[ os.path.basename(fname) ] = self.uploads[ fname ] = self.uploads[ name ] = {"src": "hda", "id": hid} @@ -242,7 +244,7 @@ key = self.api_key data = data.copy() data['key'] = key - return requests.post( "%s/%s" % (self.api_url, path), data=data, files=files ) + return post_request( "%s/%s" % (self.api_url, path), data=data, files=files ) def __get( self, path, data={}, key=None ): if not key: @@ -252,7 +254,7 @@ if path.startswith("/api"): path = path[ len("/api"): ] url = "%s/%s" % (self.api_url, path) - return requests.get( url, params=data ) + return get_request( url, params=data ) class GalaxyInteractorTwill( object ): @@ -398,3 +400,66 @@ 'api': GalaxyInteractorApi, 'twill': GalaxyInteractorTwill, } + + +# Lets just try to use requests if it is available, but if not provide fallback +# on custom implementations of limited requests get/post functionality. +try: + from requests import get as get_request + from requests import post as post_request +except ImportError: + import urllib2 + import httplib + + class RequestsLikeResponse( object ): + + def __init__( self, content ): + self.content = content + + def json( self ): + return loads( self.content ) + + def get_request( url, params={} ): + argsep = '&' + if '?' not in url: + argsep = '?' + url = url + argsep + '&'.join( [ '%s=%s' % (k, v) for k, v in params.iteritems() ] ) + #req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' } ) + return RequestsLikeResponse(urllib2.urlopen( url ).read() ) + + def post_request( url, data, files ): + parsed_url = urllib2.urlparse.urlparse( url ) + return __post_multipart( host=parsed_url.netloc, selector=parsed_url.path, fields=data.iteritems(), files=(files or {}).iteritems() ) + + # http://stackoverflow.com/a/681182 + def __post_multipart(host, selector, fields, files): + content_type, body = __encode_multipart_formdata(fields, files) + h = httplib.HTTP(host) + h.putrequest('POST', selector) + h.putheader('content-type', content_type) + h.putheader('content-length', str(len(body))) + h.endheaders() + h.send(body) + errcode, errmsg, headers = h.getreply() + return RequestsLikeResponse(h.file.read()) + + def __encode_multipart_formdata(fields, files): + LIMIT = '----------lImIt_of_THE_fIle_eW_$' + CRLF = '\r\n' + L = [] + for (key, value) in fields: + L.append('--' + LIMIT) + L.append('Content-Disposition: form-data; name="%s"' % key) + L.append('') + L.append(value) + for (key, value) in files: + L.append('--' + LIMIT) + L.append('Content-Disposition: form-data; name="%s"; filename="%s";' % (key, key)) + L.append('Content-Type: application/octet-stream') + L.append('') + L.append(value.read()) + L.append('--' + LIMIT + '--') + L.append('') + body = CRLF.join(L) + content_type = 'multipart/form-data; boundary=%s' % LIMIT + return content_type, body https://bitbucket.org/galaxy/galaxy-central/commits/cf493be30252/ Changeset: cf493be30252 User: jmchilton Date: 2013-11-22 02:17:12 Summary: Twill-less tool tests - handle input metadata tags. Affected #: 1 file diff -r e3731610302a9de11bc15c3b039fac7a17a17558 -r cf493be302529fe2271b2a339f8a3fada63301c8 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -104,6 +104,9 @@ "file_type": test_data[ 'ftype' ], "dbkey": test_data[ 'dbkey' ], } + for elem in test_data.get('metadata', []): + tool_input["files_metadata|%s" % elem.get( 'name' )] = elem.get( 'value' ) + composite_data = test_data[ 'composite_data' ] if composite_data: files = {} https://bitbucket.org/galaxy/galaxy-central/commits/45c1d11cfff5/ Changeset: 45c1d11cfff5 User: jmchilton Date: 2013-11-22 02:17:12 Summary: Tool functional tests - allow checking extra files without checking a primary file. For output datatypes where primary file is meaningless. Affected #: 1 file diff -r cf493be302529fe2271b2a339f8a3fada63301c8 -r 45c1d11cfff552ce095c0561b5fba2cb5d3610e3 lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -252,8 +252,6 @@ assert_list = self.__parse_assert_list( output_elem ) file = attrib.pop( 'file', None ) # File no longer required if an list of assertions was present. - if not assert_list and file is None: - raise Exception( "Test output does not have a 'file' to compare with or list of assertions to check") attributes = {} # Method of comparison attributes['compare'] = attrib.pop( 'compare', 'diff' ).lower() @@ -262,12 +260,15 @@ # Allow a file size to vary if sim_size compare attributes['delta'] = int( attrib.pop( 'delta', '10000' ) ) attributes['sort'] = string_as_bool( attrib.pop( 'sort', False ) ) - attributes['extra_files'] = [] - attributes['assert_list'] = assert_list + extra_files = [] if 'ftype' in attrib: attributes['ftype'] = attrib['ftype'] for extra in output_elem.findall( 'extra_files' ): - attributes['extra_files'].append( self.__parse_extra_files_elem( extra ) ) + extra_files.append( self.__parse_extra_files_elem( extra ) ) + if not (assert_list or file or extra_files): + raise Exception( "Test output defines not checks (e.g. must have a 'file' check against, assertions to check, etc...)") + attributes['assert_list'] = assert_list + attributes['extra_files'] = extra_files self.__add_output( name, file, attributes ) except Exception, e: self.error = True https://bitbucket.org/galaxy/galaxy-central/commits/01f05ab8df70/ Changeset: 01f05ab8df70 User: jmchilton Date: 2013-11-22 02:17:13 Summary: Extend tool functional test framework to allow testing output dataset metadata. Adding file test/functional/tools/metadata.xml demonstrating how to check output metadata - this file also demonstrates setting metadata on uploaded datasets and verifies both of these functionalities. Checking output metadata is only available for new API driven tool testing. Affected #: 4 files diff -r 45c1d11cfff552ce095c0561b5fba2cb5d3610e3 -r 01f05ab8df7092b6e03f35faf7f80b47acafa0eb lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -265,10 +265,14 @@ attributes['ftype'] = attrib['ftype'] for extra in output_elem.findall( 'extra_files' ): extra_files.append( self.__parse_extra_files_elem( extra ) ) - if not (assert_list or file or extra_files): + metadata = {} + for metadata_elem in output_elem.findall( 'metadata' ): + metadata[ metadata_elem.get('name') ] = metadata_elem.get( 'value' ) + if not (assert_list or file or extra_files or metadata): raise Exception( "Test output defines not checks (e.g. must have a 'file' check against, assertions to check, etc...)") attributes['assert_list'] = assert_list attributes['extra_files'] = extra_files + attributes['metadata'] = metadata self.__add_output( name, file, attributes ) except Exception, e: self.error = True diff -r 45c1d11cfff552ce095c0561b5fba2cb5d3610e3 -r 01f05ab8df7092b6e03f35faf7f80b47acafa0eb test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -88,6 +88,21 @@ fetcher = self.__dataset_fetcher( history_id ) ## TODO: Twill version verifys dataset is 'ok' in here. self.twill_test_case.verify_hid( outfile, hda_id=hid, attributes=attributes, dataset_fetcher=fetcher, shed_tool_id=shed_tool_id ) + metadata = attributes.get( 'metadata', {} ) + if metadata: + dataset = self.__get( "histories/%s/contents/%s" % ( history_id, hid ) ).json() + for key, value in metadata.iteritems(): + dataset_key = "metadata_%s" % key + try: + dataset_value = dataset.get( dataset_key, None ) + if dataset_value != value: + msg = "Dataset metadata verification for [%s] failed, expected [%s] but found [%s]." + msg_params = ( key, value, dataset_value ) + msg = msg % msg_params + raise Exception( msg ) + except KeyError: + msg = "Failed to verify dataset metadata, metadata key [%s] was not found." % key + raise Exception( msg ) def get_job_stream( self, history_id, output_data, stream ): hid = output_data.get( 'id' ) diff -r 45c1d11cfff552ce095c0561b5fba2cb5d3610e3 -r 01f05ab8df7092b6e03f35faf7f80b47acafa0eb test/functional/tools/metadata.xml --- /dev/null +++ b/test/functional/tools/metadata.xml @@ -0,0 +1,29 @@ +<tool id="metadata" name="metadata" version="1.0.0"> + <command>mkdir $output_copy_of_input.extra_files_path; cp $input.extra_files_path/* $output_copy_of_input.extra_files_path; echo "$input.metadata.base_name" > $output_of_input_metadata</command> + <inputs> + <param name="input" type="data" format="velvet" label="Velvet Dataset" help="Prepared by velveth."/> + </inputs> + <outputs> + <data format="txt" name="output_of_input_metadata" /> + <data format="velvet" name="output_copy_of_input" /> + </outputs> + <tests> + <test> + <param name="input" value="velveth_test1/output.html" ftype="velvet" > + <composite_data value='velveth_test1/Sequences' ftype="Sequences"/> + <composite_data value='velveth_test1/Roadmaps' ftype="Roadmaps"/> + <composite_data value='velveth_test1/Log'/> + <metadata name="base_name" value="Example Metadata" /> + </param> +  + <output name="output_of_input_metadata"> + <assert_contents> + <has_line line="Example Metadata" /> + </assert_contents> + </output> +  + <output name="output_copy_of_input" file="velveth_test1/output.html"> + <metadata name="base_name" value="velvet" /> + </output> + </test> + </tests></tool> diff -r 45c1d11cfff552ce095c0561b5fba2cb5d3610e3 -r 01f05ab8df7092b6e03f35faf7f80b47acafa0eb test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -9,4 +9,5 @@ <tool file="multi_select.xml" /><tool file="multi_output.xml" /><tool file="composite_output.xml" /> + <tool file="metadata.xml" /></toolbox> \ No newline at end of file https://bitbucket.org/galaxy/galaxy-central/commits/d05be33ad6b7/ Changeset: d05be33ad6b7 User: jmchilton Date: 2013-11-22 02:17:13 Summary: Allow outputs in tool functional tests to be specified in any order when using API interactor. Outputs must specify name and must be specified in the same order with the Twill variant. This second restriction is entirely arbitrary using API so dropping it here. Adding tool demonstrating this functionality (test/functional/tools/output_order.xml) which fails with twill interactor but works fine with API interactor. Affected #: 3 files diff -r 01f05ab8df7092b6e03f35faf7f80b47acafa0eb -r d05be33ad6b772c2a688d875c4cf895d6afac4e3 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -57,12 +57,16 @@ def __verify_outputs( self, testdef, history, shed_tool_id, data_list, galaxy_interactor ): maxseconds = testdef.maxseconds - output_index = 0 - len( testdef.outputs ) - for output_tuple in testdef.outputs: + for output_index, output_tuple in enumerate(testdef.outputs): # Get the correct hid - output_data = data_list[ output_index ] + name, outfile, attributes = output_tuple + try: + output_data = data_list[ name ] + except (TypeError, KeyError): + # Legacy - fall back on ordered data list access if data_list is + # just a list (case with twill variant) + output_data = data_list[ len(data_list) - len(testdef.outputs) + output_index ] self.assertTrue( output_data is not None ) - name, outfile, attributes = output_tuple try: galaxy_interactor.verify_output( history, output_data, outfile, attributes=attributes, shed_tool_id=shed_tool_id, maxseconds=maxseconds ) except Exception: @@ -71,8 +75,6 @@ print >>sys.stderr, self._format_stream( stream_output, stream=stream, format=True ) raise - output_index += 1 - class GalaxyInteractorApi( object ): @@ -199,10 +201,21 @@ datasets = self.__submit_tool( history_id, tool_id=testdef.tool.id, tool_input=tool_input ) datasets_object = datasets.json() try: - return datasets_object[ 'outputs' ] + return self.__dictify_outputs( datasets_object ) except KeyError: raise Exception( datasets_object[ 'message' ] ) + def __dictify_outputs( self, datasets_object ): + ## Convert outputs list to a dictionary that can be accessed by + ## output_name so can be more flexiable about ordering of outputs + ## but also allows fallback to legacy access as list mode. + outputs_dict = {} + index = 0 + for output in datasets_object[ 'outputs' ]: + outputs_dict[ index ] = outputs_dict[ output.get("output_name") ] = output + index += 1 + return outputs_dict + def output_hid( self, output_data ): return output_data[ 'id' ] diff -r 01f05ab8df7092b6e03f35faf7f80b47acafa0eb -r d05be33ad6b772c2a688d875c4cf895d6afac4e3 test/functional/tools/output_order.xml --- /dev/null +++ b/test/functional/tools/output_order.xml @@ -0,0 +1,25 @@ +<tool id="output_order" name="output_order" version="0.1.0"> + <command>echo $pa > $output_a; echo $pb > $output_b</command> + <inputs> + <param name="pa" type="integer" value="1" /> + <param name="pb" type="integer" value="2" /> + </inputs> + <outputs> + <data format="txt" name="output_a" /> + <data format="txt" name="output_b" /> + </outputs> + <tests> + <test> + <output name="output_b"> + <assert_contents> + <has_line line="2" /> + </assert_contents> + </output> + <output name="output_a"> + <assert_contents> + <has_line line="1" /> + </assert_contents> + </output> + </test> + </tests> +</tool> diff -r 01f05ab8df7092b6e03f35faf7f80b47acafa0eb -r d05be33ad6b772c2a688d875c4cf895d6afac4e3 test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -10,4 +10,5 @@ <tool file="multi_output.xml" /><tool file="composite_output.xml" /><tool file="metadata.xml" /> + <tool file="output_order.xml" /></toolbox> \ No newline at end of file https://bitbucket.org/galaxy/galaxy-central/commits/1cb070e3da5b/ Changeset: 1cb070e3da5b User: jmchilton Date: 2013-11-22 02:17:13 Summary: Refactor lib/galaxy/tools/test.py parsing into smaller methods. Affected #: 1 file diff -r d05be33ad6b772c2a688d875c4cf895d6afac4e3 -r 1cb070e3da5b77f16eb3b8591a1cacd74389c34c lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -192,10 +192,6 @@ return value def __parse_elem( self, test_elem, i, default_interactor ): - # Composite datasets need a unique name: each test occurs in a fresh - # history, but we'll keep it unique per set of tests - use i (test #) - # and composite_data_names_counter (instance per test #) - composite_data_names_counter = 0 try: # Mechanism test code uses for interacting with Galaxy instance, # until 'api' is the default switch this to API to use its new @@ -203,81 +199,92 @@ # features or workarounds. self.interactor = test_elem.get( 'interactor', default_interactor ) - for param_elem in test_elem.findall( "param" ): - attrib = dict( param_elem.attrib ) - if 'values' in attrib: - value = attrib[ 'values' ].split( ',' ) - elif 'value' in attrib: - value = attrib['value'] - else: - value = None - attrib['children'] = list( param_elem.getchildren() ) - if attrib['children']: - # At this time, we can assume having children only - # occurs on DataToolParameter test items but this could - # change and would cause the below parsing to change - # based upon differences in children items - attrib['metadata'] = [] - attrib['composite_data'] = [] - attrib['edit_attributes'] = [] - # Composite datasets need to be renamed uniquely - composite_data_name = None - for child in attrib['children']: - if child.tag == 'composite_data': - attrib['composite_data'].append( child ) - if composite_data_name is None: - # Generate a unique name; each test uses a - # fresh history - composite_data_name = '_COMPOSITE_RENAMED_t%i_d%i' \ - % ( i, composite_data_names_counter ) - composite_data_names_counter += 1 - elif child.tag == 'metadata': - attrib['metadata'].append( child ) - elif child.tag == 'metadata': - attrib['metadata'].append( child ) - elif child.tag == 'edit_attributes': - attrib['edit_attributes'].append( child ) - if composite_data_name: - # Composite datasets need implicit renaming; - # inserted at front of list so explicit declarations - # take precedence - attrib['edit_attributes'].insert( 0, { 'type': 'name', 'value': composite_data_name } ) - self.__add_param( attrib.pop( 'name' ), value, attrib ) - for output_elem in test_elem.findall( "output" ): - attrib = dict( output_elem.attrib ) - name = attrib.pop( 'name', None ) - if name is None: - raise Exception( "Test output does not have a 'name'" ) + self.__parse_inputs_elems( test_elem, i ) - assert_list = self.__parse_assert_list( output_elem ) - file = attrib.pop( 'file', None ) - # File no longer required if an list of assertions was present. - attributes = {} - # Method of comparison - attributes['compare'] = attrib.pop( 'compare', 'diff' ).lower() - # Number of lines to allow to vary in logs (for dates, etc) - attributes['lines_diff'] = int( attrib.pop( 'lines_diff', '0' ) ) - # Allow a file size to vary if sim_size compare - attributes['delta'] = int( attrib.pop( 'delta', '10000' ) ) - attributes['sort'] = string_as_bool( attrib.pop( 'sort', False ) ) - extra_files = [] - if 'ftype' in attrib: - attributes['ftype'] = attrib['ftype'] - for extra in output_elem.findall( 'extra_files' ): - extra_files.append( self.__parse_extra_files_elem( extra ) ) - metadata = {} - for metadata_elem in output_elem.findall( 'metadata' ): - metadata[ metadata_elem.get('name') ] = metadata_elem.get( 'value' ) - if not (assert_list or file or extra_files or metadata): - raise Exception( "Test output defines not checks (e.g. must have a 'file' check against, assertions to check, etc...)") - attributes['assert_list'] = assert_list - attributes['extra_files'] = extra_files - attributes['metadata'] = metadata - self.__add_output( name, file, attributes ) + self.__parse_output_elems( test_elem ) except Exception, e: self.error = True self.exception = e + def __parse_inputs_elems( self, test_elem, i ): + # Composite datasets need a unique name: each test occurs in a fresh + # history, but we'll keep it unique per set of tests - use i (test #) + # and composite_data_names_counter (instance per test #) + composite_data_names_counter = 0 + for param_elem in test_elem.findall( "param" ): + attrib = dict( param_elem.attrib ) + if 'values' in attrib: + value = attrib[ 'values' ].split( ',' ) + elif 'value' in attrib: + value = attrib['value'] + else: + value = None + attrib['children'] = list( param_elem.getchildren() ) + if attrib['children']: + # At this time, we can assume having children only + # occurs on DataToolParameter test items but this could + # change and would cause the below parsing to change + # based upon differences in children items + attrib['metadata'] = [] + attrib['composite_data'] = [] + attrib['edit_attributes'] = [] + # Composite datasets need to be renamed uniquely + composite_data_name = None + for child in attrib['children']: + if child.tag == 'composite_data': + attrib['composite_data'].append( child ) + if composite_data_name is None: + # Generate a unique name; each test uses a + # fresh history + composite_data_name = '_COMPOSITE_RENAMED_t%i_d%i' \ + % ( i, composite_data_names_counter ) + composite_data_names_counter += 1 + elif child.tag == 'metadata': + attrib['metadata'].append( child ) + elif child.tag == 'metadata': + attrib['metadata'].append( child ) + elif child.tag == 'edit_attributes': + attrib['edit_attributes'].append( child ) + if composite_data_name: + # Composite datasets need implicit renaming; + # inserted at front of list so explicit declarations + # take precedence + attrib['edit_attributes'].insert( 0, { 'type': 'name', 'value': composite_data_name } ) + self.__add_param( attrib.pop( 'name' ), value, attrib ) + + def __parse_output_elems( self, test_elem ): + for output_elem in test_elem.findall( "output" ): + attrib = dict( output_elem.attrib ) + name = attrib.pop( 'name', None ) + if name is None: + raise Exception( "Test output does not have a 'name'" ) + + assert_list = self.__parse_assert_list( output_elem ) + file = attrib.pop( 'file', None ) + # File no longer required if an list of assertions was present. + attributes = {} + # Method of comparison + attributes['compare'] = attrib.pop( 'compare', 'diff' ).lower() + # Number of lines to allow to vary in logs (for dates, etc) + attributes['lines_diff'] = int( attrib.pop( 'lines_diff', '0' ) ) + # Allow a file size to vary if sim_size compare + attributes['delta'] = int( attrib.pop( 'delta', '10000' ) ) + attributes['sort'] = string_as_bool( attrib.pop( 'sort', False ) ) + extra_files = [] + if 'ftype' in attrib: + attributes['ftype'] = attrib['ftype'] + for extra in output_elem.findall( 'extra_files' ): + extra_files.append( self.__parse_extra_files_elem( extra ) ) + metadata = {} + for metadata_elem in output_elem.findall( 'metadata' ): + metadata[ metadata_elem.get('name') ] = metadata_elem.get( 'value' ) + if not (assert_list or file or extra_files or metadata): + raise Exception( "Test output defines not checks (e.g. must have a 'file' check against, assertions to check, etc...)") + attributes['assert_list'] = assert_list + attributes['extra_files'] = extra_files + attributes['metadata'] = metadata + self.__add_output( name, file, attributes ) + def __parse_assert_list( self, output_elem ): assert_elem = output_elem.find("assert_contents") assert_list = None https://bitbucket.org/galaxy/galaxy-central/commits/aa6ec1842ba6/ Changeset: aa6ec1842ba6 User: jmchilton Date: 2013-11-22 02:17:13 Summary: Utilize tool test runtime tree analysis earlier to more correctly determine which inputs are data params. Turns out previous changesets attempting to separately disambiguate and supply implicit defaults for conditionals do not really work with data inputs - because they are not recognized as dataset input params during parsing. Additionally, this change should eliminate any ordering requirements of tool test params - i.e. you shouldn't need to provide the test parameters in an order that makes it clear what branch you are on before suppling a dataset in order for the parser to realize it is a dataset. Finally, this changeset allows deletion of a lot of code and creates some more robust abstractions. Affected #: 2 files diff -r 1cb070e3da5b77f16eb3b8591a1cacd74389c34c -r aa6ec1842ba60b5ba7af69f8f01624c4902bc2be lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -75,29 +75,7 @@ yield data_dict - def to_dict( self, tool_inputs, declared_inputs ): - expanded_inputs = {} - for key, value in tool_inputs.items(): - if isinstance( value, grouping.Conditional ): - for i, case in enumerate( value.cases ): - if declared_inputs[ value.test_param.name ] == case.value: - pass # TODO - elif isinstance( value, grouping.Repeat ): - values = [] - for r_name, r_value in value.inputs.iteritems(): - values.append( self.to_dict( {r_name: r_value} , declared_inputs ) ) - expanded_inputs[ value.name ] = values - elif value.name not in declared_inputs: - print "%s not declared in tool test, will not change default value." % value.name - else: - expanded_inputs[ value.name ] = declared_inputs[value.name] - return expanded_inputs - - def __matching_case( self, cond, declared_inputs, prefix, index=None ): - param = cond.test_param - declared_value = self.__declared_match( declared_inputs, param.name, prefix) - if index is not None: - declared_value = declared_value[index] + def __matching_case_for_value( self, cond, declared_value ): for i, case in enumerate( cond.cases ): if declared_value is not None and (case.value == declared_value): return case @@ -107,83 +85,7 @@ return case else: return Bunch(value=declared_value, inputs=Bunch(items=lambda: [])) - print "Not matching case found for %s value %s. Test may fail in unexpected ways." % ( param.name, declared_value ) - - def expand_multi_grouping( self, tool_inputs, declared_inputs, prefix='', index=0 ): - """ - Used by API, slight generalization of expand_grouping used by Twill based interactor. Still - not quite the context/tree based specification that should exist! - """ - expanded_inputs = {} - for key, value in tool_inputs.items(): - expanded_key = value.name if not prefix else "%s|%s" % (prefix, value.name) - if isinstance( value, grouping.Conditional ): - new_prefix = expanded_key - case = self.__matching_case( value, declared_inputs, new_prefix, index=index ) - if case: - expanded_value = self.__split_if_str( case.value ) - expanded_inputs[ "%s|%s" % ( new_prefix, value.test_param.name ) ] = expanded_value - for input_name, input_value in case.inputs.items(): - expanded_inputs.update( self.expand_multi_grouping( { input_name: input_value }, declared_inputs, prefix=new_prefix ), index=index ) - elif isinstance( value, grouping.Repeat ): - repeat_index = 0 - any_children_matched = True - while any_children_matched: - any_children_matched = False - for r_name, r_value in value.inputs.iteritems(): - new_prefix = "%s_%d" % ( value.name, repeat_index ) - if prefix: - new_prefix = "%s|%s" % ( prefix, new_prefix ) - expanded_input = self.expand_multi_grouping( { new_prefix : r_value }, declared_inputs, prefix=new_prefix, index=repeat_index ) - if expanded_input: - any_children_matched = True - expanded_inputs.update( expanded_input ) - repeat_index += 1 - else: - declared_value = self.__declared_match( declared_inputs, value.name, prefix ) - if declared_value and len(declared_value) > index: - value = self.__split_if_str( declared_value[ index ] ) - expanded_inputs[ expanded_key ] = value - return expanded_inputs - - def expand_grouping( self, tool_inputs, declared_inputs, prefix='' ): - expanded_inputs = {} - for key, value in tool_inputs.items(): - expanded_key = value.name if not prefix else "%s|%s" % (prefix, value.name) - if isinstance( value, grouping.Conditional ): - new_prefix = expanded_key - case = self.__matching_case( value, declared_inputs, new_prefix ) - if case: - expanded_value = self.__split_if_str(case.value) - expanded_inputs[ "%s|%s" % ( new_prefix, value.test_param.name ) ] = expanded_value - for input_name, input_value in case.inputs.items(): - expanded_inputs.update( self.expand_grouping( { input_name: input_value }, declared_inputs, prefix=new_prefix ) ) - elif isinstance( value, grouping.Repeat ): - for repeat_index in xrange( 0, 1 ): # need to allow for and figure out how many repeats we have - for r_name, r_value in value.inputs.iteritems(): - new_prefix = "%s_%d" % ( value.name, repeat_index ) - if prefix: - new_prefix = "%s|%s" % ( prefix, new_prefix ) - expanded_inputs.update( self.expand_grouping( { new_prefix : r_value }, declared_inputs, prefix=new_prefix ) ) - else: - declared_value = self.__declared_match( declared_inputs, value.name, prefix ) - if not declared_value: - print "%s not declared in tool test, will not change default value." % value.name - else: - value = self.__split_if_str(declared_value) - expanded_inputs[expanded_key] = value - return expanded_inputs - - def __declared_match( self, declared_inputs, name, prefix ): - prefix_suffixes = [ "%s|" % part for part in prefix.split( "|" ) ] if prefix else [] - prefix_suffixes.append( name ) - prefix_suffixes.reverse() - prefixed_name = "" - for prefix_suffix in prefix_suffixes: - prefixed_name = "%s%s" % ( prefix_suffix, prefixed_name ) - if prefixed_name in declared_inputs: - return declared_inputs[prefixed_name] - return None + print "Not matching case found for %s value %s. Test may fail in unexpected ways." % ( cond.test_param.name, declared_value ) def __split_if_str( self, value ): split = isinstance(value, str) @@ -211,6 +113,7 @@ # history, but we'll keep it unique per set of tests - use i (test #) # and composite_data_names_counter (instance per test #) composite_data_names_counter = 0 + raw_inputs = {} for param_elem in test_elem.findall( "param" ): attrib = dict( param_elem.attrib ) if 'values' in attrib: @@ -250,7 +153,56 @@ # inserted at front of list so explicit declarations # take precedence attrib['edit_attributes'].insert( 0, { 'type': 'name', 'value': composite_data_name } ) - self.__add_param( attrib.pop( 'name' ), value, attrib ) + name = attrib.pop( 'name' ) + if name not in raw_inputs: + raw_inputs[ name ] = [] + raw_inputs[ name ].append( ( value, attrib ) ) + self.inputs = self.__process_raw_inputs( self.tool.inputs, raw_inputs ) + + def __process_raw_inputs( self, tool_inputs, raw_inputs, parent_context=None ): + """ + Recursively expand flat list of inputs into "tree" form of flat list + (| using to nest to new levels) structure and expand dataset + information as proceeding to populate self.required_files. + """ + parent_context = parent_context or RootParamContext() + expanded_inputs = {} + for key, value in tool_inputs.items(): + if isinstance( value, grouping.Conditional ): + cond_context = ParamContext( name=value.name, parent_context=parent_context ) + case_context = ParamContext( name=value.test_param.name, parent_context=cond_context ) + raw_input = case_context.value( raw_inputs ) + case = self.__matching_case_for_value( value, raw_input ) + if case: + expanded_value = self.__split_if_str( case.value ) + expanded_inputs[ case_context.for_state() ] = expanded_value + for input_name, input_value in case.inputs.items(): + expanded_inputs.update( self.__process_raw_inputs( { input_name: input_value }, raw_inputs, parent_context=cond_context ) ) + elif isinstance( value, grouping.Repeat ): + repeat_index = 0 + while True: + context = ParamContext( name=value.name, index=repeat_index, parent_context=parent_context ) + updated = False + for r_name, r_value in value.inputs.iteritems(): + expanded_input = self.__process_raw_inputs( { context.for_state() : r_value }, raw_inputs, parent_context=context ) + if expanded_input: + expanded_inputs.update( expanded_input ) + updated = True + if not updated: + break + repeat_index += 1 + else: + context = ParamContext( name=value.name, parent_context=parent_context ) + raw_input = context.value( raw_inputs ) + if raw_input: + (param_value, param_extra) = raw_input + if isinstance( value, basic.DataToolParameter ): + processed_value = [ self.__add_uploaded_dataset( context.for_state(), param_value, param_extra, value ) ] + else: + param_value = self.__split_if_str( param_value ) + processed_value = param_value + expanded_inputs[ context.for_state() ] = processed_value + return expanded_inputs def __parse_output_elems( self, test_elem ): for output_elem in test_elem.findall( "output" ): @@ -324,78 +276,9 @@ extra_attributes['sort'] = string_as_bool( extra.get( 'sort', False ) ) return extra_type, extra_value, extra_name, extra_attributes - def __add_param( self, name, value, extra ): - try: - if name not in self.tool.inputs: - found_parameter = False - for input_name, input_value in self.tool.inputs.items(): - if isinstance( input_value, grouping.Group ): - found_parameter, new_value = self.__expand_grouping_for_data_input(name, value, extra, input_name, input_value) - if found_parameter: - value = new_value - break - if not found_parameter: - raise ValueError( "Unable to determine parameter type of test input '%s'. " - "Ensure that the parameter exists and that any container groups are defined first." - % name ) - elif isinstance( self.tool.inputs[name], basic.DataToolParameter ): - value = self.__add_uploaded_dataset( name, value, extra, self.tool.inputs[name] ) - except Exception, e: - log.debug( "Error for tool %s: could not add test parameter %s. %s" % ( self.tool.id, name, e ) ) - self.inputs.append( ( name, value, extra ) ) - def __add_output( self, name, file, extra ): self.outputs.append( ( name, file, extra ) ) - def __expand_grouping_for_data_input( self, name, value, extra, grouping_name, grouping_value ): - # Currently handles grouping.Conditional and grouping.Repeat - if isinstance( grouping_value, grouping.Conditional ): - if name == grouping_value.test_param.name: - return True, value - case_test_param_value = None - for input in self.inputs: - if input[0] == grouping_value.test_param.name: - case_test_param_value = input[1] - break - if case_test_param_value is None: - #case for this group has not been set yet - return False, value - for case in grouping_value.cases: - if case.value == case_test_param_value: - break - if case.value != case_test_param_value: - return False, value - #assert case.value == case_test_param_value, "Current case could not be determined for parameter '%s'. Provided value '%s' could not be found in '%s'." % ( grouping_value.name, value, grouping_value.test_param.name ) - if name in case.inputs: - if isinstance( case.inputs[name], basic.DataToolParameter ): - return True, self.__add_uploaded_dataset( name, value, extra, case.inputs[name] ) - else: - return True, value - else: - for input_name, input_parameter in case.inputs.iteritems(): - if isinstance( input_parameter, grouping.Group ): - found_parameter, new_value = self.__expand_grouping_for_data_input( name, value, extra, input_name, input_parameter ) - if found_parameter: - return True, new_value - elif isinstance( grouping_value, grouping.Repeat ): - # FIXME: grouping.Repeat can only handle 1 repeat param element since the param name - # is something like "input2" and the expanded page display is something like "queries_0|input2". - # The problem is that the only param name on the page is "input2", and adding more test input params - # with the same name ( "input2" ) is not yet supported in our test code ( the last one added is the only - # one used ). - if name in grouping_value.inputs: - if isinstance( grouping_value.inputs[name], basic.DataToolParameter ): - return True, self.__add_uploaded_dataset( name, value, extra, grouping_value.inputs[name] ) - else: - return True, value - else: - for input_name, input_parameter in grouping_value.inputs.iteritems(): - if isinstance( input_parameter, grouping.Group ): - found_parameter, new_value = self.__expand_grouping_for_data_input( name, value, extra, input_name, input_parameter ) - if found_parameter: - return True, new_value - return False, value - def __add_uploaded_dataset( self, name, value, extra, input_parameter ): if value is None: assert input_parameter.optional, '%s is not optional. You must provide a valid filename.' % name @@ -413,3 +296,58 @@ break value = os.path.basename( value ) # if uploading a file in a path other than root of test-data return value + + +class ParamContext(object): + + def __init__( self, name, index=None, parent_context=None ): + self.parent_context = parent_context + self.name = name + self.index = None if index is None else int( index ) + + def for_state( self ): + name = self.name if self.index is None else "%s_%d" % ( self.name, self.index ) + parent_for_state = self.parent_context.for_state() + if parent_for_state: + return "%s|%s" % ( parent_for_state, name ) + else: + return name + + def __str__( self ): + return "Context[for_state=%s]" % self.for_state() + + def param_names( self ): + for parent_context_param in self.parent_context.param_names(): + yield "%s|%s" % ( parent_context_param, self.name ) + yield self.name + + def value( self, declared_inputs ): + for param_name in self.param_names(): + if param_name in declared_inputs: + index = self.get_index() + try: + return declared_inputs[ param_name ][ index ] + except IndexError: + return None + return None + + def get_index( self ): + if self.index is not None: + return self.index + else: + return self.parent_context.get_index() + + +class RootParamContext(object): + + def __init__( self ): + pass + + def for_state( self ): + return "" + + def param_names( self ): + return [] + + def get_index( self ): + return 0 diff -r 1cb070e3da5b77f16eb3b8591a1cacd74389c34c -r aa6ec1842ba60b5ba7af69f8f01624c4902bc2be test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -164,41 +164,21 @@ def run_tool( self, testdef, history_id ): # We need to handle the case where we've uploaded a valid compressed file since the upload # tool will have uncompressed it on the fly. - all_inputs = {} - for name, value, _ in testdef.inputs: - # TODO: Restrict this to param inputs. - if value in self.uploads: - value = self.uploads[ value ] - if name in all_inputs: - all_inputs[name].append( value ) - else: - all_inputs[name] = [ value ] + inputs_tree = testdef.inputs.copy() + for key, value in inputs_tree.iteritems(): + values = [value] if not isinstance(value, list) else value + for value in values: + if value in self.uploads: + inputs_tree[ key ] = self.uploads[ value ] - # TODO: Handle pages? - # TODO: Handle force_history_refresh? - flat_inputs = True - if flat_inputs: - # Build up tool_input flately (e.g {"a_repeat_0|a_repeat_param" : "value1"}) - expanded_inputs = {} - expanded_inputs.update(testdef.expand_multi_grouping(testdef.tool.inputs_by_page[0], all_inputs)) - for i in range( 1, testdef.tool.npages ): - expanded_inputs.update(testdef.expand_multi_grouping(testdef.tool.inputs_by_page[i], all_inputs)) + # # HACK: Flatten single-value lists. Required when using expand_grouping + for key, value in inputs_tree.iteritems(): + if isinstance(value, list) and len(value) == 1: + inputs_tree[key] = value[0] - # # HACK: Flatten single-value lists. Required when using expand_grouping - for key, value in expanded_inputs.iteritems(): - if isinstance(value, list) and len(value) == 1: - expanded_inputs[key] = value[0] - tool_input = expanded_inputs - else: - # Build up tool_input as nested dictionary (e.g. {"a_repeat": [{"a_repeat_param" : "value1"}]}) - # Doesn't work with the tool API at this time. - tool_input = {} - tool_input.update(testdef.to_dict(testdef.tool.inputs_by_page[0], all_inputs)) - for i in range( 1, testdef.tool.npages ): - tool_input.update(testdef.to_dict(testdef.tool.inputs_by_page[i], all_inputs)) - - datasets = self.__submit_tool( history_id, tool_id=testdef.tool.id, tool_input=tool_input ) + log.info( "Submiting tool with params %s" % inputs_tree ) + datasets = self.__submit_tool( history_id, tool_id=testdef.tool.id, tool_input=inputs_tree ) datasets_object = datasets.json() try: return self.__dictify_outputs( datasets_object ) @@ -322,9 +302,11 @@ def run_tool( self, testdef, test_history ): # We need to handle the case where we've uploaded a valid compressed file since the upload # tool will have uncompressed it on the fly. + + # Lose tons of information to accomodate legacy repeat handling. all_inputs = {} - for name, value, _ in testdef.inputs: - all_inputs[ name ] = value + for key, value in testdef.inputs.iteritems(): + all_inputs[ key.split("|")[-1] ] = value # See if we have a grouping.Repeat element repeat_name = None @@ -340,15 +322,20 @@ else: job_finish_by_output_count = False + # Strip out just a given page of inputs from inputs "tree". + def filter_page_inputs( n ): + page_input_keys = testdef.tool.inputs_by_page[ n ].keys() + return dict( [ (k, v) for k, v in testdef.inputs.iteritems() if k.split("|")[0] in page_input_keys ] ) + # Do the first page - page_inputs = testdef.expand_grouping(testdef.tool.inputs_by_page[0], all_inputs) + page_inputs = filter_page_inputs( 0 ) # Run the tool self.twill_test_case.run_tool( testdef.tool.id, repeat_name=repeat_name, **page_inputs ) print "page_inputs (0)", page_inputs # Do other pages if they exist for i in range( 1, testdef.tool.npages ): - page_inputs = testdef.expand_grouping(testdef.tool.inputs_by_page[i], all_inputs) + page_inputs = filter_page_inputs( i ) self.twill_test_case.submit_form( **page_inputs ) print "page_inputs (%i)" % i, page_inputs https://bitbucket.org/galaxy/galaxy-central/commits/76bd49e65887/ Changeset: 76bd49e65887 User: jmchilton Date: 2013-11-22 02:17:13 Summary: Previous attempt to allow implicit defaults for conditional test params had many problems. This rewrite has fewer. Affected #: 1 file diff -r aa6ec1842ba60b5ba7af69f8f01624c4902bc2be -r 76bd49e658876f5cc90ec5b80033f13b058593b8 lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -3,7 +3,6 @@ from parameters import basic from parameters import grouping from galaxy.util import string_as_bool -from galaxy.util.bunch import Bunch import logging log = logging.getLogger( __name__ ) @@ -76,16 +75,43 @@ yield data_dict def __matching_case_for_value( self, cond, declared_value ): + test_param = cond.test_param + if isinstance(test_param, basic.BooleanToolParameter): + if declared_value is None: + # No explicit value for param in test case, determine from default + query_value = test_param.checked + else: + # Test case supplied value, check cases against this. + query_value = string_as_bool( declared_value ) + matches_declared_value = lambda case_value: string_as_bool(case_value) == query_value + elif isinstance(test_param, basic.SelectToolParameter): + if declared_value is not None: + # Test case supplied explicit value to check against. + matches_declared_value = lambda case_value: case_value == declared_value + elif test_param.static_options: + # No explicit value in test case, not much to do if options are dynamic but + # if static options are available can find the one specified as default or + # fallback on top most option (like GUI). + for (name, value, selected) in test_param.static_options: + if selected: + default_option = name + else: + default_option = test_param.static_options[0] + matches_declared_value = lambda case_value: case_value == default_option + else: + # No explicit value for this param and cannot determine a + # default - give up. Previously this would just result in a key + # error exception. + msg = "Failed to find test parameter specification required for conditional %s" % cond + raise Exception( msg ) + + # Check the tool's defined cases against predicate to determine + # selected or default. for i, case in enumerate( cond.cases ): - if declared_value is not None and (case.value == declared_value): - return case - if declared_value is None: - # TODO: Default might not be top value, fix this. - # TODO: Also may be boolean, got to look at checked. + if matches_declared_value( case.value ): return case else: - return Bunch(value=declared_value, inputs=Bunch(items=lambda: [])) - print "Not matching case found for %s value %s. Test may fail in unexpected ways." % ( cond.test_param.name, declared_value ) + log.info("Failed to find case matching test parameter specification for cond %s. Remainder of test behavior is unspecified." % cond) def __split_if_str( self, value ): split = isinstance(value, str) @@ -172,7 +198,8 @@ cond_context = ParamContext( name=value.name, parent_context=parent_context ) case_context = ParamContext( name=value.test_param.name, parent_context=cond_context ) raw_input = case_context.value( raw_inputs ) - case = self.__matching_case_for_value( value, raw_input ) + case_value = raw_input[ 0 ] if raw_input else None + case = self.__matching_case_for_value( value, case_value ) if case: expanded_value = self.__split_if_str( case.value ) expanded_inputs[ case_context.for_state() ] = expanded_value https://bitbucket.org/galaxy/galaxy-central/commits/07d48a9f9a09/ Changeset: 07d48a9f9a09 User: jmchilton Date: 2013-11-22 02:17:13 Summary: Make disambiguate_cond more difficult. Affected #: 2 files diff -r 76bd49e658876f5cc90ec5b80033f13b058593b8 -r 07d48a9f9a09edd916bf4f94e95326211cd0d255 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -9,6 +9,9 @@ from galaxy.model.mapping import context as sa_session from simplejson import dumps, loads +import logging +log = logging.getLogger( __name__ ) + toolbox = None diff -r 76bd49e658876f5cc90ec5b80033f13b058593b8 -r 07d48a9f9a09edd916bf4f94e95326211cd0d255 test/functional/tools/disambiguate_cond.xml --- a/test/functional/tools/disambiguate_cond.xml +++ b/test/functional/tools/disambiguate_cond.xml @@ -1,7 +1,8 @@ <tool id="handle_cond" name="handle_cond"><description>tail-to-head</description> + <command> - echo "$p1.p1v $p2.p2v $p3.p3v" > $out_file1 + echo "$p1.p1v $p2.p2v $p3.p3v" > $out_file1; cat "$files.p4.file" >> $out_file1; </command><inputs><conditional name="p1"> @@ -30,7 +31,18 @@ <when value="false"><param name="p3v" value="7" type="integer" /></when> - </conditional> + </conditional> + <conditional name="files"> + <param name="attach_files" type="boolean" checked="true" /> + <when value="true"> + <conditional name="p4"> + <param type="boolean" name="use" /> + <when value="true"> + <param type="data" name="file" /> + </when> + </conditional> + </when> + </conditional></inputs><outputs><data name="out_file1" format="txt" /> @@ -40,9 +52,12 @@ <param name="p1|use" value="True"/><param name="p2|use" value="False"/><param name="p3|use" value="True"/> + <param name="p4|use" value="True" /> + <param name="p4|file" value="simple_line.txt" /><output name="out_file1"><assert_contents><has_line line="4 7 4" /> + <has_line line="This is a line of text." /></assert_contents></output></test> https://bitbucket.org/galaxy/galaxy-central/commits/265e67c9dca8/ Changeset: 265e67c9dca8 User: jmchilton Date: 2013-11-22 02:17:13 Summary: Allow robust disambiguation of repeat statements in tool tests. Imagine a tool with a repeat statement like: <repeat name="rinst" title="Repeat Parameter"><param name="int_param" type="integer" value="1" /><param name="float_param" type="float" value="2.0" /></repeat> 3 test instances overidding int_param default in all three, but leaving the float_param default in place for the first and last can specified as follows: <param name="rinst_0|int_param" value="4" /><param name="rinst_1|int_param" value="5" /><param name="rtnst_2|int_param" value="6" /><param name="rinst_1|float_param" value="4.0" /> This syntax can be mixed and matched with specifing conditionals and nested repeats, etc.... One could imagine an alternative syntax like: <param name="rinst|int_param" value="4" /><param name="rinst|int_param" value="5" /><param name="rtnst|int_param" value="6" /><param name="rinst|float_param" value="4.0" /> Upon consideration, I have determined that this ambigious syntax is more difficult to reason about, leaves so much ambigiouity in place especially in the context of default and nested elements, and is harder to support. So the plan is to not support it at this time. The following changeset will add "the right" way to do this anyway: <repeat name="rinst"><param name="rinst|int_param" value="4" /></repeat><repeat name="rinst"><param name="rinst|int_param" value="5" /><param name="rinst|float_param" value="4.0" /></repeat><repeat name="rinst"><param name="rinst|int_param" value="6" /></repeat> Affected #: 7 files diff -r 07d48a9f9a09edd916bf4f94e95326211cd0d255 -r 265e67c9dca81c496ce42ba6517b10444ad56d59 lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -139,7 +139,7 @@ # history, but we'll keep it unique per set of tests - use i (test #) # and composite_data_names_counter (instance per test #) composite_data_names_counter = 0 - raw_inputs = {} + raw_inputs = [] for param_elem in test_elem.findall( "param" ): attrib = dict( param_elem.attrib ) if 'values' in attrib: @@ -180,9 +180,7 @@ # take precedence attrib['edit_attributes'].insert( 0, { 'type': 'name', 'value': composite_data_name } ) name = attrib.pop( 'name' ) - if name not in raw_inputs: - raw_inputs[ name ] = [] - raw_inputs[ name ].append( ( value, attrib ) ) + raw_inputs.append( ( name, value, attrib ) ) self.inputs = self.__process_raw_inputs( self.tool.inputs, raw_inputs ) def __process_raw_inputs( self, tool_inputs, raw_inputs, parent_context=None ): @@ -197,8 +195,8 @@ if isinstance( value, grouping.Conditional ): cond_context = ParamContext( name=value.name, parent_context=parent_context ) case_context = ParamContext( name=value.test_param.name, parent_context=cond_context ) - raw_input = case_context.value( raw_inputs ) - case_value = raw_input[ 0 ] if raw_input else None + raw_input = case_context.extract_value( raw_inputs ) + case_value = raw_input[ 1 ] if raw_input else None case = self.__matching_case_for_value( value, case_value ) if case: expanded_value = self.__split_if_str( case.value ) @@ -220,9 +218,9 @@ repeat_index += 1 else: context = ParamContext( name=value.name, parent_context=parent_context ) - raw_input = context.value( raw_inputs ) + raw_input = context.extract_value( raw_inputs ) if raw_input: - (param_value, param_extra) = raw_input + (name, param_value, param_extra) = raw_input if isinstance( value, basic.DataToolParameter ): processed_value = [ self.__add_uploaded_dataset( context.for_state(), param_value, param_extra, value ) ] else: @@ -345,24 +343,33 @@ def param_names( self ): for parent_context_param in self.parent_context.param_names(): - yield "%s|%s" % ( parent_context_param, self.name ) - yield self.name + if self.index is not None: + yield "%s|%s_%d" % ( parent_context_param, self.name, self.index ) + else: + yield "%s|%s" % ( parent_context_param, self.name ) + if self.index is not None: + yield "%s_%d" % ( self.name, self.index ) + else: + yield self.name - def value( self, declared_inputs ): + def extract_value( self, raw_inputs ): for param_name in self.param_names(): - if param_name in declared_inputs: - index = self.get_index() - try: - return declared_inputs[ param_name ][ index ] - except IndexError: - return None + value = self.__raw_param_found( param_name, raw_inputs) + if value: + return value return None - def get_index( self ): - if self.index is not None: - return self.index + def __raw_param_found( self, param_name, raw_inputs ): + index = None + for i, raw_input in enumerate( raw_inputs ): + if raw_input[ 0 ] == param_name: + index = i + if index is not None: + raw_input = raw_inputs[ index ] + del raw_inputs[ index ] + return raw_input else: - return self.parent_context.get_index() + return None class RootParamContext(object): diff -r 07d48a9f9a09edd916bf4f94e95326211cd0d255 -r 265e67c9dca81c496ce42ba6517b10444ad56d59 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -325,10 +325,16 @@ else: job_finish_by_output_count = False + inputs_tree = testdef.inputs + # # # HACK: Flatten single-value lists. Required when using expand_grouping + # #for key, value in inputs_tree.iteritems(): + # if isinstance(value, list) and len(value) == 1: + # inputs_tree[key] = value[0] + # Strip out just a given page of inputs from inputs "tree". def filter_page_inputs( n ): page_input_keys = testdef.tool.inputs_by_page[ n ].keys() - return dict( [ (k, v) for k, v in testdef.inputs.iteritems() if k.split("|")[0] in page_input_keys ] ) + return dict( [ (k, v) for k, v in inputs_tree.iteritems() if k.split("|")[0] or k.split("|")[0].resplit("_", 1)[0] in page_input_keys ] ) # Do the first page page_inputs = filter_page_inputs( 0 ) diff -r 07d48a9f9a09edd916bf4f94e95326211cd0d255 -r 265e67c9dca81c496ce42ba6517b10444ad56d59 test/functional/tools/disambiguate_repeats.xml --- /dev/null +++ b/test/functional/tools/disambiguate_repeats.xml @@ -0,0 +1,28 @@ +<tool id="disambiguate_repeats" name="disambiguate_repeats"> + <command> + cat #for $q in $queries# ${q.input} #end for# #for $q in $more_queries# ${q.input} #end for# > $out_file1 + </command> + <inputs> + <repeat name="queries" title="Dataset"> + <param name="input" type="data" label="Select" /> + </repeat> + <repeat name="more_queries" title="Dataset"> + <param name="input" type="data" label="Select" /> + </repeat> + </inputs> + <outputs> + <data name="out_file1" format="txt" /> + </outputs> + <tests> +  + <test> + <param name="queries_1|input" value="simple_line_alternative.txt"/> + <param name="queries_0|input" value="simple_line.txt"/> + + <param name="more_queries_1|input" value="simple_line_alternative.txt" /> + <param name="more_queries_0|input" value="simple_line.txt"/> + + <output name="out_file1" file="simple_lines_interleaved.txt"/> + </test> + </tests> +</tool> diff -r 07d48a9f9a09edd916bf4f94e95326211cd0d255 -r 265e67c9dca81c496ce42ba6517b10444ad56d59 test/functional/tools/multi_repeats.xml --- a/test/functional/tools/multi_repeats.xml +++ b/test/functional/tools/multi_repeats.xml @@ -21,6 +21,12 @@ <param name="input2" value="simple_line.txt"/><output name="out_file1" file="simple_line_x2.txt"/></test> + <test><param name="input1" value="simple_line.txt"/><param name="input2" value="simple_line.txt"/> diff -r 07d48a9f9a09edd916bf4f94e95326211cd0d255 -r 265e67c9dca81c496ce42ba6517b10444ad56d59 test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -11,4 +11,5 @@ <tool file="composite_output.xml" /><tool file="metadata.xml" /><tool file="output_order.xml" /> + <tool file="disambiguate_repeats.xml" /></toolbox> \ No newline at end of file diff -r 07d48a9f9a09edd916bf4f94e95326211cd0d255 -r 265e67c9dca81c496ce42ba6517b10444ad56d59 test/functional/tools/test-data/simple_line_alternative.txt --- /dev/null +++ b/test/functional/tools/test-data/simple_line_alternative.txt @@ -0,0 +1,1 @@ +This is a different line of text. \ No newline at end of file diff -r 07d48a9f9a09edd916bf4f94e95326211cd0d255 -r 265e67c9dca81c496ce42ba6517b10444ad56d59 test/functional/tools/test-data/simple_lines_interleaved.txt --- /dev/null +++ b/test/functional/tools/test-data/simple_lines_interleaved.txt @@ -0,0 +1,4 @@ +This is a line of text. +This is a different line of text. +This is a line of text. +This is a different line of text. \ No newline at end of file https://bitbucket.org/galaxy/galaxy-central/commits/e86455cbd0a5/ Changeset: e86455cbd0a5 User: jmchilton Date: 2013-11-22 02:17:13 Summary: Tree-like input specification in tool XML file. Allow nestable <repeat> and <conditional> statements in test tool definitions. For instance, imagine a tool with a repeat statement like: <repeat name="rinst" title="Repeat Parameter"><param name="int_param" type="integer" value="1" /><param name="float_param" type="float" value="2.0" /></repeat> 3 test instances overidding int_param default in all three, but leaving the float_param default in place for the first and last can specified as follows: <repeat name="rinst"><param name="rinst|int_param" value="4" /></repeat><repeat name="rinst"><param name="rinst|int_param" value="5" /><param name="rinst|float_param" value="4.0" /></repeat><repeat name="rinst"><param name="rinst|int_param" value="6" /></repeat> Likewise, <conditional name="cinst"> can be used analogously though this is for grouping and disambiguation - does not allow multiple instances to specified obviously. Affected #: 2 files diff -r 265e67c9dca81c496ce42ba6517b10444ad56d59 -r e86455cbd0a554613eee96852a1240205f65696b lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -127,6 +127,7 @@ # features or workarounds. self.interactor = test_elem.get( 'interactor', default_interactor ) + self.__preprocess_input_elems( test_elem ) self.__parse_inputs_elems( test_elem, i ) self.__parse_output_elems( test_elem ) @@ -134,6 +135,9 @@ self.error = True self.exception = e + def __preprocess_input_elems( self, test_elem ): + expand_input_elems( test_elem ) + def __parse_inputs_elems( self, test_elem, i ): # Composite datasets need a unique name: each test occurs in a fresh # history, but we'll keep it unique per set of tests - use i (test #) @@ -385,3 +389,46 @@ def get_index( self ): return 0 + + +def expand_input_elems( root_elem, prefix="" ): + __append_prefix_to_params( root_elem, prefix ) + + repeat_elems = root_elem.findall( 'repeat' ) + indices = {} + for repeat_elem in repeat_elems: + name = repeat_elem.get( "name" ) + if name not in indices: + indices[ name ] = 0 + index = 0 + else: + index = indices[ name ] + 1 + indices[ name ] = index + + new_prefix = __prefix_join( prefix, name, index=index ) + expand_input_elems( repeat_elem, new_prefix ) + __pull_up_params( root_elem, repeat_elem ) + root_elem.remove( repeat_elem ) + + cond_elems = root_elem.findall( 'conditional' ) + for cond_elem in cond_elems: + new_prefix = __prefix_join( prefix, cond_elem.get( "name" ) ) + expand_input_elems( cond_elem, new_prefix ) + __pull_up_params( root_elem, cond_elem ) + root_elem.remove( cond_elem ) + + +def __append_prefix_to_params( elem, prefix ): + for param_elem in elem.findall( 'param' ): + param_elem.set( "name", __prefix_join( prefix, param_elem.get( "name" ) ) ) + + +def __pull_up_params( parent_elem, child_elem ): + for param_elem in child_elem.findall( 'param' ): + parent_elem.append( param_elem ) + child_elem.remove( param_elem ) + + +def __prefix_join( prefix, name, index=None ): + name = name if index is None else "%s_%d" % ( name, index ) + return name if not prefix else "%s|%s" % ( prefix, name ) diff -r 265e67c9dca81c496ce42ba6517b10444ad56d59 -r e86455cbd0a554613eee96852a1240205f65696b test/functional/tools/disambiguate_repeats.xml --- a/test/functional/tools/disambiguate_repeats.xml +++ b/test/functional/tools/disambiguate_repeats.xml @@ -24,5 +24,22 @@ <output name="out_file1" file="simple_lines_interleaved.txt"/></test> + + <test> + <repeat name="queries"> + <param name="input" value="simple_line.txt"/> + </repeat> + <repeat name="queries"> + <param name="input" value="simple_line_alternative.txt"/> + </repeat> + <repeat name="more_queries"> + <param name="input" value="simple_line.txt"/> + </repeat> + <repeat name="more_queries"> + <param name="input" value="simple_line_alternative.txt"/> + </repeat> + <output name="out_file1" file="simple_lines_interleaved.txt"/> + </test> + </tests></tool> https://bitbucket.org/galaxy/galaxy-central/commits/d4f7fbbc3017/ Changeset: d4f7fbbc3017 User: jmchilton Date: 2013-11-22 02:17:13 Summary: Lazy load tool test cases. Running any tool test, will force everything in toolbox to parse out, but at least just starting Galaxy doesn't require parsing all test cases. Affected #: 1 file diff -r e86455cbd0a554613eee96852a1240205f65696b -r d4f7fbbc3017529eef45719fa8a6c54a12e6de78 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1219,14 +1219,9 @@ for key, value in uihints_elem.attrib.iteritems(): self.uihints[ key ] = value # Tests - tests_elem = root.find( "tests" ) - if tests_elem: - try: - self.tests = parse_tests_elem( self, tests_elem ) - except: - log.exception( "Failed to parse tool tests" ) - else: - self.tests = None + self.__tests_elem = root.find( "tests" ) + self.__tests_populated = False + # Requirements (dependencies) self.requirements = parse_requirements_from_xml( root ) # Determine if this tool can be used in workflows @@ -1237,6 +1232,21 @@ self.trackster_conf = TracksterConfig.parse( trackster_conf ) else: self.trackster_conf = None + + @property + def tests( self ): + if not self.__tests_populated: + tests_elem = self.__tests_elem + if tests_elem: + try: + self.__tests = parse_tests_elem( self, tests_elem ) + except: + log.exception( "Failed to parse tool tests" ) + else: + self.__tests = None + self.__tests_populated = True + return self.__tests + def parse_inputs( self, root ): """ Parse the "<inputs>" element and create appropriate `ToolParameter`s. https://bitbucket.org/galaxy/galaxy-central/commits/0923ef9b1edb/ Changeset: 0923ef9b1edb User: jmchilton Date: 2013-11-22 02:17:13 Summary: Small clean up of argument processing in functional_tests.py Affected #: 1 file diff -r d4f7fbbc3017529eef45719fa8a6c54a12e6de78 -r 0923ef9b1edba9d90de4b9d032f4818942489df1 scripts/functional_tests.py --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -176,12 +176,11 @@ tool_path = os.environ.get( 'GALAXY_TEST_TOOL_PATH', 'tools' ) if 'HTTP_ACCEPT_LANGUAGE' not in os.environ: os.environ[ 'HTTP_ACCEPT_LANGUAGE' ] = default_galaxy_locales - testing_migrated_tools = '-migrated' in sys.argv - testing_installed_tools = '-installed' in sys.argv + testing_migrated_tools = __check_arg( '-migrated' ) + testing_installed_tools = __check_arg( '-installed' ) datatypes_conf_override = None if testing_migrated_tools or testing_installed_tools: - sys.argv.pop() # Store a jsonified dictionary of tool_id : GALAXY_TEST_FILE_DIR pairs. galaxy_tool_shed_test_file = 'shed_tools_dict' # We need the upload tool for functional tests, so we'll create a temporary tool panel config that defines it. @@ -198,9 +197,8 @@ # Exclude all files except test_toolbox.py. ignore_files = ( re.compile( r'^test_[adghlmsu]*' ), re.compile( r'^test_ta*' ) ) else: - framework_test = '-framework' in sys.argv # Run through suite of tests testing framework. + framework_test = __check_arg( '-framework' ) # Run through suite of tests testing framework. if framework_test: - sys.argv.pop() framework_tool_dir = os.path.join('test', 'functional', 'tools') tool_conf = os.path.join( framework_tool_dir, 'samples_tool_conf.xml' ) datatypes_conf_override = os.path.join( framework_tool_dir, 'sample_datatypes_conf.xml' ) @@ -522,5 +520,19 @@ else: return 1 + +def __check_arg( name, param=False ): + try: + index = sys.argv.index( name ) + del sys.argv[ index ] + if param: + ret_val = sys.argv[ index ] + del sys.argv[ index ] + else: + ret_val = True + except ValueError: + ret_val = False + return ret_val + if __name__ == "__main__": sys.exit( main() ) https://bitbucket.org/galaxy/galaxy-central/commits/d5c0f73b3db4/ Changeset: d5c0f73b3db4 User: jmchilton Date: 2013-11-22 02:17:13 Summary: Refactor interactors out of test_toolbox into base. For potential reuse in other tests. Affected #: 2 files diff -r 0923ef9b1edba9d90de4b9d032f4818942489df1 -r d5c0f73b3db4cdb0b984b8a64da72dfe64ab6f0d test/base/interactor.py --- /dev/null +++ b/test/base/interactor.py @@ -0,0 +1,380 @@ +import os +from galaxy.tools.parameters import grouping +import galaxy.model +from galaxy.model.orm import and_, desc +from galaxy.model.mapping import context as sa_session +from simplejson import dumps, loads + + +def build_interactor( test_case, type="api" ): + interactor_class = GALAXY_INTERACTORS[ type ] + return interactor_class( test_case ) + + +class GalaxyInteractorApi( object ): + + def __init__( self, twill_test_case ): + self.twill_test_case = twill_test_case + self.api_url = "%s/api" % twill_test_case.url.rstrip("/") + self.api_key = self.__get_user_key( twill_test_case.user_api_key, twill_test_case.master_api_key ) + self.uploads = {} + + def verify_output( self, history_id, output_data, outfile, attributes, shed_tool_id, maxseconds ): + self.twill_test_case.wait_for( lambda: not self.__history_ready( history_id ), maxseconds=maxseconds) + hid = output_data.get( 'id' ) + fetcher = self.__dataset_fetcher( history_id ) + ## TODO: Twill version verifys dataset is 'ok' in here. + self.twill_test_case.verify_hid( outfile, hda_id=hid, attributes=attributes, dataset_fetcher=fetcher, shed_tool_id=shed_tool_id ) + metadata = attributes.get( 'metadata', {} ) + if metadata: + dataset = self.__get( "histories/%s/contents/%s" % ( history_id, hid ) ).json() + for key, value in metadata.iteritems(): + dataset_key = "metadata_%s" % key + try: + dataset_value = dataset.get( dataset_key, None ) + if dataset_value != value: + msg = "Dataset metadata verification for [%s] failed, expected [%s] but found [%s]." + msg_params = ( key, value, dataset_value ) + msg = msg % msg_params + raise Exception( msg ) + except KeyError: + msg = "Failed to verify dataset metadata, metadata key [%s] was not found." % key + raise Exception( msg ) + + def get_job_stream( self, history_id, output_data, stream ): + hid = output_data.get( 'id' ) + data = self.__get( "histories/%s/contents/%s/provenance" % (history_id, hid) ).json() + return data.get( stream, '' ) + + def new_history( self ): + history_json = self.__post( "histories", {"name": "test_history"} ).json() + return history_json[ 'id' ] + + def stage_data_async( self, test_data, history_id, shed_tool_id, async=True ): + fname = test_data[ 'fname' ] + tool_input = { + "file_type": test_data[ 'ftype' ], + "dbkey": test_data[ 'dbkey' ], + } + for elem in test_data.get('metadata', []): + tool_input["files_metadata|%s" % elem.get( 'name' )] = elem.get( 'value' ) + + composite_data = test_data[ 'composite_data' ] + if composite_data: + files = {} + for i, composite_file in enumerate( composite_data ): + file_name = self.twill_test_case.get_filename( composite_file.get( 'value' ), shed_tool_id=shed_tool_id ) + files["files_%s|file_data" % i] = open( file_name, 'rb' ) + tool_input.update({ + #"files_%d|NAME" % i: name, + "files_%d|type" % i: "upload_dataset", + ## TODO: + #"files_%d|space_to_tab" % i: composite_file.get( 'space_to_tab', False ) + }) + name = test_data[ 'name' ] + else: + file_name = self.twill_test_case.get_filename( fname, shed_tool_id=shed_tool_id ) + name = test_data.get( 'name', None ) + if not name: + name = os.path.basename( file_name ) + + tool_input.update({ + "files_0|NAME": name, + "files_0|type": "upload_dataset", + }) + files = { + "files_0|file_data": open( file_name, 'rb') + } + submit_response_object = self.__submit_tool( history_id, "upload1", tool_input, extra_data={"type": "upload_dataset"}, files=files ) + submit_response = submit_response_object.json() + try: + dataset = submit_response["outputs"][0] + except KeyError: + raise Exception(submit_response) + #raise Exception(str(dataset)) + hid = dataset['id'] + self.uploads[ os.path.basename(fname) ] = self.uploads[ fname ] = self.uploads[ name ] = {"src": "hda", "id": hid} + return self.__wait_for_history( history_id ) + + def run_tool( self, testdef, history_id ): + # We need to handle the case where we've uploaded a valid compressed file since the upload + # tool will have uncompressed it on the fly. + + inputs_tree = testdef.inputs.copy() + for key, value in inputs_tree.iteritems(): + values = [value] if not isinstance(value, list) else value + for value in values: + if value in self.uploads: + inputs_tree[ key ] = self.uploads[ value ] + + # # HACK: Flatten single-value lists. Required when using expand_grouping + for key, value in inputs_tree.iteritems(): + if isinstance(value, list) and len(value) == 1: + inputs_tree[key] = value[0] + + datasets = self.__submit_tool( history_id, tool_id=testdef.tool.id, tool_input=inputs_tree ) + datasets_object = datasets.json() + try: + return self.__dictify_outputs( datasets_object ) + except KeyError: + raise Exception( datasets_object[ 'message' ] ) + + def __dictify_outputs( self, datasets_object ): + ## Convert outputs list to a dictionary that can be accessed by + ## output_name so can be more flexiable about ordering of outputs + ## but also allows fallback to legacy access as list mode. + outputs_dict = {} + index = 0 + for output in datasets_object[ 'outputs' ]: + outputs_dict[ index ] = outputs_dict[ output.get("output_name") ] = output + index += 1 + return outputs_dict + + def output_hid( self, output_data ): + return output_data[ 'id' ] + + def delete_history( self, history ): + return None + + def __wait_for_history( self, history_id ): + def wait(): + while not self.__history_ready( history_id ): + pass + return wait + + def __history_ready( self, history_id ): + history_json = self.__get( "histories/%s" % history_id ).json() + state = history_json[ 'state' ] + if state == 'ok': + return True + elif state == 'error': + raise Exception("History in error state.") + return False + + def __submit_tool( self, history_id, tool_id, tool_input, extra_data={}, files=None ): + data = dict( + history_id=history_id, + tool_id=tool_id, + inputs=dumps( tool_input ), + **extra_data + ) + return self.__post( "tools", files=files, data=data ) + + def __get_user_key( self, user_key, admin_key ): + if user_key: + return user_key + all_users = self.__get( 'users', key=admin_key ).json() + try: + test_user = [ user for user in all_users if user["email"] == 'test(a)bx.psu.edu' ][0] + except IndexError: + data = dict( + email='test(a)bx.psu.edu', + password='testuser', + username='admin-user', + ) + test_user = self.__post( 'users', data, key=admin_key ).json() + return self.__post( "users/%s/api_key" % test_user['id'], key=admin_key ).json() + + def __dataset_fetcher( self, history_id ): + def fetcher( hda_id, base_name=None ): + url = "histories/%s/contents/%s/display?raw=true" % (history_id, hda_id) + if base_name: + url += "&filename=%s" % base_name + return self.__get( url ).content + + return fetcher + + def __post( self, path, data={}, files=None, key=None): + if not key: + key = self.api_key + data = data.copy() + data['key'] = key + return post_request( "%s/%s" % (self.api_url, path), data=data, files=files ) + + def __get( self, path, data={}, key=None ): + if not key: + key = self.api_key + data = data.copy() + data['key'] = key + if path.startswith("/api"): + path = path[ len("/api"): ] + url = "%s/%s" % (self.api_url, path) + return get_request( url, params=data ) + + +class GalaxyInteractorTwill( object ): + + def __init__( self, twill_test_case ): + self.twill_test_case = twill_test_case + + def verify_output( self, history, output_data, outfile, attributes, shed_tool_id, maxseconds ): + hid = output_data.get( 'hid' ) + self.twill_test_case.verify_dataset_correctness( outfile, hid=hid, attributes=attributes, shed_tool_id=shed_tool_id, maxseconds=maxseconds ) + + def get_job_stream( self, history_id, output_data, stream ): + return self.twill_test_case._get_job_stream_output( output_data.get( 'id' ), stream=stream, format=False ) + + def stage_data_async( self, test_data, history, shed_tool_id, async=True ): + name = test_data.get( 'name', None ) + if name: + async = False + self.twill_test_case.upload_file( test_data['fname'], + ftype=test_data['ftype'], + dbkey=test_data['dbkey'], + metadata=test_data['metadata'], + composite_data=test_data['composite_data'], + shed_tool_id=shed_tool_id, + wait=(not async) ) + if name: + hda_id = self.twill_test_case.get_history_as_data_list()[-1].get( 'id' ) + try: + self.twill_test_case.edit_hda_attribute_info( hda_id=str(hda_id), new_name=name ) + except: + print "### call to edit_hda failed for hda_id %s, new_name=%s" % (hda_id, name) + return lambda: self.twill_test_case.wait() + + def run_tool( self, testdef, test_history ): + # We need to handle the case where we've uploaded a valid compressed file since the upload + # tool will have uncompressed it on the fly. + + # Lose tons of information to accomodate legacy repeat handling. + all_inputs = {} + for key, value in testdef.inputs.iteritems(): + all_inputs[ key.split("|")[-1] ] = value + + # See if we have a grouping.Repeat element + repeat_name = None + for input_name, input_value in testdef.tool.inputs_by_page[0].items(): + if isinstance( input_value, grouping.Repeat ) and all_inputs.get( input_name, 1 ) not in [ 0, "0" ]: # default behavior is to test 1 repeat, for backwards compatibility + if not input_value.min: # If input_value.min == 1, the element is already on the page don't add new element. + repeat_name = input_name + break + + #check if we need to verify number of outputs created dynamically by tool + if testdef.tool.force_history_refresh: + job_finish_by_output_count = len( self.twill_test_case.get_history_as_data_list() ) + else: + job_finish_by_output_count = False + + inputs_tree = testdef.inputs + # # # HACK: Flatten single-value lists. Required when using expand_grouping + # #for key, value in inputs_tree.iteritems(): + # if isinstance(value, list) and len(value) == 1: + # inputs_tree[key] = value[0] + + # Strip out just a given page of inputs from inputs "tree". + def filter_page_inputs( n ): + page_input_keys = testdef.tool.inputs_by_page[ n ].keys() + return dict( [ (k, v) for k, v in inputs_tree.iteritems() if k.split("|")[0] or k.split("|")[0].resplit("_", 1)[0] in page_input_keys ] ) + + # Do the first page + page_inputs = filter_page_inputs( 0 ) + + # Run the tool + self.twill_test_case.run_tool( testdef.tool.id, repeat_name=repeat_name, **page_inputs ) + print "page_inputs (0)", page_inputs + # Do other pages if they exist + for i in range( 1, testdef.tool.npages ): + page_inputs = filter_page_inputs( i ) + self.twill_test_case.submit_form( **page_inputs ) + print "page_inputs (%i)" % i, page_inputs + + # Check the results ( handles single or multiple tool outputs ). Make sure to pass the correct hid. + # The output datasets from the tool should be in the same order as the testdef.outputs. + data_list = None + while data_list is None: + data_list = self.twill_test_case.get_history_as_data_list() + if job_finish_by_output_count and len( testdef.outputs ) > ( len( data_list ) - job_finish_by_output_count ): + data_list = None + return data_list + + def new_history( self ): + # Start with a new history + self.twill_test_case.logout() + self.twill_test_case.login( email='test(a)bx.psu.edu' ) + admin_user = sa_session.query( galaxy.model.User ).filter( galaxy.model.User.table.c.email == 'test(a)bx.psu.edu' ).one() + self.twill_test_case.new_history() + latest_history = sa_session.query( galaxy.model.History ) \ + .filter( and_( galaxy.model.History.table.c.deleted == False, + galaxy.model.History.table.c.user_id == admin_user.id ) ) \ + .order_by( desc( galaxy.model.History.table.c.create_time ) ) \ + .first() + assert latest_history is not None, "Problem retrieving latest_history from database" + if len( self.twill_test_case.get_history_as_data_list() ) > 0: + raise AssertionError("ToolTestCase.do_it failed") + return latest_history + + def delete_history( self, latest_history ): + self.twill_test_case.delete_history( id=self.twill_test_case.security.encode_id( latest_history.id ) ) + + def output_hid( self, output_data ): + return output_data.get( 'hid' ) + + +GALAXY_INTERACTORS = { + 'api': GalaxyInteractorApi, + 'twill': GalaxyInteractorTwill, +} + + +# Lets just try to use requests if it is available, but if not provide fallback +# on custom implementations of limited requests get/post functionality. +try: + from requests import get as get_request + from requests import post as post_request +except ImportError: + import urllib2 + import httplib + + class RequestsLikeResponse( object ): + + def __init__( self, content ): + self.content = content + + def json( self ): + return loads( self.content ) + + def get_request( url, params={} ): + argsep = '&' + if '?' not in url: + argsep = '?' + url = url + argsep + '&'.join( [ '%s=%s' % (k, v) for k, v in params.iteritems() ] ) + #req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' } ) + return RequestsLikeResponse(urllib2.urlopen( url ).read() ) + + def post_request( url, data, files ): + parsed_url = urllib2.urlparse.urlparse( url ) + return __post_multipart( host=parsed_url.netloc, selector=parsed_url.path, fields=data.iteritems(), files=(files or {}).iteritems() ) + + # http://stackoverflow.com/a/681182 + def __post_multipart(host, selector, fields, files): + content_type, body = __encode_multipart_formdata(fields, files) + h = httplib.HTTP(host) + h.putrequest('POST', selector) + h.putheader('content-type', content_type) + h.putheader('content-length', str(len(body))) + h.endheaders() + h.send(body) + errcode, errmsg, headers = h.getreply() + return RequestsLikeResponse(h.file.read()) + + def __encode_multipart_formdata(fields, files): + LIMIT = '----------lImIt_of_THE_fIle_eW_$' + CRLF = '\r\n' + L = [] + for (key, value) in fields: + L.append('--' + LIMIT) + L.append('Content-Disposition: form-data; name="%s"' % key) + L.append('') + L.append(value) + for (key, value) in files: + L.append('--' + LIMIT) + L.append('Content-Disposition: form-data; name="%s"; filename="%s";' % (key, key)) + L.append('Content-Type: application/octet-stream') + L.append('') + L.append(value.read()) + L.append('--' + LIMIT + '--') + L.append('') + body = CRLF.join(L) + content_type = 'multipart/form-data; boundary=%s' % LIMIT + return content_type, body diff -r 0923ef9b1edba9d90de4b9d032f4818942489df1 -r d5c0f73b3db4cdb0b984b8a64da72dfe64ab6f0d test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -1,14 +1,7 @@ +import new import sys -import new -import os -from galaxy.tools.parameters import grouping -from galaxy.util import string_as_bool from base.twilltestcase import TwillTestCase -import galaxy.model -from galaxy.model.orm import and_, desc -from galaxy.model.mapping import context as sa_session -from simplejson import dumps, loads - +from base.interactor import build_interactor import logging log = logging.getLogger( __name__ ) @@ -45,9 +38,7 @@ galaxy_interactor.delete_history( test_history ) def __galaxy_interactor( self, testdef ): - interactor_key = testdef.interactor - interactor_class = GALAXY_INTERACTORS[ interactor_key ] - return interactor_class( self ) + return build_interactor( self, testdef.interactor ) def __handle_test_def_errors(self, testdef): # If the test generation had an error, raise @@ -79,307 +70,6 @@ raise -class GalaxyInteractorApi( object ): - - def __init__( self, twill_test_case ): - self.twill_test_case = twill_test_case - self.api_url = "%s/api" % twill_test_case.url.rstrip("/") - self.api_key = self.__get_user_key( twill_test_case.user_api_key, twill_test_case.master_api_key ) - self.uploads = {} - - def verify_output( self, history_id, output_data, outfile, attributes, shed_tool_id, maxseconds ): - self.twill_test_case.wait_for( lambda: not self.__history_ready( history_id ), maxseconds=maxseconds) - hid = output_data.get( 'id' ) - fetcher = self.__dataset_fetcher( history_id ) - ## TODO: Twill version verifys dataset is 'ok' in here. - self.twill_test_case.verify_hid( outfile, hda_id=hid, attributes=attributes, dataset_fetcher=fetcher, shed_tool_id=shed_tool_id ) - metadata = attributes.get( 'metadata', {} ) - if metadata: - dataset = self.__get( "histories/%s/contents/%s" % ( history_id, hid ) ).json() - for key, value in metadata.iteritems(): - dataset_key = "metadata_%s" % key - try: - dataset_value = dataset.get( dataset_key, None ) - if dataset_value != value: - msg = "Dataset metadata verification for [%s] failed, expected [%s] but found [%s]." - msg_params = ( key, value, dataset_value ) - msg = msg % msg_params - raise Exception( msg ) - except KeyError: - msg = "Failed to verify dataset metadata, metadata key [%s] was not found." % key - raise Exception( msg ) - - def get_job_stream( self, history_id, output_data, stream ): - hid = output_data.get( 'id' ) - data = self.__get( "histories/%s/contents/%s/provenance" % (history_id, hid) ).json() - return data.get( stream, '' ) - - def new_history( self ): - history_json = self.__post( "histories", {"name": "test_history"} ).json() - return history_json[ 'id' ] - - def stage_data_async( self, test_data, history_id, shed_tool_id, async=True ): - fname = test_data[ 'fname' ] - tool_input = { - "file_type": test_data[ 'ftype' ], - "dbkey": test_data[ 'dbkey' ], - } - for elem in test_data.get('metadata', []): - tool_input["files_metadata|%s" % elem.get( 'name' )] = elem.get( 'value' ) - - composite_data = test_data[ 'composite_data' ] - if composite_data: - files = {} - for i, composite_file in enumerate( composite_data ): - file_name = self.twill_test_case.get_filename( composite_file.get( 'value' ), shed_tool_id=shed_tool_id ) - files["files_%s|file_data" % i] = open( file_name, 'rb' ) - tool_input.update({ - #"files_%d|NAME" % i: name, - "files_%d|type" % i: "upload_dataset", - ## TODO: - #"files_%d|space_to_tab" % i: composite_file.get( 'space_to_tab', False ) - }) - name = test_data[ 'name' ] - else: - file_name = self.twill_test_case.get_filename( fname, shed_tool_id=shed_tool_id ) - name = test_data.get( 'name', None ) - if not name: - name = os.path.basename( file_name ) - - tool_input.update({ - "files_0|NAME": name, - "files_0|type": "upload_dataset", - }) - files = { - "files_0|file_data": open( file_name, 'rb') - } - submit_response_object = self.__submit_tool( history_id, "upload1", tool_input, extra_data={"type": "upload_dataset"}, files=files ) - submit_response = submit_response_object.json() - try: - dataset = submit_response["outputs"][0] - except KeyError: - raise Exception(submit_response) - #raise Exception(str(dataset)) - hid = dataset['id'] - self.uploads[ os.path.basename(fname) ] = self.uploads[ fname ] = self.uploads[ name ] = {"src": "hda", "id": hid} - return self.__wait_for_history( history_id ) - - def run_tool( self, testdef, history_id ): - # We need to handle the case where we've uploaded a valid compressed file since the upload - # tool will have uncompressed it on the fly. - - inputs_tree = testdef.inputs.copy() - for key, value in inputs_tree.iteritems(): - values = [value] if not isinstance(value, list) else value - for value in values: - if value in self.uploads: - inputs_tree[ key ] = self.uploads[ value ] - - # # HACK: Flatten single-value lists. Required when using expand_grouping - for key, value in inputs_tree.iteritems(): - if isinstance(value, list) and len(value) == 1: - inputs_tree[key] = value[0] - - log.info( "Submiting tool with params %s" % inputs_tree ) - datasets = self.__submit_tool( history_id, tool_id=testdef.tool.id, tool_input=inputs_tree ) - datasets_object = datasets.json() - try: - return self.__dictify_outputs( datasets_object ) - except KeyError: - raise Exception( datasets_object[ 'message' ] ) - - def __dictify_outputs( self, datasets_object ): - ## Convert outputs list to a dictionary that can be accessed by - ## output_name so can be more flexiable about ordering of outputs - ## but also allows fallback to legacy access as list mode. - outputs_dict = {} - index = 0 - for output in datasets_object[ 'outputs' ]: - outputs_dict[ index ] = outputs_dict[ output.get("output_name") ] = output - index += 1 - return outputs_dict - - def output_hid( self, output_data ): - return output_data[ 'id' ] - - def delete_history( self, history ): - return None - - def __wait_for_history( self, history_id ): - def wait(): - while not self.__history_ready( history_id ): - pass - return wait - - def __history_ready( self, history_id ): - history_json = self.__get( "histories/%s" % history_id ).json() - state = history_json[ 'state' ] - if state == 'ok': - return True - elif state == 'error': - raise Exception("History in error state.") - return False - - def __submit_tool( self, history_id, tool_id, tool_input, extra_data={}, files=None ): - data = dict( - history_id=history_id, - tool_id=tool_id, - inputs=dumps( tool_input ), - **extra_data - ) - return self.__post( "tools", files=files, data=data ) - - def __get_user_key( self, user_key, admin_key ): - if user_key: - return user_key - all_users = self.__get( 'users', key=admin_key ).json() - try: - test_user = [ user for user in all_users if user["email"] == 'test(a)bx.psu.edu' ][0] - except IndexError: - data = dict( - email='test(a)bx.psu.edu', - password='testuser', - username='admin-user', - ) - test_user = self.__post( 'users', data, key=admin_key ).json() - return self.__post( "users/%s/api_key" % test_user['id'], key=admin_key ).json() - - def __dataset_fetcher( self, history_id ): - def fetcher( hda_id, base_name=None ): - url = "histories/%s/contents/%s/display?raw=true" % (history_id, hda_id) - if base_name: - url += "&filename=%s" % base_name - return self.__get( url ).content - - return fetcher - - def __post( self, path, data={}, files=None, key=None): - if not key: - key = self.api_key - data = data.copy() - data['key'] = key - return post_request( "%s/%s" % (self.api_url, path), data=data, files=files ) - - def __get( self, path, data={}, key=None ): - if not key: - key = self.api_key - data = data.copy() - data['key'] = key - if path.startswith("/api"): - path = path[ len("/api"): ] - url = "%s/%s" % (self.api_url, path) - return get_request( url, params=data ) - - -class GalaxyInteractorTwill( object ): - - def __init__( self, twill_test_case ): - self.twill_test_case = twill_test_case - - def verify_output( self, history, output_data, outfile, attributes, shed_tool_id, maxseconds ): - hid = output_data.get( 'hid' ) - self.twill_test_case.verify_dataset_correctness( outfile, hid=hid, attributes=attributes, shed_tool_id=shed_tool_id, maxseconds=maxseconds ) - - def get_job_stream( self, history_id, output_data, stream ): - return self.twill_test_case._get_job_stream_output( output_data.get( 'id' ), stream=stream, format=False ) - - def stage_data_async( self, test_data, history, shed_tool_id, async=True ): - name = test_data.get( 'name', None ) - if name: - async = False - self.twill_test_case.upload_file( test_data['fname'], - ftype=test_data['ftype'], - dbkey=test_data['dbkey'], - metadata=test_data['metadata'], - composite_data=test_data['composite_data'], - shed_tool_id=shed_tool_id, - wait=(not async) ) - if name: - hda_id = self.twill_test_case.get_history_as_data_list()[-1].get( 'id' ) - try: - self.twill_test_case.edit_hda_attribute_info( hda_id=str(hda_id), new_name=name ) - except: - print "### call to edit_hda failed for hda_id %s, new_name=%s" % (hda_id, name) - return lambda: self.twill_test_case.wait() - - def run_tool( self, testdef, test_history ): - # We need to handle the case where we've uploaded a valid compressed file since the upload - # tool will have uncompressed it on the fly. - - # Lose tons of information to accomodate legacy repeat handling. - all_inputs = {} - for key, value in testdef.inputs.iteritems(): - all_inputs[ key.split("|")[-1] ] = value - - # See if we have a grouping.Repeat element - repeat_name = None - for input_name, input_value in testdef.tool.inputs_by_page[0].items(): - if isinstance( input_value, grouping.Repeat ) and all_inputs.get( input_name, 1 ) not in [ 0, "0" ]: # default behavior is to test 1 repeat, for backwards compatibility - if not input_value.min: # If input_value.min == 1, the element is already on the page don't add new element. - repeat_name = input_name - break - - #check if we need to verify number of outputs created dynamically by tool - if testdef.tool.force_history_refresh: - job_finish_by_output_count = len( self.twill_test_case.get_history_as_data_list() ) - else: - job_finish_by_output_count = False - - inputs_tree = testdef.inputs - # # # HACK: Flatten single-value lists. Required when using expand_grouping - # #for key, value in inputs_tree.iteritems(): - # if isinstance(value, list) and len(value) == 1: - # inputs_tree[key] = value[0] - - # Strip out just a given page of inputs from inputs "tree". - def filter_page_inputs( n ): - page_input_keys = testdef.tool.inputs_by_page[ n ].keys() - return dict( [ (k, v) for k, v in inputs_tree.iteritems() if k.split("|")[0] or k.split("|")[0].resplit("_", 1)[0] in page_input_keys ] ) - - # Do the first page - page_inputs = filter_page_inputs( 0 ) - - # Run the tool - self.twill_test_case.run_tool( testdef.tool.id, repeat_name=repeat_name, **page_inputs ) - print "page_inputs (0)", page_inputs - # Do other pages if they exist - for i in range( 1, testdef.tool.npages ): - page_inputs = filter_page_inputs( i ) - self.twill_test_case.submit_form( **page_inputs ) - print "page_inputs (%i)" % i, page_inputs - - # Check the results ( handles single or multiple tool outputs ). Make sure to pass the correct hid. - # The output datasets from the tool should be in the same order as the testdef.outputs. - data_list = None - while data_list is None: - data_list = self.twill_test_case.get_history_as_data_list() - if job_finish_by_output_count and len( testdef.outputs ) > ( len( data_list ) - job_finish_by_output_count ): - data_list = None - return data_list - - def new_history( self ): - # Start with a new history - self.twill_test_case.logout() - self.twill_test_case.login( email='test(a)bx.psu.edu' ) - admin_user = sa_session.query( galaxy.model.User ).filter( galaxy.model.User.table.c.email == 'test(a)bx.psu.edu' ).one() - self.twill_test_case.new_history() - latest_history = sa_session.query( galaxy.model.History ) \ - .filter( and_( galaxy.model.History.table.c.deleted == False, - galaxy.model.History.table.c.user_id == admin_user.id ) ) \ - .order_by( desc( galaxy.model.History.table.c.create_time ) ) \ - .first() - assert latest_history is not None, "Problem retrieving latest_history from database" - if len( self.twill_test_case.get_history_as_data_list() ) > 0: - raise AssertionError("ToolTestCase.do_it failed") - return latest_history - - def delete_history( self, latest_history ): - self.twill_test_case.delete_history( id=self.twill_test_case.security.encode_id( latest_history.id ) ) - - def output_hid( self, output_data ): - return output_data.get( 'hid' ) - - def build_tests( testing_shed_tools=False, master_api_key=None, user_api_key=None ): """ If the module level variable `toolbox` is set, generate `ToolTestCase` @@ -421,72 +111,3 @@ # from baseclasses (which should be a tuple of classes) and with namespace dict. new_class_obj = new.classobj( name, baseclasses, namespace ) G[ name ] = new_class_obj - - -GALAXY_INTERACTORS = { - 'api': GalaxyInteractorApi, - 'twill': GalaxyInteractorTwill, -} - - -# Lets just try to use requests if it is available, but if not provide fallback -# on custom implementations of limited requests get/post functionality. -try: - from requests import get as get_request - from requests import post as post_request -except ImportError: - import urllib2 - import httplib - - class RequestsLikeResponse( object ): - - def __init__( self, content ): - self.content = content - - def json( self ): - return loads( self.content ) - - def get_request( url, params={} ): - argsep = '&' - if '?' not in url: - argsep = '?' - url = url + argsep + '&'.join( [ '%s=%s' % (k, v) for k, v in params.iteritems() ] ) - #req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' } ) - return RequestsLikeResponse(urllib2.urlopen( url ).read() ) - - def post_request( url, data, files ): - parsed_url = urllib2.urlparse.urlparse( url ) - return __post_multipart( host=parsed_url.netloc, selector=parsed_url.path, fields=data.iteritems(), files=(files or {}).iteritems() ) - - # http://stackoverflow.com/a/681182 - def __post_multipart(host, selector, fields, files): - content_type, body = __encode_multipart_formdata(fields, files) - h = httplib.HTTP(host) - h.putrequest('POST', selector) - h.putheader('content-type', content_type) - h.putheader('content-length', str(len(body))) - h.endheaders() - h.send(body) - errcode, errmsg, headers = h.getreply() - return RequestsLikeResponse(h.file.read()) - - def __encode_multipart_formdata(fields, files): - LIMIT = '----------lImIt_of_THE_fIle_eW_$' - CRLF = '\r\n' - L = [] - for (key, value) in fields: - L.append('--' + LIMIT) - L.append('Content-Disposition: form-data; name="%s"' % key) - L.append('') - L.append(value) - for (key, value) in files: - L.append('--' + LIMIT) - L.append('Content-Disposition: form-data; name="%s"; filename="%s";' % (key, key)) - L.append('Content-Type: application/octet-stream') - L.append('') - L.append(value.read()) - L.append('--' + LIMIT + '--') - L.append('') - body = CRLF.join(L) - content_type = 'multipart/form-data; boundary=%s' % LIMIT - return content_type, body https://bitbucket.org/galaxy/galaxy-central/commits/f4fd082fdcd2/ Changeset: f4fd082fdcd2 User: jmchilton Date: 2013-11-22 02:17:13 Summary: Simplify test composite history item naming using UUIDs. It needs to be unique, so lets use uuids instead of a composite name counter to make the code more portable. Affected #: 1 file diff -r d5c0f73b3db4cdb0b984b8a64da72dfe64ab6f0d -r f4fd082fdcd25f06b2f0dfcfa35c1c3ed3046732 lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -1,5 +1,6 @@ import os import os.path +import uuid from parameters import basic from parameters import grouping from galaxy.util import string_as_bool @@ -139,10 +140,6 @@ expand_input_elems( test_elem ) def __parse_inputs_elems( self, test_elem, i ): - # Composite datasets need a unique name: each test occurs in a fresh - # history, but we'll keep it unique per set of tests - use i (test #) - # and composite_data_names_counter (instance per test #) - composite_data_names_counter = 0 raw_inputs = [] for param_elem in test_elem.findall( "param" ): attrib = dict( param_elem.attrib ) @@ -168,10 +165,9 @@ attrib['composite_data'].append( child ) if composite_data_name is None: # Generate a unique name; each test uses a - # fresh history - composite_data_name = '_COMPOSITE_RENAMED_t%i_d%i' \ - % ( i, composite_data_names_counter ) - composite_data_names_counter += 1 + # fresh history. + composite_data_name = '_COMPOSITE_RENAMED_t%d_%s' \ + % ( i, uuid.uuid1().hex ) elif child.tag == 'metadata': attrib['metadata'].append( child ) elif child.tag == 'metadata': https://bitbucket.org/galaxy/galaxy-central/commits/b51a17a1d3ce/ Changeset: b51a17a1d3ce User: jmchilton Date: 2013-11-22 02:17:13 Summary: Refactor tools test param parsing for reuse (in particular w/workflow tests). They will be used in a slightly different context, but will need to describe inputs and outputs so lets reuse this code if possible. None of these methods used tool or self anyway, so they are probably more appropriate outside of ToolTestBuilder and ToolTestBuilder is now much more manageable. Affected #: 1 file diff -r f4fd082fdcd25f06b2f0dfcfa35c1c3ed3046732 -r b51a17a1d3cec70820ec921b66e477d6d8f6bc60 lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -54,26 +54,7 @@ """ Iterator over metadata representing the required files for upload. """ - for fname, extra in self.required_files: - data_dict = dict( - fname=fname, - metadata=extra.get( 'metadata', [] ), - composite_data=extra.get( 'composite_data', [] ), - ftype=extra.get( 'ftype', DEFAULT_FTYPE ), - dbkey=extra.get( 'dbkey', DEFAULT_DBKEY ), - ) - edit_attributes = extra.get( 'edit_attributes', [] ) - - #currently only renaming is supported - for edit_att in edit_attributes: - if edit_att.get( 'type', None ) == 'name': - new_name = edit_att.get( 'value', None ) - assert new_name, 'You must supply the new dataset name as the value tag of the edit_attributes tag' - data_dict['name'] = new_name - else: - raise Exception( 'edit_attributes type (%s) is unimplemented' % edit_att.get( 'type', None ) ) - - yield data_dict + return test_data_iter( self.required_files ) def __matching_case_for_value( self, cond, declared_value ): test_param = cond.test_param @@ -131,7 +112,7 @@ self.__preprocess_input_elems( test_elem ) self.__parse_inputs_elems( test_elem, i ) - self.__parse_output_elems( test_elem ) + self.outputs = parse_output_elems( test_elem ) except Exception, e: self.error = True self.exception = e @@ -142,44 +123,7 @@ def __parse_inputs_elems( self, test_elem, i ): raw_inputs = [] for param_elem in test_elem.findall( "param" ): - attrib = dict( param_elem.attrib ) - if 'values' in attrib: - value = attrib[ 'values' ].split( ',' ) - elif 'value' in attrib: - value = attrib['value'] - else: - value = None - attrib['children'] = list( param_elem.getchildren() ) - if attrib['children']: - # At this time, we can assume having children only - # occurs on DataToolParameter test items but this could - # change and would cause the below parsing to change - # based upon differences in children items - attrib['metadata'] = [] - attrib['composite_data'] = [] - attrib['edit_attributes'] = [] - # Composite datasets need to be renamed uniquely - composite_data_name = None - for child in attrib['children']: - if child.tag == 'composite_data': - attrib['composite_data'].append( child ) - if composite_data_name is None: - # Generate a unique name; each test uses a - # fresh history. - composite_data_name = '_COMPOSITE_RENAMED_t%d_%s' \ - % ( i, uuid.uuid1().hex ) - elif child.tag == 'metadata': - attrib['metadata'].append( child ) - elif child.tag == 'metadata': - attrib['metadata'].append( child ) - elif child.tag == 'edit_attributes': - attrib['edit_attributes'].append( child ) - if composite_data_name: - # Composite datasets need implicit renaming; - # inserted at front of list so explicit declarations - # take precedence - attrib['edit_attributes'].insert( 0, { 'type': 'name', 'value': composite_data_name } ) - name = attrib.pop( 'name' ) + name, value, attrib = parse_param_elem( param_elem, i ) raw_inputs.append( ( name, value, attrib ) ) self.inputs = self.__process_raw_inputs( self.tool.inputs, raw_inputs ) @@ -229,98 +173,174 @@ expanded_inputs[ context.for_state() ] = processed_value return expanded_inputs - def __parse_output_elems( self, test_elem ): - for output_elem in test_elem.findall( "output" ): - attrib = dict( output_elem.attrib ) - name = attrib.pop( 'name', None ) - if name is None: - raise Exception( "Test output does not have a 'name'" ) - - assert_list = self.__parse_assert_list( output_elem ) - file = attrib.pop( 'file', None ) - # File no longer required if an list of assertions was present. - attributes = {} - # Method of comparison - attributes['compare'] = attrib.pop( 'compare', 'diff' ).lower() - # Number of lines to allow to vary in logs (for dates, etc) - attributes['lines_diff'] = int( attrib.pop( 'lines_diff', '0' ) ) - # Allow a file size to vary if sim_size compare - attributes['delta'] = int( attrib.pop( 'delta', '10000' ) ) - attributes['sort'] = string_as_bool( attrib.pop( 'sort', False ) ) - extra_files = [] - if 'ftype' in attrib: - attributes['ftype'] = attrib['ftype'] - for extra in output_elem.findall( 'extra_files' ): - extra_files.append( self.__parse_extra_files_elem( extra ) ) - metadata = {} - for metadata_elem in output_elem.findall( 'metadata' ): - metadata[ metadata_elem.get('name') ] = metadata_elem.get( 'value' ) - if not (assert_list or file or extra_files or metadata): - raise Exception( "Test output defines not checks (e.g. must have a 'file' check against, assertions to check, etc...)") - attributes['assert_list'] = assert_list - attributes['extra_files'] = extra_files - attributes['metadata'] = metadata - self.__add_output( name, file, attributes ) - - def __parse_assert_list( self, output_elem ): - assert_elem = output_elem.find("assert_contents") - assert_list = None - - # Trying to keep testing patch as localized as - # possible, this function should be relocated - # somewhere more conventional. - def convert_elem(elem): - """ Converts and XML element to a dictionary format, used by assertion checking code. """ - tag = elem.tag - attributes = dict( elem.attrib ) - child_elems = list( elem.getchildren() ) - converted_children = [] - for child_elem in child_elems: - converted_children.append( convert_elem(child_elem) ) - return {"tag": tag, "attributes": attributes, "children": converted_children} - if assert_elem is not None: - assert_list = [] - for assert_child in list(assert_elem): - assert_list.append(convert_elem(assert_child)) - - return assert_list - - def __parse_extra_files_elem( self, extra ): - # File or directory, when directory, compare basename - # by basename - extra_type = extra.get( 'type', 'file' ) - extra_name = extra.get( 'name', None ) - assert extra_type == 'directory' or extra_name is not None, \ - 'extra_files type (%s) requires a name attribute' % extra_type - extra_value = extra.get( 'value', None ) - assert extra_value is not None, 'extra_files requires a value attribute' - extra_attributes = {} - extra_attributes['compare'] = extra.get( 'compare', 'diff' ).lower() - extra_attributes['delta'] = extra.get( 'delta', '0' ) - extra_attributes['lines_diff'] = int( extra.get( 'lines_diff', '0' ) ) - extra_attributes['sort'] = string_as_bool( extra.get( 'sort', False ) ) - return extra_type, extra_value, extra_name, extra_attributes - - def __add_output( self, name, file, extra ): - self.outputs.append( ( name, file, extra ) ) - def __add_uploaded_dataset( self, name, value, extra, input_parameter ): if value is None: assert input_parameter.optional, '%s is not optional. You must provide a valid filename.' % name return value - if ( value, extra ) not in self.required_files: - self.required_files.append( ( value, extra ) ) # these files will be uploaded - name_change = [ att for att in extra.get( 'edit_attributes', [] ) if att.get( 'type' ) == 'name' ] - if name_change: - name_change = name_change[-1].get( 'value' ) # only the last name change really matters - value = name_change # change value for select to renamed uploaded file for e.g. composite dataset - else: - for end in [ '.zip', '.gz' ]: - if value.endswith( end ): - value = value[ :-len( end ) ] - break - value = os.path.basename( value ) # if uploading a file in a path other than root of test-data - return value + return require_file( name, value, extra, self.required_files ) + + +def test_data_iter( required_files ): + for fname, extra in required_files: + data_dict = dict( + fname=fname, + metadata=extra.get( 'metadata', [] ), + composite_data=extra.get( 'composite_data', [] ), + ftype=extra.get( 'ftype', DEFAULT_FTYPE ), + dbkey=extra.get( 'dbkey', DEFAULT_DBKEY ), + ) + edit_attributes = extra.get( 'edit_attributes', [] ) + + #currently only renaming is supported + for edit_att in edit_attributes: + if edit_att.get( 'type', None ) == 'name': + new_name = edit_att.get( 'value', None ) + assert new_name, 'You must supply the new dataset name as the value tag of the edit_attributes tag' + data_dict['name'] = new_name + else: + raise Exception( 'edit_attributes type (%s) is unimplemented' % edit_att.get( 'type', None ) ) + + yield data_dict + + +def require_file( name, value, extra, required_files ): + if ( value, extra ) not in required_files: + required_files.append( ( value, extra ) ) # these files will be uploaded + name_change = [ att for att in extra.get( 'edit_attributes', [] ) if att.get( 'type' ) == 'name' ] + if name_change: + name_change = name_change[-1].get( 'value' ) # only the last name change really matters + value = name_change # change value for select to renamed uploaded file for e.g. composite dataset + else: + for end in [ '.zip', '.gz' ]: + if value.endswith( end ): + value = value[ :-len( end ) ] + break + value = os.path.basename( value ) # if uploading a file in a path other than root of test-data + return value + + +def parse_param_elem( param_elem, i=0 ): + attrib = dict( param_elem.attrib ) + if 'values' in attrib: + value = attrib[ 'values' ].split( ',' ) + elif 'value' in attrib: + value = attrib['value'] + else: + value = None + attrib['children'] = list( param_elem.getchildren() ) + if attrib['children']: + # At this time, we can assume having children only + # occurs on DataToolParameter test items but this could + # change and would cause the below parsing to change + # based upon differences in children items + attrib['metadata'] = [] + attrib['composite_data'] = [] + attrib['edit_attributes'] = [] + # Composite datasets need to be renamed uniquely + composite_data_name = None + for child in attrib['children']: + if child.tag == 'composite_data': + attrib['composite_data'].append( child ) + if composite_data_name is None: + # Generate a unique name; each test uses a + # fresh history. + composite_data_name = '_COMPOSITE_RENAMED_t%d_%s' \ + % ( i, uuid.uuid1().hex ) + elif child.tag == 'metadata': + attrib['metadata'].append( child ) + elif child.tag == 'metadata': + attrib['metadata'].append( child ) + elif child.tag == 'edit_attributes': + attrib['edit_attributes'].append( child ) + if composite_data_name: + # Composite datasets need implicit renaming; + # inserted at front of list so explicit declarations + # take precedence + attrib['edit_attributes'].insert( 0, { 'type': 'name', 'value': composite_data_name } ) + name = attrib.pop( 'name' ) + return ( name, value, attrib ) + + +def parse_output_elems( test_elem ): + outputs = [] + for output_elem in test_elem.findall( "output" ): + name, file, attributes = __parse_output_elem( output_elem ) + outputs.append( ( name, file, attributes ) ) + return outputs + + +def __parse_output_elem( output_elem ): + attrib = dict( output_elem.attrib ) + name = attrib.pop( 'name', None ) + if name is None: + raise Exception( "Test output does not have a 'name'" ) + + assert_list = __parse_assert_list( output_elem ) + file = attrib.pop( 'file', None ) + # File no longer required if an list of assertions was present. + attributes = {} + # Method of comparison + attributes['compare'] = attrib.pop( 'compare', 'diff' ).lower() + # Number of lines to allow to vary in logs (for dates, etc) + attributes['lines_diff'] = int( attrib.pop( 'lines_diff', '0' ) ) + # Allow a file size to vary if sim_size compare + attributes['delta'] = int( attrib.pop( 'delta', '10000' ) ) + attributes['sort'] = string_as_bool( attrib.pop( 'sort', False ) ) + extra_files = [] + if 'ftype' in attrib: + attributes['ftype'] = attrib['ftype'] + for extra in output_elem.findall( 'extra_files' ): + extra_files.append( __parse_extra_files_elem( extra ) ) + metadata = {} + for metadata_elem in output_elem.findall( 'metadata' ): + metadata[ metadata_elem.get('name') ] = metadata_elem.get( 'value' ) + if not (assert_list or file or extra_files or metadata): + raise Exception( "Test output defines not checks (e.g. must have a 'file' check against, assertions to check, etc...)") + attributes['assert_list'] = assert_list + attributes['extra_files'] = extra_files + attributes['metadata'] = metadata + return name, file, attributes + + +def __parse_assert_list( output_elem ): + assert_elem = output_elem.find("assert_contents") + assert_list = None + + # Trying to keep testing patch as localized as + # possible, this function should be relocated + # somewhere more conventional. + def convert_elem(elem): + """ Converts and XML element to a dictionary format, used by assertion checking code. """ + tag = elem.tag + attributes = dict( elem.attrib ) + child_elems = list( elem.getchildren() ) + converted_children = [] + for child_elem in child_elems: + converted_children.append( convert_elem(child_elem) ) + return {"tag": tag, "attributes": attributes, "children": converted_children} + if assert_elem is not None: + assert_list = [] + for assert_child in list(assert_elem): + assert_list.append(convert_elem(assert_child)) + + return assert_list + + +def __parse_extra_files_elem( extra ): + # File or directory, when directory, compare basename + # by basename + extra_type = extra.get( 'type', 'file' ) + extra_name = extra.get( 'name', None ) + assert extra_type == 'directory' or extra_name is not None, \ + 'extra_files type (%s) requires a name attribute' % extra_type + extra_value = extra.get( 'value', None ) + assert extra_value is not None, 'extra_files requires a value attribute' + extra_attributes = {} + extra_attributes['compare'] = extra.get( 'compare', 'diff' ).lower() + extra_attributes['delta'] = extra.get( 'delta', '0' ) + extra_attributes['lines_diff'] = int( extra.get( 'lines_diff', '0' ) ) + extra_attributes['sort'] = string_as_bool( extra.get( 'sort', False ) ) + return extra_type, extra_value, extra_name, extra_attributes class ParamContext(object): https://bitbucket.org/galaxy/galaxy-central/commits/6fadd2ce40e1/ Changeset: 6fadd2ce40e1 User: jmchilton Date: 2013-11-22 02:17:13 Summary: Further refactoring for iteractors for reuse. Affected #: 2 files diff -r b51a17a1d3cec70820ec921b66e477d6d8f6bc60 -r 6fadd2ce40e107fdc9f754cb0b45f1a1263d5921 test/base/interactor.py --- a/test/base/interactor.py +++ b/test/base/interactor.py @@ -5,12 +5,25 @@ from galaxy.model.mapping import context as sa_session from simplejson import dumps, loads +from logging import getLogger +log = getLogger( __name__ ) + def build_interactor( test_case, type="api" ): interactor_class = GALAXY_INTERACTORS[ type ] return interactor_class( test_case ) +def stage_data_in_history( galaxy_interactor, all_test_data, history, shed_tool_id=None ): + # Upload any needed files + upload_waits = [] + + for test_data in all_test_data: + upload_waits.append( galaxy_interactor.stage_data_async( test_data, history, shed_tool_id ) ) + for upload_wait in upload_waits: + upload_wait() + + class GalaxyInteractorApi( object ): def __init__( self, twill_test_case ): @@ -20,14 +33,14 @@ self.uploads = {} def verify_output( self, history_id, output_data, outfile, attributes, shed_tool_id, maxseconds ): - self.twill_test_case.wait_for( lambda: not self.__history_ready( history_id ), maxseconds=maxseconds) - hid = output_data.get( 'id' ) + self.wait_for_history( history_id, maxseconds ) + hid = self.__output_id( output_data ) fetcher = self.__dataset_fetcher( history_id ) ## TODO: Twill version verifys dataset is 'ok' in here. self.twill_test_case.verify_hid( outfile, hda_id=hid, attributes=attributes, dataset_fetcher=fetcher, shed_tool_id=shed_tool_id ) metadata = attributes.get( 'metadata', {} ) if metadata: - dataset = self.__get( "histories/%s/contents/%s" % ( history_id, hid ) ).json() + dataset = self._get( "histories/%s/contents/%s" % ( history_id, hid ) ).json() for key, value in metadata.iteritems(): dataset_key = "metadata_%s" % key try: @@ -41,15 +54,27 @@ msg = "Failed to verify dataset metadata, metadata key [%s] was not found." % key raise Exception( msg ) + def wait_for_history( self, history_id, maxseconds ): + self.twill_test_case.wait_for( lambda: not self.__history_ready( history_id ), maxseconds=maxseconds) + def get_job_stream( self, history_id, output_data, stream ): - hid = output_data.get( 'id' ) - data = self.__get( "histories/%s/contents/%s/provenance" % (history_id, hid) ).json() + hid = self.__output_id( output_data ) + data = self._get( "histories/%s/contents/%s/provenance" % (history_id, hid) ).json() return data.get( stream, '' ) def new_history( self ): - history_json = self.__post( "histories", {"name": "test_history"} ).json() + history_json = self._post( "histories", {"name": "test_history"} ).json() return history_json[ 'id' ] + def __output_id( self, output_data ): + # Allow data structure coming out of tools API - {id: <id>, output_name: <name>, etc...} + # or simple id as comes out of workflow API. + try: + output_id = output_data.get( 'id' ) + except AttributeError: + output_id = output_data + return output_id + def stage_data_async( self, test_data, history_id, shed_tool_id, async=True ): fname = test_data[ 'fname' ] tool_input = { @@ -143,12 +168,15 @@ return wait def __history_ready( self, history_id ): - history_json = self.__get( "histories/%s" % history_id ).json() + history_json = self._get( "histories/%s" % history_id ).json() state = history_json[ 'state' ] - if state == 'ok': + return self._state_ready( state, error_msg="History in error state." ) + + def _state_ready( self, state_str, error_msg ): + if state_str == 'ok': return True - elif state == 'error': - raise Exception("History in error state.") + elif state_str == 'error': + raise Exception( error_msg ) return False def __submit_tool( self, history_id, tool_id, tool_input, extra_data={}, files=None ): @@ -158,12 +186,12 @@ inputs=dumps( tool_input ), **extra_data ) - return self.__post( "tools", files=files, data=data ) + return self._post( "tools", files=files, data=data ) def __get_user_key( self, user_key, admin_key ): if user_key: return user_key - all_users = self.__get( 'users', key=admin_key ).json() + all_users = self._get( 'users', key=admin_key ).json() try: test_user = [ user for user in all_users if user["email"] == 'test(a)bx.psu.edu' ][0] except IndexError: @@ -172,26 +200,26 @@ password='testuser', username='admin-user', ) - test_user = self.__post( 'users', data, key=admin_key ).json() - return self.__post( "users/%s/api_key" % test_user['id'], key=admin_key ).json() + test_user = self._post( 'users', data, key=admin_key ).json() + return self._post( "users/%s/api_key" % test_user['id'], key=admin_key ).json() def __dataset_fetcher( self, history_id ): def fetcher( hda_id, base_name=None ): url = "histories/%s/contents/%s/display?raw=true" % (history_id, hda_id) if base_name: url += "&filename=%s" % base_name - return self.__get( url ).content + return self._get( url ).content return fetcher - def __post( self, path, data={}, files=None, key=None): + def _post( self, path, data={}, files=None, key=None): if not key: key = self.api_key data = data.copy() data['key'] = key return post_request( "%s/%s" % (self.api_url, path), data=data, files=files ) - def __get( self, path, data={}, key=None ): + def _get( self, path, data={}, key=None ): if not key: key = self.api_key data = data.copy() diff -r b51a17a1d3cec70820ec921b66e477d6d8f6bc60 -r 6fadd2ce40e107fdc9f754cb0b45f1a1263d5921 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -1,7 +1,7 @@ import new import sys from base.twilltestcase import TwillTestCase -from base.interactor import build_interactor +from base.interactor import build_interactor, stage_data_in_history import logging log = logging.getLogger( __name__ ) @@ -23,12 +23,7 @@ test_history = galaxy_interactor.new_history() - # Upload any needed files - upload_waits = [] - for test_data in testdef.test_data(): - upload_waits.append( galaxy_interactor.stage_data_async( test_data, test_history, shed_tool_id ) ) - for upload_wait in upload_waits: - upload_wait() + stage_data_in_history( galaxy_interactor, testdef.test_data(), test_history, shed_tool_id ) data_list = galaxy_interactor.run_tool( testdef, test_history ) self.assertTrue( data_list ) https://bitbucket.org/galaxy/galaxy-central/commits/7c5305e54c03/ Changeset: 7c5305e54c03 User: jmchilton Date: 2013-11-22 02:17:13 Summary: Functional testing of workflows (for Galaxy). ... tool shed integration will be more challenging, but this can be used for directly configured Galaxy tools and outlines a syntax that could be shared with a tool shed driven approach. The one tricky part is how to match workflow outputs to things to check. Right now it is based on the index of the output across the workflow. This is both difficult to determine and very brittle to workflow modifications - how to proceed - require annontation string to be setup? Modify workflow data model to all assigning names to outputs the way inputs havenames? At any rate this current syntax should be considered beta and may change - if it does Galaxy will not continue to support this syntax. To run the sample test execute the following test command: sh run_functional_tests.sh -workflow test-data/workflows/1.xml Affected #: 8 files diff -r 6fadd2ce40e107fdc9f754cb0b45f1a1263d5921 -r 7c5305e54c03373b9922d60e2c75e90607c61c6f run_functional_tests.sh --- a/run_functional_tests.sh +++ b/run_functional_tests.sh @@ -13,6 +13,7 @@ echo "'run_functional_tests.sh -list' for listing all the tool ids" echo "'run_functional_tests.sh -toolshed' for running all the test scripts in the ./test/tool_shed/functional directory" echo "'run_functional_tests.sh -toolshed testscriptname' for running one test script named testscriptname in the .test/tool_shed/functional directory" + echo "'run_functional_tests.sh -workflow test.xml' for running a workflow test case as defined by supplied workflow xml test file" echo "'run_functional_tests.sh -framework' for running through example tool tests testing framework features in test/functional/tools" echo "'run_functional_tests.sh -framework -id toolid' for testing one framework tool (in test/functional/tools/) with id 'toolid'" elif [ $1 = '-id' ]; then @@ -48,6 +49,8 @@ else python ./test/tool_shed/functional_tests.py -v --with-nosehtml --html-report-file ./test/tool_shed/run_functional_tests.html $2 fi +elif [ $1 = '-workflow' ]; then + python ./scripts/functional_tests.py -v functional.test_workflow:WorkflowTestCase --with-nosehtml --html-report-file ./test/tool_shed/run_functional_tests.html -workflow $2 elif [ $1 = '-framework' ]; then if [ ! $2 ]; then python ./scripts/functional_tests.py -v functional.test_toolbox --with-nosehtml --html-report-file run_functional_tests.html -framework diff -r 6fadd2ce40e107fdc9f754cb0b45f1a1263d5921 -r 7c5305e54c03373b9922d60e2c75e90607c61c6f scripts/functional_tests.py --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -420,7 +420,7 @@ new_path = [ os.path.join( cwd, "test" ) ] new_path.extend( sys.path[1:] ) sys.path = new_path - import functional.test_toolbox + # ---- Find tests --------------------------------------------------------- if galaxy_test_proxy_port: log.info( "Functional tests will be run against %s:%s" % ( galaxy_test_host, galaxy_test_proxy_port ) ) @@ -436,6 +436,13 @@ os.environ[ 'GALAXY_TEST_HOST' ] = galaxy_test_host def _run_functional_test( testing_shed_tools=None ): + workflow_test = __check_arg( '-workflow', param=True ) + if workflow_test: + import functional.test_workflow + functional.test_workflow.WorkflowTestCase.workflow_test_file = workflow_test + functional.test_workflow.WorkflowTestCase.master_api_key = master_api_key + functional.test_workflow.WorkflowTestCase.user_api_key = os.environ.get( "GALAXY_TEST_USER_API_KEY", default_galaxy_user_key ) + import functional.test_toolbox functional.test_toolbox.toolbox = app.toolbox functional.test_toolbox.build_tests( testing_shed_tools=testing_shed_tools, diff -r 6fadd2ce40e107fdc9f754cb0b45f1a1263d5921 -r 7c5305e54c03373b9922d60e2c75e90607c61c6f test-data/workflows/1.ga --- /dev/null +++ b/test-data/workflows/1.ga @@ -0,0 +1,145 @@ +{ + "a_galaxy_workflow": "true", + "annotation": "", + "format-version": "0.1", + "name": "TestWorkflowAlpha", + "steps": { + "0": { + "annotation": "", + "id": 0, + "input_connections": {}, + "inputs": [ + { + "description": "", + "name": "Input Dataset1" + } + ], + "name": "Input dataset", + "outputs": [], + "position": { + "left": 206, + "top": 207 + }, + "tool_errors": null, + "tool_id": null, + "tool_state": "{\"name\": \"Input Dataset1\"}", + "tool_version": null, + "type": "data_input", + "user_outputs": [] + }, + "1": { + "annotation": "", + "id": 1, + "input_connections": {}, + "inputs": [ + { + "description": "", + "name": "Input Dataset2" + } + ], + "name": "Input dataset", + "outputs": [], + "position": { + "left": 200, + "top": 320 + }, + "tool_errors": null, + "tool_id": null, + "tool_state": "{\"name\": \"Input Dataset2\"}", + "tool_version": null, + "type": "data_input", + "user_outputs": [] + }, + "2": { + "annotation": "", + "id": 2, + "input_connections": { + "input1": { + "id": 0, + "output_name": "output" + }, + "queries_0|input2": { + "id": 1, + "output_name": "output" + } + }, + "inputs": [], + "name": "Concatenate datasets", + "outputs": [ + { + "name": "out_file1", + "type": "input" + } + ], + "position": { + "left": 420, + "top": 200 + }, + "post_job_actions": {}, + "tool_errors": null, + "tool_id": "cat1", + "tool_state": "{\"__page__\": 0, \"__rerun_remap_job_id__\": null, \"input1\": \"null\", \"chromInfo\": \"\\\"/home/john/workspace/galaxy-central/tool-data/shared/ucsc/chrom/?.len\\\"\", \"queries\": \"[{\\\"input2\\\": null, \\\"__index__\\\": 0}]\"}", + "tool_version": "1.0.0", + "type": "tool", + "user_outputs": [] + }, + "3": { + "annotation": "", + "id": 3, + "input_connections": { + "input": { + "id": 2, + "output_name": "out_file1" + } + }, + "inputs": [], + "name": "Convert", + "outputs": [ + { + "name": "out_file1", + "type": "tabular" + } + ], + "position": { + "left": 640, + "top": 200 + }, + "post_job_actions": {}, + "tool_errors": null, + "tool_id": "Convert characters1", + "tool_state": "{\"__page__\": 0, \"convert_from\": \"\\\"s\\\"\", \"__rerun_remap_job_id__\": null, \"chromInfo\": \"\\\"/home/john/workspace/galaxy-central/tool-data/shared/ucsc/chrom/?.len\\\"\", \"input\": \"null\"}", + "tool_version": "1.0.0", + "type": "tool", + "user_outputs": [] + }, + "4": { + "annotation": "", + "id": 4, + "input_connections": { + "input": { + "id": 3, + "output_name": "out_file1" + } + }, + "inputs": [], + "name": "Add column", + "outputs": [ + { + "name": "out_file1", + "type": "input" + } + ], + "position": { + "left": 860, + "top": 200 + }, + "post_job_actions": {}, + "tool_errors": null, + "tool_id": "addValue", + "tool_state": "{\"__page__\": 0, \"__rerun_remap_job_id__\": null, \"exp\": \"\\\"1\\\"\", \"iterate\": \"\\\"yes\\\"\", \"input\": \"null\", \"chromInfo\": \"\\\"/home/john/workspace/galaxy-central/tool-data/shared/ucsc/chrom/?.len\\\"\"}", + "tool_version": "1.0.0", + "type": "tool", + "user_outputs": [] + } + } +} \ No newline at end of file diff -r 6fadd2ce40e107fdc9f754cb0b45f1a1263d5921 -r 7c5305e54c03373b9922d60e2c75e90607c61c6f test-data/workflows/1.xml --- /dev/null +++ b/test-data/workflows/1.xml @@ -0,0 +1,5 @@ +<test file="1.ga"> + <input name="Input Dataset1" value="workflows/i1.txt" /> + <input name="Input Dataset2" value="workflows/i2.txt" /> + <output name="2" file="workflows/o1.txt" /> +</test> diff -r 6fadd2ce40e107fdc9f754cb0b45f1a1263d5921 -r 7c5305e54c03373b9922d60e2c75e90607c61c6f test-data/workflows/i1.txt --- /dev/null +++ b/test-data/workflows/i1.txt @@ -0,0 +1,1 @@ +1 2 3 diff -r 6fadd2ce40e107fdc9f754cb0b45f1a1263d5921 -r 7c5305e54c03373b9922d60e2c75e90607c61c6f test-data/workflows/i2.txt --- /dev/null +++ b/test-data/workflows/i2.txt @@ -0,0 +1,1 @@ +4 5 6 diff -r 6fadd2ce40e107fdc9f754cb0b45f1a1263d5921 -r 7c5305e54c03373b9922d60e2c75e90607c61c6f test-data/workflows/o1.txt --- /dev/null +++ b/test-data/workflows/o1.txt @@ -0,0 +1,2 @@ +1 2 3 1 +4 5 6 2 diff -r 6fadd2ce40e107fdc9f754cb0b45f1a1263d5921 -r 7c5305e54c03373b9922d60e2c75e90607c61c6f test/functional/test_workflow.py --- /dev/null +++ b/test/functional/test_workflow.py @@ -0,0 +1,185 @@ +import os +import sys +from base.twilltestcase import TwillTestCase +from base.interactor import GalaxyInteractorApi, stage_data_in_history + +from galaxy.util import parse_xml +from galaxy.tools.test import parse_param_elem, require_file, test_data_iter, parse_output_elems +from simplejson import load, dumps + +from logging import getLogger +log = getLogger( __name__ ) + + +class WorkflowTestCase( TwillTestCase ): + """ + Kind of a shell of a test case for running workflow tests. Probably + needs to look more like test_toolbox. + """ + workflow_test_file = None + user_api_key = None + master_api_key = None + + def test_workflow( self, workflow_test_file=None ): + maxseconds = 120 + workflow_test_file = workflow_test_file or WorkflowTestCase.workflow_test_file + assert workflow_test_file + workflow_test = parse_test_file( workflow_test_file ) + galaxy_interactor = GalaxyWorkflowInteractor( self ) + + # Calling workflow https://github.com/jmchilton/blend4j/blob/master/src/test/java/com/github/j… + + # Import workflow + workflow_id, step_id_map, output_defs = self.__import_workflow( galaxy_interactor, workflow_test.workflow ) + + # Stage data and history for workflow + test_history = galaxy_interactor.new_history() + stage_data_in_history( galaxy_interactor, workflow_test.test_data(), test_history ) + + # Build workflow parameters + uploads = galaxy_interactor.uploads + ds_map = {} + for step_index, input_dataset_label in workflow_test.input_datasets(): + # Upload is {"src": "hda", "id": hid} + try: + upload = uploads[ workflow_test.upload_name( input_dataset_label ) ] + except KeyError: + raise AssertionError( "Failed to find upload with label %s in uploaded datasets %s" % ( input_dataset_label, uploads ) ) + + ds_map[ step_id_map[ step_index ] ] = upload + + payload = { + "history": "hist_id=%s" % test_history, + "ds_map": dumps( ds_map ), + "workflow_id": workflow_id, + } + run_response = galaxy_interactor.run_workflow( payload ).json() + + outputs = run_response[ 'outputs' ] + if not len( outputs ) == len( output_defs ): + msg_template = "Number of outputs [%d] created by workflow execution does not equal expected number from input file [%d]." + msg = msg_template % ( len( outputs ), len( output_defs ) ) + raise AssertionError( msg ) + + galaxy_interactor.wait_for_ids( test_history, outputs ) + + for expected_output_def in workflow_test.outputs: + # Get the correct hid + name, outfile, attributes = expected_output_def + + output_data = outputs[ int( name ) ] + try: + galaxy_interactor.verify_output( test_history, output_data, outfile, attributes=attributes, shed_tool_id=None, maxseconds=maxseconds ) + except Exception: + for stream in ['stdout', 'stderr']: + stream_output = galaxy_interactor.get_job_stream( test_history, output_data, stream=stream ) + print >>sys.stderr, self._format_stream( stream_output, stream=stream, format=True ) + raise + + def __import_workflow( self, galaxy_interactor, workflow ): + """ + Import workflow into Galaxy and return id and mapping of step ids. + """ + workflow_info = galaxy_interactor.import_workflow( workflow ).json() + try: + workflow_id = workflow_info[ 'id' ] + except KeyError: + raise AssertionError( "Failed to find id for workflow import response %s" % workflow_info ) + + # Well ideally the local copy of the workflow would have the same step ids + # as the one imported through the API, but API workflow imports are 1-indexed + # and GUI exports 0-indexed as of mid-november 2013. + + imported_workflow = galaxy_interactor.read_workflow( workflow_id ) + #log.info("local %s\nimported%s" % (workflow, imported_workflow)) + step_id_map = {} + local_steps_ids = sorted( [ int( step_id ) for step_id in workflow[ 'steps' ].keys() ] ) + imported_steps_ids = sorted( [ int( step_id ) for step_id in imported_workflow[ 'steps' ].keys() ] ) + for local_step_id, imported_step_id in zip( local_steps_ids, imported_steps_ids ): + step_id_map[ local_step_id ] = imported_step_id + + output_defs = [] + for local_step_id in local_steps_ids: + step_def = workflow['steps'][ str( local_step_id ) ] + output_defs.extend( step_def.get( "outputs", [] ) ) + + return workflow_id, step_id_map, output_defs + + +def parse_test_file( workflow_test_file ): + tree = parse_xml( workflow_test_file ) + root = tree.getroot() + input_elems = root.findall( "input" ) + required_files = [] + dataset_dict = {} + for input_elem in input_elems: + name, value, attrib = parse_param_elem( input_elem ) + require_file( name, value, attrib, required_files ) + dataset_dict[ name ] = value + + outputs = parse_output_elems( root ) + + workflow_file_rel_path = root.get( 'file' ) + if not workflow_file_rel_path: + raise Exception( "Workflow test XML must declare file attribute pointing to workflow under test." ) + + # TODO: Normalize this path, prevent it from accessing arbitrary files on system. + worfklow_file_abs_path = os.path.join( os.path.dirname( workflow_test_file ), workflow_file_rel_path ) + + return WorkflowTest( + dataset_dict, + required_files, + worfklow_file_abs_path, + outputs=outputs, + ) + + +class WorkflowTest( object ): + + def __init__( self, dataset_dict, required_files, workflow_file, outputs ): + self.dataset_dict = dataset_dict + self.required_files = required_files + self.workflow = load( open( workflow_file, "r" ) ) + self.outputs = outputs + + def test_data( self ): + return test_data_iter( self.required_files ) + + def upload_name( self, input_dataset_label ): + return self.dataset_dict[ input_dataset_label ] + + def input_datasets( self ): + steps = self.workflow[ "steps" ] + log.info("in input_datasets with steps %s" % steps) + for step_index, step_dict in steps.iteritems(): + if step_dict.get( "name", None ) == "Input dataset": + yield int( step_index ), step_dict[ "inputs" ][0][ "name" ] + + +class GalaxyWorkflowInteractor(GalaxyInteractorApi): + + def __init__( self, twill_test_case ): + super(GalaxyWorkflowInteractor, self).__init__( twill_test_case ) + + def import_workflow( self, workflow_rep ): + payload = { "workflow": dumps( workflow_rep ) } + return self._post( "workflows/upload", data=payload ) + + def run_workflow( self, data ): + return self._post( "workflows", data=data ) + + def read_workflow( self, id ): + return self._get( "workflows/%s" % id ).json() + + def wait_for_ids( self, history_id, ids ): + self.twill_test_case.wait_for( lambda: not all( [ self.__dataset_ready( history_id, id ) for id in ids ] ), maxseconds=120 ) + + def __dataset_ready( self, history_id, id ): + contents = self._get( 'histories/%s/contents' % history_id ).json() + for content in contents: + + if content["id"] == id: + state = content[ 'state' ] + state_ready = self._state_ready( state, error_msg="Dataset creation failed for dataset with name %s." % content[ 'name' ] ) + return state_ready + return False https://bitbucket.org/galaxy/galaxy-central/commits/87f3e11ced5d/ Changeset: 87f3e11ced5d User: jmchilton Date: 2013-11-22 02:17:13 Summary: Fix for using API driver with tool outputs that change their name. E.g. composite data types. API should probably return the original output name since that is the purpose of the parameter but I am not sure how much work that would be. Affected #: 2 files diff -r 7c5305e54c03373b9922d60e2c75e90607c61c6f -r 87f3e11ced5dbab70f50a2299fcda2859643ff62 test/base/interactor.py --- a/test/base/interactor.py +++ b/test/base/interactor.py @@ -1,5 +1,6 @@ import os from galaxy.tools.parameters import grouping +from galaxy.util.odict import odict import galaxy.model from galaxy.model.orm import and_, desc from galaxy.model.mapping import context as sa_session @@ -148,7 +149,7 @@ ## Convert outputs list to a dictionary that can be accessed by ## output_name so can be more flexiable about ordering of outputs ## but also allows fallback to legacy access as list mode. - outputs_dict = {} + outputs_dict = odict() index = 0 for output in datasets_object[ 'outputs' ]: outputs_dict[ index ] = outputs_dict[ output.get("output_name") ] = output diff -r 7c5305e54c03373b9922d60e2c75e90607c61c6f -r 87f3e11ced5dbab70f50a2299fcda2859643ff62 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -53,8 +53,12 @@ output_data = data_list[ name ] except (TypeError, KeyError): # Legacy - fall back on ordered data list access if data_list is - # just a list (case with twill variant) - output_data = data_list[ len(data_list) - len(testdef.outputs) + output_index ] + # just a list (case with twill variant or if output changes its + # name). + if hasattr(data_list, "values"): + output_data = data_list.values()[ output_index ] + else: + output_data = data_list[ len(data_list) - len(testdef.outputs) + output_index ] self.assertTrue( output_data is not None ) try: galaxy_interactor.verify_output( history, output_data, outfile, attributes=attributes, shed_tool_id=shed_tool_id, maxseconds=maxseconds ) https://bitbucket.org/galaxy/galaxy-central/commits/f302dae15be9/ Changeset: f302dae15be9 User: jmchilton Date: 2013-11-22 02:17:13 Summary: API Driven Tests: Option for very detailed per dataset error messages for when histories error out. Affected #: 1 file diff -r 87f3e11ced5dbab70f50a2299fcda2859643ff62 -r f302dae15be9185139d8cc17cd0208a448e7e9b0 test/base/interactor.py --- a/test/base/interactor.py +++ b/test/base/interactor.py @@ -1,4 +1,5 @@ import os +from StringIO import StringIO from galaxy.tools.parameters import grouping from galaxy.util.odict import odict import galaxy.model @@ -9,6 +10,9 @@ from logging import getLogger log = getLogger( __name__ ) +VERBOSE_ERRORS = True +ERROR_MESSAGE_DATASET_SEP = "--------------------------------------" + def build_interactor( test_case, type="api" ): interactor_class = GALAXY_INTERACTORS[ type ] @@ -60,7 +64,7 @@ def get_job_stream( self, history_id, output_data, stream ): hid = self.__output_id( output_data ) - data = self._get( "histories/%s/contents/%s/provenance" % (history_id, hid) ).json() + data = self._dataset_provenance( history_id, hid ) return data.get( stream, '' ) def new_history( self ): @@ -171,7 +175,61 @@ def __history_ready( self, history_id ): history_json = self._get( "histories/%s" % history_id ).json() state = history_json[ 'state' ] - return self._state_ready( state, error_msg="History in error state." ) + try: + return self._state_ready( state, error_msg="History in error state." ) + except Exception: + if VERBOSE_ERRORS: + self._summarize_history_errors( history_id ) + raise + + def _summarize_history_errors( self, history_id ): + print "History with id %s in error - summary of datasets in error below." % history_id + try: + history_contents = self.__contents( history_id ) + except Exception: + print "*TEST FRAMEWORK FAILED TO FETCH HISTORY DETAILS*" + + for dataset in history_contents: + if dataset[ 'state' ] != 'error': + continue + + print ERROR_MESSAGE_DATASET_SEP + dataset_id = dataset.get( 'id', None ) + print "| %d - %s (HID - NAME) " % ( int( dataset['hid'] ), dataset['name'] ) + try: + dataset_info = self._dataset_info( history_id, dataset_id ) + print "| Dataset Blurb:" + print self.format_for_error( dataset_info.get( "misc_blurb", "" ), "Dataset blurb was empty." ) + print "| Dataset Info:" + print self.format_for_error( dataset_info.get( "misc_info", "" ), "Dataset info is empty." ) + except Exception: + print "| *TEST FRAMEWORK ERROR FETCHING DATASET DETAILS*" + try: + provenance_info = self._dataset_provenance( history_id, dataset_id ) + print "| Dataset Job Standard Output:" + print self.format_for_error( provenance_info.get( "stdout", "" ), "Standard output was empty." ) + print "| Dataset Job Standard Error:" + print self.format_for_error( provenance_info.get( "stderr", "" ), "Standard error was empty." ) + except Exception: + print "| *TEST FRAMEWORK ERROR FETCHING JOB DETAILS*" + print "|" + print ERROR_MESSAGE_DATASET_SEP + + def format_for_error( self, blob, empty_message, prefix="| " ): + contents = "\n".join([ "%s%s" % (prefix, line.strip()) for line in StringIO(blob).readlines() if line.rstrip("\n\r") ] ) + return contents or "%s*%s*" % ( prefix, empty_message ) + + def _dataset_provenance( self, history_id, id ): + provenance = self._get( "histories/%s/contents/%s/provenance" % ( history_id, id ) ).json() + return provenance + + def _dataset_info( self, history_id, id ): + dataset_json = self._get( "histories/%s/contents/%s" % ( history_id, id ) ).json() + return dataset_json + + def __contents( self, history_id ): + history_contents_json = self._get( "histories/%s/contents" % history_id ).json() + return history_contents_json def _state_ready( self, state_str, error_msg ): if state_str == 'ok': Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: JS MVC: add SessionStorageModel to base-mvc
by commits-noreply＠bitbucket.org 21 Nov '13

21 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/5471fa863712/ Changeset: 5471fa863712 User: carlfeberhard Date: 2013-11-21 22:43:43 Summary: JS MVC: add SessionStorageModel to base-mvc Affected #: 2 files diff -r 32ecdc2fdc00a93652b27774123cde91bd3c512c -r 5471fa863712ba3f89ad9adceb194daf0baeafa2 static/scripts/mvc/base-mvc.js --- a/static/scripts/mvc/base-mvc.js +++ b/static/scripts/mvc/base-mvc.js @@ -89,6 +89,59 @@ //============================================================================== +/** Backbone model that syncs to the browser's sessionStorage API. + */ +var SessionStorageModel = Backbone.Model.extend({ + initialize : function( hash, x, y, z ){ + if( !hash || !hash.hasOwnProperty( 'id' ) ){ + throw new Error( 'SessionStorageModel needs an id on init' ); + } + // immed. save the passed in model and save on any change to it + this.save(); + this.on( 'change', function(){ + this.save(); + }); + }, + sync : function( method, model, options ){ + model.trigger('request', model, {}, options); + var returned; + switch( method ){ + case 'create' : returned = this._create( model ); break; + case 'read' : returned = this._read( model ); break; + case 'update' : returned = this._update( model ); break; + case 'delete' : returned = this._delete( model ); break; + } + if( returned !== undefined || returned !== null ){ + if( options.success ){ options.success(); } + } else { + if( options.error ){ options.error(); } + } + return returned; + }, + _create : function( model ){ + var json = model.toJSON(), + set = sessionStorage.setItem( model.id, JSON.stringify( json ) ); + return ( set === null )?( set ):( json ); + }, + _read : function( model ){ + return JSON.parse( sessionStorage.getItem( model.id, JSON.stringify( model.toJSON() ) ) ); + }, + _update : function( model ){ + return model._create( model ); + }, + _delete : function( model ){ + return sessionStorage.removeItem( model.id ); + }, + isNew : function(){ + return !sessionStorage.hasOwnProperty( this.id ); + } +}); +(function(){ + SessionStorageModel.prototype = _.omit( SessionStorageModel.prototype, 'url', 'urlRoot' ); +}()); + + +//============================================================================== /** * @class persistent storage adapter. * Provides an easy interface to object based storage using method chaining. diff -r 32ecdc2fdc00a93652b27774123cde91bd3c512c -r 5471fa863712ba3f89ad9adceb194daf0baeafa2 static/scripts/packed/mvc/base-mvc.js --- a/static/scripts/packed/mvc/base-mvc.js +++ b/static/scripts/packed/mvc/base-mvc.js @@ -1,1 +1,1 @@ -var BaseModel=Backbone.RelationalModel.extend({defaults:{name:null,hidden:false},show:function(){this.set("hidden",false)},hide:function(){this.set("hidden",true)},is_visible:function(){return !this.attributes.hidden}});var BaseView=Backbone.View.extend({initialize:function(){this.model.on("change:hidden",this.update_visible,this);this.update_visible()},update_visible:function(){if(this.model.attributes.hidden){this.$el.hide()}else{this.$el.show()}}});var LoggableMixin={logger:null,log:function(){if(this.logger){var a=this.logger.log;if(typeof this.logger.log==="object"){a=Function.prototype.bind.call(this.logger.log,this.logger)}return a.apply(this.logger,arguments)}return undefined}};var PersistentStorage=function(k,g){if(!k){throw ("PersistentStorage needs storageKey argument")}g=g||{};var i=sessionStorage,c=function j(m){var n=this.getItem(m);return(n!==null)?(JSON.parse(this.getItem(m))):(null)},b=function e(m,n){return this.setItem(m,JSON.stringify(n))},d=function f(m){return this.removeItem(m)};function a(n,m){n=n||{};m=m||null;return{get:function(o){if(o===undefined){return n}else{if(n.hasOwnProperty(o)){return(jQuery.type(n[o])==="object")?(new a(n[o],this)):(n[o])}}return undefined},set:function(o,p){n[o]=p;this._save();return this},deleteKey:function(o){delete n[o];this._save();return this},_save:function(){return m._save()},toString:function(){return("StorageRecursionHelper("+n+")")}}}var l={},h=c.call(i,k);if(h===null||h===undefined){h=jQuery.extend(true,{},g);b.call(i,k,h)}l=new a(h);jQuery.extend(l,{_save:function(m){return b.call(i,k,l.get())},destroy:function(){return d.call(i,k)},toString:function(){return"PersistentStorage("+k+")"}});return l};var HiddenUntilActivatedViewMixin={hiddenUntilActivated:function(a,c){c=c||{};this.HUAVOptions={$elementShown:this.$el,showFn:jQuery.prototype.toggle,showSpeed:"fast"};_.extend(this.HUAVOptions,c||{});this.HUAVOptions.hasBeenShown=this.HUAVOptions.$elementShown.is(":visible");if(a){var b=this;a.on("click",function(d){b.toggle(b.HUAVOptions.showSpeed)})}},toggle:function(){if(this.HUAVOptions.$elementShown.is(":hidden")){if(!this.HUAVOptions.hasBeenShown){if(_.isFunction(this.HUAVOptions.onshowFirstTime)){this.HUAVOptions.hasBeenShown=true;this.HUAVOptions.onshowFirstTime.call(this)}}else{if(_.isFunction(this.HUAVOptions.onshow)){this.HUAVOptions.onshow.call(this)}}}return this.HUAVOptions.showFn.apply(this.HUAVOptions.$elementShown,arguments)}}; \ No newline at end of file +var BaseModel=Backbone.RelationalModel.extend({defaults:{name:null,hidden:false},show:function(){this.set("hidden",false)},hide:function(){this.set("hidden",true)},is_visible:function(){return !this.attributes.hidden}});var BaseView=Backbone.View.extend({initialize:function(){this.model.on("change:hidden",this.update_visible,this);this.update_visible()},update_visible:function(){if(this.model.attributes.hidden){this.$el.hide()}else{this.$el.show()}}});var LoggableMixin={logger:null,log:function(){if(this.logger){var a=this.logger.log;if(typeof this.logger.log==="object"){a=Function.prototype.bind.call(this.logger.log,this.logger)}return a.apply(this.logger,arguments)}return undefined}};var SessionStorageModel=Backbone.Model.extend({initialize:function(b,a,d,c){if(!b||!b.hasOwnProperty("id")){throw new Error("SessionStorageModel needs an id on init")}this.save();this.on("change",function(){this.save()})},sync:function(d,b,a){b.trigger("request",b,{},a);var c;switch(d){case"create":c=this._create(b);break;case"read":c=this._read(b);break;case"update":c=this._update(b);break;case"delete":c=this._delete(b);break}if(c!==undefined||c!==null){if(a.success){a.success()}}else{if(a.error){a.error()}}return c},_create:function(a){var b=a.toJSON(),c=sessionStorage.setItem(a.id,JSON.stringify(b));return(c===null)?(c):(b)},_read:function(a){return JSON.parse(sessionStorage.getItem(a.id,JSON.stringify(a.toJSON())))},_update:function(a){return a._create(a)},_delete:function(a){return sessionStorage.removeItem(a.id)},isNew:function(){return !sessionStorage.hasOwnProperty(this.id)}});(function(){SessionStorageModel.prototype=_.omit(SessionStorageModel.prototype,"url","urlRoot")}());var PersistentStorage=function(k,g){if(!k){throw ("PersistentStorage needs storageKey argument")}g=g||{};var i=sessionStorage,c=function j(m){var n=this.getItem(m);return(n!==null)?(JSON.parse(this.getItem(m))):(null)},b=function e(m,n){return this.setItem(m,JSON.stringify(n))},d=function f(m){return this.removeItem(m)};function a(n,m){n=n||{};m=m||null;return{get:function(o){if(o===undefined){return n}else{if(n.hasOwnProperty(o)){return(jQuery.type(n[o])==="object")?(new a(n[o],this)):(n[o])}}return undefined},set:function(o,p){n[o]=p;this._save();return this},deleteKey:function(o){delete n[o];this._save();return this},_save:function(){return m._save()},toString:function(){return("StorageRecursionHelper("+n+")")}}}var l={},h=c.call(i,k);if(h===null||h===undefined){h=jQuery.extend(true,{},g);b.call(i,k,h)}l=new a(h);jQuery.extend(l,{_save:function(m){return b.call(i,k,l.get())},destroy:function(){return d.call(i,k)},toString:function(){return"PersistentStorage("+k+")"}});return l};var HiddenUntilActivatedViewMixin={hiddenUntilActivated:function(a,c){c=c||{};this.HUAVOptions={$elementShown:this.$el,showFn:jQuery.prototype.toggle,showSpeed:"fast"};_.extend(this.HUAVOptions,c||{});this.HUAVOptions.hasBeenShown=this.HUAVOptions.$elementShown.is(":visible");if(a){var b=this;a.on("click",function(d){b.toggle(b.HUAVOptions.showSpeed)})}},toggle:function(){if(this.HUAVOptions.$elementShown.is(":hidden")){if(!this.HUAVOptions.hasBeenShown){if(_.isFunction(this.HUAVOptions.onshowFirstTime)){this.HUAVOptions.hasBeenShown=true;this.HUAVOptions.onshowFirstTime.call(this)}}else{if(_.isFunction(this.HUAVOptions.onshow)){this.HUAVOptions.onshow.call(this)}}}return this.HUAVOptions.showFn.apply(this.HUAVOptions.$elementShown,arguments)}}; \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: UI: move loading indicator into ui.js, add cover option
by commits-noreply＠bitbucket.org 21 Nov '13

21 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/32ecdc2fdc00/ Changeset: 32ecdc2fdc00 User: carlfeberhard Date: 2013-11-21 22:22:58 Summary: UI: move loading indicator into ui.js, add cover option Affected #: 4 files diff -r a39f452a7f657f3b43dce87d08e143ed6f02867e -r 32ecdc2fdc00a93652b27774123cde91bd3c512c static/scripts/mvc/base-mvc.js --- a/static/scripts/mvc/base-mvc.js +++ b/static/scripts/mvc/base-mvc.js @@ -260,54 +260,3 @@ return this.HUAVOptions.showFn.apply( this.HUAVOptions.$elementShown, arguments ); } }; - - - -//============================================================================== -function LoadingIndicator( $where, options ){ - options = options || {}; - var self = this; - - function render(){ - var html = [ - '<div class="loading-indicator">', - '<span class="fa fa-spinner fa-spin fa-lg" style="color: grey"></span>', - '<span style="margin-left: 8px; color: grey"><i>loading...</i></span>', - '</div>' - ].join( '\n' ); - - return $( html ).css( options.css || { - 'position' : 'fixed', - 'margin' : '12px 0px 0px 10px', - 'opacity' : '0.85' - }).hide(); - } - - self.show = function( msg, speed, callback ){ - msg = msg || 'loading...'; - speed = speed || 'fast'; - // since position is fixed - we insert as sibling - self.$indicator = render().insertBefore( $where ); - self.message( msg ); - self.$indicator.fadeIn( speed, callback ); - return self; - }; - - self.message = function( msg ){ - self.$indicator.find( 'i' ).text( msg ); - }; - - self.hide = function( speed, callback ){ - speed = speed || 'fast'; - if( self.$indicator && self.$indicator.size() ){ - self.$indicator.fadeOut( speed, function(){ - self.$indicator.remove(); - if( callback ){ callback(); } - }); - } else { - if( callback ){ callback(); } - } - return self; - }; - return self; -} diff -r a39f452a7f657f3b43dce87d08e143ed6f02867e -r 32ecdc2fdc00a93652b27774123cde91bd3c512c static/scripts/mvc/ui.js --- a/static/scripts/mvc/ui.js +++ b/static/scripts/mvc/ui.js @@ -465,8 +465,10 @@ return popupMenusCreated; }; + +//============================================================================== var faIconButton = function( options ){ - //TODO: move out of global +//TODO: move out of global options = options || {}; options.tooltipConfig = options.tooltipConfig || { placement: 'bottom' }; @@ -491,6 +493,8 @@ return $button; }; + +//============================================================================== var searchInput = function( options ){ //TODO: move out of global //TODO: consolidate with tool menu functionality, use there @@ -589,3 +593,88 @@ } return $searchInput.append([ $input(), $clearBtn() ]); }; + + +//============================================================================== +function LoadingIndicator( $where, options ){ +//TODO: move out of global +//TODO: too specific to history panel + + var self = this; + // defaults + options = jQuery.extend({ + cover : false + }, options || {} ); + + function render(){ + var html = [ + '<div class="loading-indicator">', + '<div class="loading-indicator-text">', + '<span class="fa fa-spinner fa-spin fa-lg"></span>', + '<span class="loading-indicator-message">loading...</span>', + '</div>', + '</div>' + ].join( '\n' ); + + var $indicator = $( html ).hide().css( options.css || { + position : 'fixed' + }), + $text = $indicator.children( '.loading-indicator-text' ); + + if( options.cover ){ + $indicator.css({ + 'z-index' : 2, + top : $where.css( 'top' ), + bottom : $where.css( 'bottom' ), + left : $where.css( 'left' ), + right : $where.css( 'right' ), + opacity : 0.5, + 'background-color': 'white', + 'text-align': 'center' + }); + $text = $indicator.children( '.loading-indicator-text' ).css({ + 'margin-top' : '20px' + }); + + } else { + $text = $indicator.children( '.loading-indicator-text' ).css({ + margin : '12px 0px 0px 10px', + opacity : '0.85', + color : 'grey' + }); + $text.children( '.loading-indicator-message' ).css({ + margin : '0px 8px 0px 0px', + 'font-style' : 'italic' + }); + } + return $indicator; + } + + self.show = function( msg, speed, callback ){ + msg = msg || 'loading...'; + speed = speed || 'fast'; + // since position is fixed - we insert as sibling + self.$indicator = render().insertBefore( $where ); + self.message( msg ); + self.$indicator.fadeIn( speed, callback ); + return self; + }; + + self.message = function( msg ){ + self.$indicator.find( 'i' ).text( msg ); + }; + + self.hide = function( speed, callback ){ + speed = speed || 'fast'; + if( self.$indicator && self.$indicator.size() ){ + self.$indicator.fadeOut( speed, function(){ + self.$indicator.remove(); + if( callback ){ callback(); } + }); + } else { + if( callback ){ callback(); } + } + return self; + }; + return self; +} diff -r a39f452a7f657f3b43dce87d08e143ed6f02867e -r 32ecdc2fdc00a93652b27774123cde91bd3c512c static/scripts/packed/mvc/base-mvc.js --- a/static/scripts/packed/mvc/base-mvc.js +++ b/static/scripts/packed/mvc/base-mvc.js @@ -1,1 +1,1 @@ -var BaseModel=Backbone.RelationalModel.extend({defaults:{name:null,hidden:false},show:function(){this.set("hidden",false)},hide:function(){this.set("hidden",true)},is_visible:function(){return !this.attributes.hidden}});var BaseView=Backbone.View.extend({initialize:function(){this.model.on("change:hidden",this.update_visible,this);this.update_visible()},update_visible:function(){if(this.model.attributes.hidden){this.$el.hide()}else{this.$el.show()}}});var LoggableMixin={logger:null,log:function(){if(this.logger){var a=this.logger.log;if(typeof this.logger.log==="object"){a=Function.prototype.bind.call(this.logger.log,this.logger)}return a.apply(this.logger,arguments)}return undefined}};var PersistentStorage=function(k,g){if(!k){throw ("PersistentStorage needs storageKey argument")}g=g||{};var i=sessionStorage,c=function j(m){var n=this.getItem(m);return(n!==null)?(JSON.parse(this.getItem(m))):(null)},b=function e(m,n){return this.setItem(m,JSON.stringify(n))},d=function f(m){return this.removeItem(m)};function a(n,m){n=n||{};m=m||null;return{get:function(o){if(o===undefined){return n}else{if(n.hasOwnProperty(o)){return(jQuery.type(n[o])==="object")?(new a(n[o],this)):(n[o])}}return undefined},set:function(o,p){n[o]=p;this._save();return this},deleteKey:function(o){delete n[o];this._save();return this},_save:function(){return m._save()},toString:function(){return("StorageRecursionHelper("+n+")")}}}var l={},h=c.call(i,k);if(h===null||h===undefined){h=jQuery.extend(true,{},g);b.call(i,k,h)}l=new a(h);jQuery.extend(l,{_save:function(m){return b.call(i,k,l.get())},destroy:function(){return d.call(i,k)},toString:function(){return"PersistentStorage("+k+")"}});return l};var HiddenUntilActivatedViewMixin={hiddenUntilActivated:function(a,c){c=c||{};this.HUAVOptions={$elementShown:this.$el,showFn:jQuery.prototype.toggle,showSpeed:"fast"};_.extend(this.HUAVOptions,c||{});this.HUAVOptions.hasBeenShown=this.HUAVOptions.$elementShown.is(":visible");if(a){var b=this;a.on("click",function(d){b.toggle(b.HUAVOptions.showSpeed)})}},toggle:function(){if(this.HUAVOptions.$elementShown.is(":hidden")){if(!this.HUAVOptions.hasBeenShown){if(_.isFunction(this.HUAVOptions.onshowFirstTime)){this.HUAVOptions.hasBeenShown=true;this.HUAVOptions.onshowFirstTime.call(this)}}else{if(_.isFunction(this.HUAVOptions.onshow)){this.HUAVOptions.onshow.call(this)}}}return this.HUAVOptions.showFn.apply(this.HUAVOptions.$elementShown,arguments)}};function LoadingIndicator(a,c){c=c||{};var b=this;function d(){var e=['<div class="loading-indicator">','<span class="fa fa-spinner fa-spin fa-lg" style="color: grey"></span>','<span style="margin-left: 8px; color: grey"><i>loading...</i></span>',"</div>"].join("\n");return $(e).css(c.css||{position:"fixed",margin:"12px 0px 0px 10px",opacity:"0.85"}).hide()}b.show=function(f,e,g){f=f||"loading...";e=e||"fast";b.$indicator=d().insertBefore(a);b.message(f);b.$indicator.fadeIn(e,g);return b};b.message=function(e){b.$indicator.find("i").text(e)};b.hide=function(e,f){e=e||"fast";if(b.$indicator&&b.$indicator.size()){b.$indicator.fadeOut(e,function(){b.$indicator.remove();if(f){f()}})}else{if(f){f()}}return b};return b}; \ No newline at end of file +var BaseModel=Backbone.RelationalModel.extend({defaults:{name:null,hidden:false},show:function(){this.set("hidden",false)},hide:function(){this.set("hidden",true)},is_visible:function(){return !this.attributes.hidden}});var BaseView=Backbone.View.extend({initialize:function(){this.model.on("change:hidden",this.update_visible,this);this.update_visible()},update_visible:function(){if(this.model.attributes.hidden){this.$el.hide()}else{this.$el.show()}}});var LoggableMixin={logger:null,log:function(){if(this.logger){var a=this.logger.log;if(typeof this.logger.log==="object"){a=Function.prototype.bind.call(this.logger.log,this.logger)}return a.apply(this.logger,arguments)}return undefined}};var PersistentStorage=function(k,g){if(!k){throw ("PersistentStorage needs storageKey argument")}g=g||{};var i=sessionStorage,c=function j(m){var n=this.getItem(m);return(n!==null)?(JSON.parse(this.getItem(m))):(null)},b=function e(m,n){return this.setItem(m,JSON.stringify(n))},d=function f(m){return this.removeItem(m)};function a(n,m){n=n||{};m=m||null;return{get:function(o){if(o===undefined){return n}else{if(n.hasOwnProperty(o)){return(jQuery.type(n[o])==="object")?(new a(n[o],this)):(n[o])}}return undefined},set:function(o,p){n[o]=p;this._save();return this},deleteKey:function(o){delete n[o];this._save();return this},_save:function(){return m._save()},toString:function(){return("StorageRecursionHelper("+n+")")}}}var l={},h=c.call(i,k);if(h===null||h===undefined){h=jQuery.extend(true,{},g);b.call(i,k,h)}l=new a(h);jQuery.extend(l,{_save:function(m){return b.call(i,k,l.get())},destroy:function(){return d.call(i,k)},toString:function(){return"PersistentStorage("+k+")"}});return l};var HiddenUntilActivatedViewMixin={hiddenUntilActivated:function(a,c){c=c||{};this.HUAVOptions={$elementShown:this.$el,showFn:jQuery.prototype.toggle,showSpeed:"fast"};_.extend(this.HUAVOptions,c||{});this.HUAVOptions.hasBeenShown=this.HUAVOptions.$elementShown.is(":visible");if(a){var b=this;a.on("click",function(d){b.toggle(b.HUAVOptions.showSpeed)})}},toggle:function(){if(this.HUAVOptions.$elementShown.is(":hidden")){if(!this.HUAVOptions.hasBeenShown){if(_.isFunction(this.HUAVOptions.onshowFirstTime)){this.HUAVOptions.hasBeenShown=true;this.HUAVOptions.onshowFirstTime.call(this)}}else{if(_.isFunction(this.HUAVOptions.onshow)){this.HUAVOptions.onshow.call(this)}}}return this.HUAVOptions.showFn.apply(this.HUAVOptions.$elementShown,arguments)}}; \ No newline at end of file diff -r a39f452a7f657f3b43dce87d08e143ed6f02867e -r 32ecdc2fdc00a93652b27774123cde91bd3c512c static/scripts/packed/mvc/ui.js --- a/static/scripts/packed/mvc/ui.js +++ b/static/scripts/packed/mvc/ui.js @@ -1,1 +1,1 @@ -var IconButton=Backbone.Model.extend({defaults:{title:"",icon_class:"",on_click:null,menu_options:null,is_menu_button:true,id:null,href:null,target:null,enabled:true,visible:true,tooltip_config:{}}});var IconButtonView=Backbone.View.extend({initialize:function(){this.model.attributes.tooltip_config={placement:"bottom"};this.model.bind("change",this.render,this)},render:function(){this.$el.tooltip("hide");var a=this.template(this.model.toJSON());a.tooltip(this.model.get("tooltip_config"));this.$el.replaceWith(a);this.setElement(a);return this},events:{click:"click"},click:function(a){if(_.isFunction(this.model.get("on_click"))){this.model.get("on_click")(a);return false}return true},template:function(b){var a='title="'+b.title+'" class="icon-button';if(b.is_menu_button){a+=" menu-button"}a+=" "+b.icon_class;if(!b.enabled){a+="_disabled"}a+='"';if(b.id){a+=' id="'+b.id+'"'}a+=' href="'+b.href+'"';if(b.target){a+=' target="'+b.target+'"'}if(!b.visible){a+=' style="display: none;"'}if(b.enabled){a="<a "+a+"/>"}else{a="<span "+a+"/>"}return $(a)}});var IconButtonCollection=Backbone.Collection.extend({model:IconButton});var IconButtonMenuView=Backbone.View.extend({tagName:"div",initialize:function(){this.render()},render:function(){var a=this;this.collection.each(function(d){var b=$("<a/>").attr("href","javascript:void(0)").attr("title",d.attributes.title).addClass("icon-button menu-button").addClass(d.attributes.icon_class).appendTo(a.$el).click(d.attributes.on_click);if(d.attributes.tooltip_config){b.tooltip(d.attributes.tooltip_config)}var c=d.get("options");if(c){make_popupmenu(b,c)}});return this}});var create_icon_buttons_menu=function(b,a){if(!a){a={}}var c=new IconButtonCollection(_.map(b,function(d){return new IconButton(_.extend(d,a))}));return new IconButtonMenuView({collection:c})};var Grid=Backbone.Collection.extend({});var GridView=Backbone.View.extend({});var PopupMenu=Backbone.View.extend({initialize:function(b,a){this.$button=b||$("<div/>");this.options=a||[];var c=this;this.$button.click(function(d){c._renderAndShow(d);return false})},_renderAndShow:function(a){this.render();this.$el.appendTo("body");this.$el.css(this._getShownPosition(a));this._setUpCloseBehavior();this.$el.show()},render:function(){this.$el.addClass("popmenu-wrapper").hide().css({position:"absolute"}).html(this.template(this.$button.attr("id"),this.options));if(this.options.length){var a=this;this.$el.find("li").each(function(c,b){var d=a.options[c];if(d.func){$(this).children("a.popupmenu-option").click(function(e){d.func.call(a,e,d)})}})}return this},template:function(b,a){return['<ul id="',b,'-menu" class="dropdown-menu">',this._templateOptions(a),"</ul>"].join("")},_templateOptions:function(a){if(!a.length){return"<li>(no options)</li>"}return _.map(a,function(d){if(d.divider){return'<li class="divider"></li>'}else{if(d.header){return['<li class="head"><a href="javascript:void(0);">',d.html,"</a></li>"].join("")}}var c=d.href||"javascript:void(0);",e=(d.target)?(' target="'+d.target+'"'):(""),b=(d.checked)?('<span class="fa fa-check"></span>'):("");return['<li><a class="popupmenu-option" href="',c,'"',e,">",b,d.html,"</a></li>"].join("")}).join("")},_getShownPosition:function(b){var c=this.$el.width();var a=b.pageX-c/2;a=Math.min(a,$(document).scrollLeft()+$(window).width()-c-5);a=Math.max(a,$(document).scrollLeft()+5);return{top:b.pageY,left:a}},_setUpCloseBehavior:function(){var b=this;var a=function(c){c.one("click.close_popup",function(){b.remove()})};a($(window.document));a($(window.top.document));_.each(window.top.frames,function(c){a($(c.document))})},addItem:function(b,a){a=(a>=0)?a:this.options.length;this.options.splice(a,0,b);return this},removeItem:function(a){if(a>=0){this.options.splice(a,1)}return this},findIndexByHtml:function(b){for(var a=0;a<this.options.length;a++){if(_.has(this.options[a],"html")&&(this.options[a].html===b)){return a}}return null},findItemByHtml:function(a){return this.options[(this.findIndexByHtml(a))]},toString:function(){return"PopupMenu"}});PopupMenu.make_popupmenu=function(b,c){var a=[];_.each(c,function(f,d){var e={html:d};if(f===null){e.header=true}else{if(jQuery.type(f)==="function"){e.func=f}}a.push(e)});return new PopupMenu($(b),a)};PopupMenu.convertLinksToOptions=function(c,a){c=$(c);a=a||"a";var b=[];c.find(a).each(function(g,e){var f={},d=$(g);f.html=d.text();if(d.attr("href")){var j=d.attr("href"),k=d.attr("target"),h=d.attr("confirm");f.func=function(){if((h)&&(!confirm(h))){return}switch(k){case"_parent":window.parent.location=j;break;case"_top":window.top.location=j;break;default:window.location=j}}}b.push(f)});return b};PopupMenu.fromExistingDom=function(d,c,a){d=$(d);c=$(c);var b=PopupMenu.convertLinksToOptions(c,a);c.remove();return new PopupMenu(d,b)};PopupMenu.make_popup_menus=function(c,b,d){c=c||document;b=b||"div[popupmenu]";d=d||function(e,f){return"#"+e.attr("popupmenu")};var a=[];$(c).find(b).each(function(){var e=$(this),f=$(c).find(d(e,c));a.push(PopupMenu.fromDom(f,e));f.addClass("popup")});return a};var faIconButton=function(a){a=a||{};a.tooltipConfig=a.tooltipConfig||{placement:"bottom"};a.classes=["icon-btn"].concat(a.classes||[]);if(a.disabled){a.classes.push("disabled")}var b=['<a class="',a.classes.join(" "),'"',((a.title)?(' title="'+a.title+'"'):("")),((a.target)?(' target="'+a.target+'"'):("")),' href="',((a.href)?(a.href):("javascript:void(0);")),'">','<span class="fa ',a.faIcon,'"></span>',"</a>"].join("");var c=$(b).tooltip(a.tooltipConfig);if(_.isFunction(a.onclick)){c.click(a.onclick)}return c};var searchInput=function(k){var a=27,h=13,i=$("<div/>"),b={initialVal:"",name:"search",placeholder:"search",classes:"",onclear:function(){},onsearch:function(l){},minSearchLen:0,escWillClear:true,oninit:function(){}};if(jQuery.type(k)==="object"){k=jQuery.extend(true,b,k)}function d(l){var m=$(this).parent().children("input");m.val("");m.trigger("clear:searchInput");k.onclear()}function j(m,l){$(this).trigger("search:searchInput",l);k.onsearch(l)}function c(){return['<input type="text" name="',k.name,'" placeholder="',k.placeholder,'" ','class="search-query ',k.classes,'" ',"/>"].join("")}function g(){return $(c()).css({width:"100%","padding-right":"24px"}).focus(function(l){$(this).select()}).keyup(function(m){if(m.which===a&&k.escWillClear){d.call(this,m)}else{var l=$(this).val();if((m.which===h)||(k.minSearchLen&&l.length>=k.minSearchLen)){j.call(this,m,l)}else{if(!l.length){d.call(this,m)}}}}).val(k.initialVal)}function f(){return'<span class="search-clear fa fa-times-circle"></span>'}function e(){return $(f()).css({position:"absolute",right:"15px","font-size":"1.4em","line-height":"23px",color:"grey"}).click(function(l){d.call(this,l)})}return i.append([g(),e()])}; \ No newline at end of file +var IconButton=Backbone.Model.extend({defaults:{title:"",icon_class:"",on_click:null,menu_options:null,is_menu_button:true,id:null,href:null,target:null,enabled:true,visible:true,tooltip_config:{}}});var IconButtonView=Backbone.View.extend({initialize:function(){this.model.attributes.tooltip_config={placement:"bottom"};this.model.bind("change",this.render,this)},render:function(){this.$el.tooltip("hide");var a=this.template(this.model.toJSON());a.tooltip(this.model.get("tooltip_config"));this.$el.replaceWith(a);this.setElement(a);return this},events:{click:"click"},click:function(a){if(_.isFunction(this.model.get("on_click"))){this.model.get("on_click")(a);return false}return true},template:function(b){var a='title="'+b.title+'" class="icon-button';if(b.is_menu_button){a+=" menu-button"}a+=" "+b.icon_class;if(!b.enabled){a+="_disabled"}a+='"';if(b.id){a+=' id="'+b.id+'"'}a+=' href="'+b.href+'"';if(b.target){a+=' target="'+b.target+'"'}if(!b.visible){a+=' style="display: none;"'}if(b.enabled){a="<a "+a+"/>"}else{a="<span "+a+"/>"}return $(a)}});var IconButtonCollection=Backbone.Collection.extend({model:IconButton});var IconButtonMenuView=Backbone.View.extend({tagName:"div",initialize:function(){this.render()},render:function(){var a=this;this.collection.each(function(d){var b=$("<a/>").attr("href","javascript:void(0)").attr("title",d.attributes.title).addClass("icon-button menu-button").addClass(d.attributes.icon_class).appendTo(a.$el).click(d.attributes.on_click);if(d.attributes.tooltip_config){b.tooltip(d.attributes.tooltip_config)}var c=d.get("options");if(c){make_popupmenu(b,c)}});return this}});var create_icon_buttons_menu=function(b,a){if(!a){a={}}var c=new IconButtonCollection(_.map(b,function(d){return new IconButton(_.extend(d,a))}));return new IconButtonMenuView({collection:c})};var Grid=Backbone.Collection.extend({});var GridView=Backbone.View.extend({});var PopupMenu=Backbone.View.extend({initialize:function(b,a){this.$button=b||$("<div/>");this.options=a||[];var c=this;this.$button.click(function(d){c._renderAndShow(d);return false})},_renderAndShow:function(a){this.render();this.$el.appendTo("body");this.$el.css(this._getShownPosition(a));this._setUpCloseBehavior();this.$el.show()},render:function(){this.$el.addClass("popmenu-wrapper").hide().css({position:"absolute"}).html(this.template(this.$button.attr("id"),this.options));if(this.options.length){var a=this;this.$el.find("li").each(function(c,b){var d=a.options[c];if(d.func){$(this).children("a.popupmenu-option").click(function(e){d.func.call(a,e,d)})}})}return this},template:function(b,a){return['<ul id="',b,'-menu" class="dropdown-menu">',this._templateOptions(a),"</ul>"].join("")},_templateOptions:function(a){if(!a.length){return"<li>(no options)</li>"}return _.map(a,function(d){if(d.divider){return'<li class="divider"></li>'}else{if(d.header){return['<li class="head"><a href="javascript:void(0);">',d.html,"</a></li>"].join("")}}var c=d.href||"javascript:void(0);",e=(d.target)?(' target="'+d.target+'"'):(""),b=(d.checked)?('<span class="fa fa-check"></span>'):("");return['<li><a class="popupmenu-option" href="',c,'"',e,">",b,d.html,"</a></li>"].join("")}).join("")},_getShownPosition:function(b){var c=this.$el.width();var a=b.pageX-c/2;a=Math.min(a,$(document).scrollLeft()+$(window).width()-c-5);a=Math.max(a,$(document).scrollLeft()+5);return{top:b.pageY,left:a}},_setUpCloseBehavior:function(){var b=this;var a=function(c){c.one("click.close_popup",function(){b.remove()})};a($(window.document));a($(window.top.document));_.each(window.top.frames,function(c){a($(c.document))})},addItem:function(b,a){a=(a>=0)?a:this.options.length;this.options.splice(a,0,b);return this},removeItem:function(a){if(a>=0){this.options.splice(a,1)}return this},findIndexByHtml:function(b){for(var a=0;a<this.options.length;a++){if(_.has(this.options[a],"html")&&(this.options[a].html===b)){return a}}return null},findItemByHtml:function(a){return this.options[(this.findIndexByHtml(a))]},toString:function(){return"PopupMenu"}});PopupMenu.make_popupmenu=function(b,c){var a=[];_.each(c,function(f,d){var e={html:d};if(f===null){e.header=true}else{if(jQuery.type(f)==="function"){e.func=f}}a.push(e)});return new PopupMenu($(b),a)};PopupMenu.convertLinksToOptions=function(c,a){c=$(c);a=a||"a";var b=[];c.find(a).each(function(g,e){var f={},d=$(g);f.html=d.text();if(d.attr("href")){var j=d.attr("href"),k=d.attr("target"),h=d.attr("confirm");f.func=function(){if((h)&&(!confirm(h))){return}switch(k){case"_parent":window.parent.location=j;break;case"_top":window.top.location=j;break;default:window.location=j}}}b.push(f)});return b};PopupMenu.fromExistingDom=function(d,c,a){d=$(d);c=$(c);var b=PopupMenu.convertLinksToOptions(c,a);c.remove();return new PopupMenu(d,b)};PopupMenu.make_popup_menus=function(c,b,d){c=c||document;b=b||"div[popupmenu]";d=d||function(e,f){return"#"+e.attr("popupmenu")};var a=[];$(c).find(b).each(function(){var e=$(this),f=$(c).find(d(e,c));a.push(PopupMenu.fromDom(f,e));f.addClass("popup")});return a};var faIconButton=function(a){a=a||{};a.tooltipConfig=a.tooltipConfig||{placement:"bottom"};a.classes=["icon-btn"].concat(a.classes||[]);if(a.disabled){a.classes.push("disabled")}var b=['<a class="',a.classes.join(" "),'"',((a.title)?(' title="'+a.title+'"'):("")),((a.target)?(' target="'+a.target+'"'):("")),' href="',((a.href)?(a.href):("javascript:void(0);")),'">','<span class="fa ',a.faIcon,'"></span>',"</a>"].join("");var c=$(b).tooltip(a.tooltipConfig);if(_.isFunction(a.onclick)){c.click(a.onclick)}return c};var searchInput=function(k){var a=27,h=13,i=$("<div/>"),b={initialVal:"",name:"search",placeholder:"search",classes:"",onclear:function(){},onsearch:function(l){},minSearchLen:0,escWillClear:true,oninit:function(){}};if(jQuery.type(k)==="object"){k=jQuery.extend(true,b,k)}function d(l){var m=$(this).parent().children("input");m.val("");m.trigger("clear:searchInput");k.onclear()}function j(m,l){$(this).trigger("search:searchInput",l);k.onsearch(l)}function c(){return['<input type="text" name="',k.name,'" placeholder="',k.placeholder,'" ','class="search-query ',k.classes,'" ',"/>"].join("")}function g(){return $(c()).css({width:"100%","padding-right":"24px"}).focus(function(l){$(this).select()}).keyup(function(m){if(m.which===a&&k.escWillClear){d.call(this,m)}else{var l=$(this).val();if((m.which===h)||(k.minSearchLen&&l.length>=k.minSearchLen)){j.call(this,m,l)}else{if(!l.length){d.call(this,m)}}}}).val(k.initialVal)}function f(){return'<span class="search-clear fa fa-times-circle"></span>'}function e(){return $(f()).css({position:"absolute",right:"15px","font-size":"1.4em","line-height":"23px",color:"grey"}).click(function(l){d.call(this,l)})}return i.append([g(),e()])};function LoadingIndicator(a,c){var b=this;c=jQuery.extend({cover:false},c||{});function d(){var e=['<div class="loading-indicator">','<div class="loading-indicator-text">','<span class="fa fa-spinner fa-spin fa-lg"></span>','<span class="loading-indicator-message">loading...</span>',"</div>","</div>"].join("\n");var g=$(e).hide().css(c.css||{position:"fixed"}),f=g.children(".loading-indicator-text");if(c.cover){g.css({"z-index":2,top:a.css("top"),bottom:a.css("bottom"),left:a.css("left"),right:a.css("right"),opacity:0.5,"background-color":"white","text-align":"center"});f=g.children(".loading-indicator-text").css({"margin-top":"20px"})}else{f=g.children(".loading-indicator-text").css({margin:"12px 0px 0px 10px",opacity:"0.85",color:"grey"});f.children(".loading-indicator-message").css({margin:"0px 8px 0px 0px","font-style":"italic"})}return g}b.show=function(f,e,g){f=f||"loading...";e=e||"fast";b.$indicator=d().insertBefore(a);b.message(f);b.$indicator.fadeIn(e,g);return b};b.message=function(e){b.$indicator.find("i").text(e)};b.hide=function(e,f){e=e||"fast";if(b.$indicator&&b.$indicator.size()){b.$indicator.fadeOut(e,function(){b.$indicator.remove();if(f){f()}})}else{if(f){f()}}return b};return b}; \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: UI, History panel: connect search to HDAs
by commits-noreply＠bitbucket.org 21 Nov '13

21 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a39f452a7f65/ Changeset: a39f452a7f65 User: carlfeberhard Date: 2013-11-21 20:16:09 Summary: UI, History panel: connect search to HDAs Affected #: 8 files diff -r 253f888144aaa4ae4eadaf15de493b3144fc5cd3 -r a39f452a7f657f3b43dce87d08e143ed6f02867e static/scripts/mvc/dataset/hda-model.js --- a/static/scripts/mvc/dataset/hda-model.js +++ b/static/scripts/mvc/dataset/hda-model.js @@ -218,30 +218,97 @@ }, // ........................................................................ sorting/filtering - /** what attributes of an HDA will be used in a search */ - searchKeys : [ + + // ........................................................................ search + /** what attributes of an HDA will be used in a text search */ + searchAttributes : [ 'name', 'file_ext', 'genome_build', 'misc_blurb', 'misc_info', 'annotation', 'tags' ], - /** search this HDA for the string searchFor - * @param {String} searchFor look for this string in all attributes listed in searchKeys (above) using indexOf - * @returns {Array} an array of attribute keys where searchFor was found + /** our attr keys don't often match the labels we display to the user - so, when using + * attribute specifiers ('name="bler"') in a term, allow passing in aliases for the + * following attr keys. + */ + searchAliases : { + title : 'name', + format : 'file_ext', + database : 'genome_build', + blurb : 'misc_blurb', + description : 'misc_blurb', + info : 'misc_info', + tag : 'tags' + }, + + /** search the attribute with key attrKey for the string searchFor; T/F if found */ + searchAttribute : function( attrKey, searchFor ){ + var attrVal = this.get( attrKey ); + //console.debug( 'searchAttribute', attrKey, attrVal, searchFor ); + // bail if empty searchFor or unsearchable values + if( !searchFor + || ( attrVal === undefined || attrVal === null ) ){ + return false; + } + // pass to sep. fn for deep search of array attributes + if( _.isArray( attrVal ) ){ return this._searchArrayAttribute( attrVal, searchFor ); } + return ( attrVal.toString().toLowerCase().indexOf( searchFor.toLowerCase() ) !== -1 ); + }, + + /** deep(er) search for array attributes; T/F if found */ + _searchArrayAttribute : function( array, searchFor ){ + //console.debug( '_searchArrayAttribute', array, searchFor ); + searchFor = searchFor.toLowerCase(); + //precondition: searchFor has already been validated as non-empty string + //precondition: assumes only 1 level array + //TODO: could possibly break up searchFor more (CSV...) + return _.any( array, function( elem ){ + return ( elem.toString().toLowerCase().indexOf( searchFor.toLowerCase() ) !== -1 ); + }); + }, + + /** search all searchAttributes for the string searchFor, + * returning a list of keys of attributes that contain searchFor */ search : function( searchFor ){ var model = this; - searchFor = searchFor.toLowerCase(); - return _.filter( this.searchKeys, function( key ){ - var attr = model.get( key ); - return ( _.isString( attr ) && attr.toLowerCase().indexOf( searchFor ) !== -1 ); + return _.filter( this.searchAttributes, function( key ){ + return model.searchAttribute( key, searchFor ); }); }, - /** alias of search, but returns a boolean - * @param {String} matchesWhat look for this string in all attributes listed in searchKeys (above) using indexOf - * @returns {Boolean} was matchesWhat found in any attributes + /** alias of search, but returns a boolean; accepts attribute specifiers where + * the attributes searched can be narrowed to a single attribute using + * the form: matches( 'genome_build=hg19' ) + * (the attribute keys allowed can also be aliases to the true attribute key; + * see searchAliases above) + * @param {String} term plain text or ATTR_SPECIFIER sep. key=val pair + * @returns {Boolean} was term found in (any) attribute(s) */ - matches : function( matchesWhat ){ - return !!this.search( matchesWhat ).length; + matches : function( term ){ + var ATTR_SPECIFIER = '=', + split = term.split( ATTR_SPECIFIER ); + // attribute is specified - search only that + if( split.length >= 2 ){ + var attrKey = split[0]; + attrKey = this.searchAliases[ attrKey ] || attrKey; + return this.searchAttribute( attrKey, split[1] ); + } + // no attribute is specified - search all attributes in searchAttributes + return !!this.search( term ).length; + }, + + /** an implicit AND search for all terms; IOW, an hda must match all terms given + * where terms is a whitespace separated value string. + * e.g. given terms of: 'blah bler database=hg19' + * an HDA would have to have attributes containing blah AND bler AND a genome_build == hg19 + * To include whitespace in terms: wrap the term in double quotations. + */ + matchesAll : function( terms ){ + var model = this; + // break the terms up by whitespace and filter out the empty strings + terms = terms.match( /(".*"|\w*=".*"|\S*)/g ).filter( function( s ){ return !!s; }); + return _.all( terms, function( term ){ + return model.matches( term ); + }); }, // ........................................................................ misc @@ -389,8 +456,28 @@ * @returns array of hda models * @see HistoryDatasetAssociation#isVisible */ - getVisible : function( show_deleted, show_hidden ){ - return this.filter( function( item ){ return item.isVisible( show_deleted, show_hidden ); }); + getVisible : function( show_deleted, show_hidden, filters ){ + filters = filters || []; + //console.debug( 'filters:', filters ); + //TODO:?? why doesn't this return a collection? + // always filter by show deleted/hidden first + var filteredHdas = new HDACollection( this.filter( function( item ){ + return item.isVisible( show_deleted, show_hidden ); + })); + + _.each( filters, function( filter_fn ){ + if( !_.isFunction( filter_fn ) ){ return; } + filteredHdas = new HDACollection( filteredHdas.filter( filter_fn ) ); + }); + //if( filteredHdas.length ){ + // console.debug( 'filteredHdas:' ); + // filteredHdas.each( function( hda ){ + // console.debug( '\t', hda ); + // }); + //} else { + // console.warn( 'no visible hdas' ); + //} + return filteredHdas; }, // ........................................................................ ajax @@ -433,7 +520,7 @@ /** String representation. */ toString : function(){ - return ( 'HDACollection()' ); + return ([ 'HDACollection(', [ this.historyId, this.length ].join(), ')' ].join( '' )); } }); diff -r 253f888144aaa4ae4eadaf15de493b3144fc5cd3 -r a39f452a7f657f3b43dce87d08e143ed6f02867e static/scripts/mvc/history/history-panel.js --- a/static/scripts/mvc/history/history-panel.js +++ b/static/scripts/mvc/history/history-panel.js @@ -78,6 +78,9 @@ /** loading indicator */ this.indicator = new LoadingIndicator( this.$el ); + /** filters for displaying hdas */ + this.filters = []; + // ---- handle models passed on init if( this.model ){ this._setUpWebStorage( attributes.initiallyExpanded, attributes.show_deleted, attributes.show_hidden ); @@ -579,21 +582,25 @@ * @fires: rendered when the panel is attached and fully visible * @see Backbone.View#render */ - render : function( callback ){ + render : function( speed, callback ){ + // send a speed of 0 to have no fade in/out performed + speed = ( speed === undefined )?( this.fxSpeed ):( speed ); var panel = this, $newRender; + // handle the possibility of no model (can occur if fetching the model returns an error) if( this.model ){ $newRender = this.renderModel(); } else { $newRender = this.renderWithoutModel(); } + // fade out existing, swap with the new, fade in, set up behaviours $( panel ).queue( 'fx', [ function( next ){ //panel.$el.fadeTo( panel.fxSpeed, 0.0001, next ); - if( panel.$el.is( ':visible' ) ){ - panel.$el.fadeOut( panel.fxSpeed, next ); + if( speed && panel.$el.is( ':visible' ) ){ + panel.$el.fadeOut( speed, next ); } else { next(); } @@ -604,12 +611,20 @@ if( $newRender ){ panel.$el.append( $newRender.children() ); } - panel.$el.fadeIn( panel.fxSpeed, next ); + next(); + }, + function( next ){ + if( speed && !panel.$el.is( ':visible' ) ){ + panel.$el.fadeIn( speed, next ); + } else { + next(); + } }, function( next ){ //TODO: ideally, these would be set up before the fade in (can't because of async save text) if( callback ){ callback.call( this ); } panel.trigger( 'rendered', this ); + next(); } ]); return this; @@ -640,12 +655,7 @@ this._setUpBehaviours( $newRender ); // render hda views (if any and any shown (show_deleted/hidden) - //TODO: this seems too elaborate - if( !this.model.hdas.length - || !this.renderHdas( $newRender.find( this.datasetsSelector ) ) ){ - // if history is empty or no hdas would be rendered, show the empty message - $newRender.find( this.emptyMsgSelector ).show(); - } + this.renderHdas( $newRender ); return $newRender; }, @@ -702,20 +712,32 @@ * @returns the number of visible hda views */ renderHdas : function( $whereTo ){ + $whereTo = $whereTo || this.$el; this.hdaViews = {}; var historyView = this, + $datasetsList = $whereTo.find( this.datasetsSelector ), // only render the shown hdas //TODO: switch to more general filtered pattern visibleHdas = this.model.hdas.getVisible( this.storage.get( 'show_deleted' ), - this.storage.get( 'show_hidden' ) + this.storage.get( 'show_hidden' ), + this.filters ); + //console.debug( 'renderHdas, visibleHdas:', visibleHdas, $whereTo ); + $datasetsList.empty(); - _.each( visibleHdas, function( hda ){ - // render it (NOTE: reverse order, newest on top (prepend)) - $whereTo.prepend( historyView.createHdaView( hda ).$el ); - }); - return visibleHdas.length; + if( visibleHdas.length ){ + visibleHdas.each( function( hda ){ + // render it (NOTE: reverse order, newest on top (prepend)) + $datasetsList.prepend( historyView.createHdaView( hda ).$el ); + }); + $whereTo.find( this.emptyMsgSelector ).hide(); + + } else { + //console.debug( 'emptyMsg:', $whereTo.find( this.emptyMsgSelector ) ) + $whereTo.find( this.emptyMsgSelector ).show(); + } + return this.hdaViews; }, // ------------------------------------------------------------------------ panel events @@ -763,6 +785,55 @@ return this.storage.get( 'show_hidden' ); }, + // ........................................................................ filters + /** render a search input for filtering datasets shown + * (see the search section in the HDA model for implementation of the actual searching) + * return will start the search + * esc will clear the search + * clicking the clear button will clear the search + * uses searchInput in ui.js + */ + renderSearchControls : function(){ + var panel = this; + //TODO: needs proper async + panel.model.hdas.fetchAllDetails({ silent: true }); + + function onSearch( searchFor ){ + //console.debug( 'onSearch', searchFor, panel ); + panel.searchFor = searchFor; + panel.filters = [ function( hda ){ return hda.matchesAll( panel.searchFor ); } ]; + panel.trigger( 'search:searching', searchFor, panel ); + panel.renderHdas(); + } + function onSearchClear(){ + //console.debug( 'onSearchClear', panel ); + panel.searchFor = ''; + panel.filters = []; + panel.trigger( 'search:clear', panel ); + panel.renderHdas(); + } + return searchInput({ + initialVal : panel.searchFor, + name : 'history-search', + placeholder : 'search datasets', + classes : 'history-search', + onsearch : onSearch, + onclear : onSearchClear + }) + .addClass( 'history-search-controls' ) + .css( 'padding', '0px 0px 8px 0px' ); + }, + + /** toggle showing/hiding the search controls (rendering first on the initial show) */ + toggleSearchControls : function(){ + var $searchInput = this.$el.find( '.history-search-controls' ); + if( !$searchInput.size() ){ + $searchInput = this.renderSearchControls().hide(); + this.$el.find( '.history-title' ).before( $searchInput ); + } + $searchInput.slideToggle( this.fxSpeed ); + }, + // ........................................................................ multi-select of hdas showSelect : function( speed ){ _.each( this.hdaViews, function( view ){ diff -r 253f888144aaa4ae4eadaf15de493b3144fc5cd3 -r a39f452a7f657f3b43dce87d08e143ed6f02867e static/scripts/mvc/ui.js --- a/static/scripts/mvc/ui.js +++ b/static/scripts/mvc/ui.js @@ -466,7 +466,7 @@ }; var faIconButton = function( options ){ - //FFS + //TODO: move out of global options = options || {}; options.tooltipConfig = options.tooltipConfig || { placement: 'bottom' }; @@ -491,4 +491,101 @@ return $button; }; -//var hideUntilActivated; +var searchInput = function( options ){ +//TODO: move out of global +//TODO: consolidate with tool menu functionality, use there +//TODO: this could and should be merged/oop'd with editableText (the behaviors are mostly the same - but not style) + var KEYCODE_ESC = 27, + KEYCODE_RETURN = 13, + $searchInput = $( '<div/>' ), + defaults = { + initialVal : '', + name : 'search', + placeholder : 'search', + classes : '', + onclear : function(){}, + onsearch : function( inputVal ){}, + minSearchLen : 0, + escWillClear : true, + oninit : function(){} + }; + + if( jQuery.type( options ) === 'object' ){ + options = jQuery.extend( true, defaults, options ); + } + //console.debug( options ); + + function clearSearchInput( event ){ + //console.debug( this, 'clear' ); + var $input = $( this ).parent().children( 'input' ); + //console.debug( 'input', $input ); + $input.val( '' ); + $input.trigger( 'clear:searchInput' ); + options.onclear(); + } + function search( event, searchTerms ){ + //console.debug( this, 'searching', searchTerms ); + $( this ).trigger( 'search:searchInput', searchTerms ); + options.onsearch( searchTerms ); + //var $input = $( this ).parent().children( 'input' ); + //console.debug( 'input', $input ); + } + + function inputTemplate(){ + // class search-query is bootstrap 2.3 style that now lives in base.less + return [ '<input type="text" name="', options.name, '" placeholder="', options.placeholder, '" ', + 'class="search-query ', options.classes, '" ', '/>' + ].join( '' ); + } + // the search input that responds to keyboard events and displays the search value + function $input(){ + return $( inputTemplate() ).css({ + 'width' : '100%', + // make space for the clear button + 'padding-right' : '24px' + }) + // select all text on a focus + .focus( function( event ){ + $( this ).select(); + }) + // attach behaviors to esc, return if desired, search on some min len string + .keyup( function( event ){ + //console.debug( event.which, $( this ).val() ) + // esc key will clear if + if( event.which === KEYCODE_ESC && options.escWillClear ){ + clearSearchInput.call( this, event ); + + } else { + var searchTerms = $( this ).val(); + // return key or the search string len > minSearchLen (if not 0) triggers search + if( ( event.which === KEYCODE_RETURN ) + || ( options.minSearchLen && searchTerms.length >= options.minSearchLen ) ){ + search.call( this, event, searchTerms ); + } else if( !searchTerms.length ){ + clearSearchInput.call( this, event ); + } + } + }) + .val( options.initialVal ); + } + + // a button for clearing the search bar, placed on the right hand side + function clearBtnTemplate(){ + return '<span class="search-clear fa fa-times-circle"></span>'; + } + function $clearBtn(){ +//TODO: to base.less +//TODO: hover effects + return $( clearBtnTemplate() ).css({ + position : 'absolute', + right : '15px', + 'font-size' : '1.4em', + 'line-height': '23px', + color : 'grey' + }) + .click( function( event ){ + clearSearchInput.call( this, event ); + }); + } + return $searchInput.append([ $input(), $clearBtn() ]); +}; diff -r 253f888144aaa4ae4eadaf15de493b3144fc5cd3 -r a39f452a7f657f3b43dce87d08e143ed6f02867e static/scripts/packed/mvc/dataset/hda-model.js --- a/static/scripts/packed/mvc/dataset/hda-model.js +++ b/static/scripts/packed/mvc/dataset/hda-model.js @@ -1,1 +1,1 @@ -define([],function(){var d=Backbone.Model.extend(LoggableMixin).extend({defaults:{history_id:null,model_class:"HistoryDatasetAssociation",hid:0,id:null,name:"(unnamed dataset)",state:"new",deleted:false,visible:true,accessible:true,purged:false,data_type:null,file_size:0,file_ext:"",meta_files:[],misc_blurb:"",misc_info:"",tags:null},urlRoot:galaxy_config.root+"api/histories/",url:function(){return this.urlRoot+this.get("history_id")+"/contents/"+this.get("id")},urls:function(){var i=this.get("id");if(!i){return{}}var h={purge:galaxy_config.root+"datasets/"+i+"/purge_async",display:galaxy_config.root+"datasets/"+i+"/display/?preview=True",edit:galaxy_config.root+"datasets/"+i+"/edit",download:galaxy_config.root+"datasets/"+i+"/display?to_ext="+this.get("file_ext"),report_error:galaxy_config.root+"dataset/errors?id="+i,rerun:galaxy_config.root+"tool_runner/rerun?id="+i,show_params:galaxy_config.root+"datasets/"+i+"/show_params",visualization:galaxy_config.root+"visualization",annotation:{get:galaxy_config.root+"dataset/get_annotation_async?id="+i,set:galaxy_config.root+"dataset/annotate_async?id="+i},meta_download:galaxy_config.root+"dataset/get_metadata_file?hda_id="+i+"&metadata_name="};return h},initialize:function(h){this.log(this+".initialize",this.attributes);this.log("\tparent history_id: "+this.get("history_id"));if(!this.get("accessible")){this.set("state",d.STATES.NOT_VIEWABLE)}this._setUpListeners()},_setUpListeners:function(){this.on("change:state",function(i,h){this.log(this+" has changed state:",i,h);if(this.inReadyState()){this.trigger("state:ready",i,h,this.previous("state"))}})},toJSON:function(){var h=Backbone.Model.prototype.toJSON.call(this);h.misc_info=jQuery.trim(h.misc_info);return h},isDeletedOrPurged:function(){return(this.get("deleted")||this.get("purged"))},isVisible:function(i,j){var h=true;if((!i)&&(this.get("deleted")||this.get("purged"))){h=false}if((!j)&&(!this.get("visible"))){h=false}return h},hidden:function(){return !this.get("visible")},inReadyState:function(){var h=_.contains(d.READY_STATES,this.get("state"));return(this.isDeletedOrPurged()||h)},hasDetails:function(){return _.has(this.attributes,"genome_build")},hasData:function(){return(this.get("file_size")>0)},"delete":function c(h){return this.save({deleted:true},h)},undelete:function a(h){return this.save({deleted:false},h)},hide:function b(h){return this.save({visible:false},h)},unhide:function g(h){return this.save({visible:true},h)},purge:function f(h){h=h||{};h.url=galaxy_config.root+"datasets/"+this.get("id")+"/purge_async";var i=this,j=jQuery.ajax(h);j.done(function(m,k,l){i.set("purged",true)});j.fail(function(o,k,n){var l=_l("Unable to purge this dataset");var m=("Removal of datasets by users is not allowed in this Galaxy instance");if(o.responseJSON&&o.responseJSON.error){l=o.responseJSON.error}else{if(o.responseText.indexOf(m)!==-1){l=m}}o.responseText=l;i.trigger("error",i,o,h,_l(l),{error:l})})},searchKeys:["name","file_ext","genome_build","misc_blurb","misc_info","annotation","tags"],search:function(h){var i=this;h=h.toLowerCase();return _.filter(this.searchKeys,function(k){var j=i.get(k);return(_.isString(j)&&j.toLowerCase().indexOf(h)!==-1)})},matches:function(h){return !!this.search(h).length},toString:function(){var h=this.get("id")||"";if(this.get("name")){h=this.get("hid")+' :"'+this.get("name")+'",'+h}return"HDA("+h+")"}});d.STATES={UPLOAD:"upload",QUEUED:"queued",RUNNING:"running",SETTING_METADATA:"setting_metadata",NEW:"new",EMPTY:"empty",OK:"ok",PAUSED:"paused",FAILED_METADATA:"failed_metadata",NOT_VIEWABLE:"noPermission",DISCARDED:"discarded",ERROR:"error"};d.READY_STATES=[d.STATES.NEW,d.STATES.OK,d.STATES.EMPTY,d.STATES.PAUSED,d.STATES.FAILED_METADATA,d.STATES.NOT_VIEWABLE,d.STATES.DISCARDED,d.STATES.ERROR];d.NOT_READY_STATES=[d.STATES.UPLOAD,d.STATES.QUEUED,d.STATES.RUNNING,d.STATES.SETTING_METADATA];var e=Backbone.Collection.extend(LoggableMixin).extend({model:d,urlRoot:galaxy_config.root+"api/histories",url:function(){return this.urlRoot+"/"+this.historyId+"/contents"},initialize:function(i,h){h=h||{};this.historyId=h.historyId},ids:function(){return this.map(function(h){return h.id})},notReady:function(){return this.filter(function(h){return !h.inReadyState()})},running:function(){var h=[];this.each(function(i){if(!i.inReadyState()){h.push(i.get("id"))}});return h},getByHid:function(h){return _.first(this.filter(function(i){return i.get("hid")===h}))},getVisible:function(h,i){return this.filter(function(j){return j.isVisible(h,i)})},fetchAllDetails:function(i){i=i||{};var h={details:"all"};i.data=(i.data)?(_.extend(i.data,h)):(h);return this.fetch(i)},matches:function(h){return this.filter(function(i){return i.matches(h)})},set:function(j,h){var i=this;j=_.map(j,function(l){var m=i.get(l.id);if(!m){return l}var k=m.toJSON();_.extend(k,l);return k});Backbone.Collection.prototype.set.call(this,j,h)},toString:function(){return("HDACollection()")}});return{HistoryDatasetAssociation:d,HDACollection:e}}); \ No newline at end of file +define([],function(){var d=Backbone.Model.extend(LoggableMixin).extend({defaults:{history_id:null,model_class:"HistoryDatasetAssociation",hid:0,id:null,name:"(unnamed dataset)",state:"new",deleted:false,visible:true,accessible:true,purged:false,data_type:null,file_size:0,file_ext:"",meta_files:[],misc_blurb:"",misc_info:"",tags:null},urlRoot:galaxy_config.root+"api/histories/",url:function(){return this.urlRoot+this.get("history_id")+"/contents/"+this.get("id")},urls:function(){var i=this.get("id");if(!i){return{}}var h={purge:galaxy_config.root+"datasets/"+i+"/purge_async",display:galaxy_config.root+"datasets/"+i+"/display/?preview=True",edit:galaxy_config.root+"datasets/"+i+"/edit",download:galaxy_config.root+"datasets/"+i+"/display?to_ext="+this.get("file_ext"),report_error:galaxy_config.root+"dataset/errors?id="+i,rerun:galaxy_config.root+"tool_runner/rerun?id="+i,show_params:galaxy_config.root+"datasets/"+i+"/show_params",visualization:galaxy_config.root+"visualization",annotation:{get:galaxy_config.root+"dataset/get_annotation_async?id="+i,set:galaxy_config.root+"dataset/annotate_async?id="+i},meta_download:galaxy_config.root+"dataset/get_metadata_file?hda_id="+i+"&metadata_name="};return h},initialize:function(h){this.log(this+".initialize",this.attributes);this.log("\tparent history_id: "+this.get("history_id"));if(!this.get("accessible")){this.set("state",d.STATES.NOT_VIEWABLE)}this._setUpListeners()},_setUpListeners:function(){this.on("change:state",function(i,h){this.log(this+" has changed state:",i,h);if(this.inReadyState()){this.trigger("state:ready",i,h,this.previous("state"))}})},toJSON:function(){var h=Backbone.Model.prototype.toJSON.call(this);h.misc_info=jQuery.trim(h.misc_info);return h},isDeletedOrPurged:function(){return(this.get("deleted")||this.get("purged"))},isVisible:function(i,j){var h=true;if((!i)&&(this.get("deleted")||this.get("purged"))){h=false}if((!j)&&(!this.get("visible"))){h=false}return h},hidden:function(){return !this.get("visible")},inReadyState:function(){var h=_.contains(d.READY_STATES,this.get("state"));return(this.isDeletedOrPurged()||h)},hasDetails:function(){return _.has(this.attributes,"genome_build")},hasData:function(){return(this.get("file_size")>0)},"delete":function c(h){return this.save({deleted:true},h)},undelete:function a(h){return this.save({deleted:false},h)},hide:function b(h){return this.save({visible:false},h)},unhide:function g(h){return this.save({visible:true},h)},purge:function f(h){h=h||{};h.url=galaxy_config.root+"datasets/"+this.get("id")+"/purge_async";var i=this,j=jQuery.ajax(h);j.done(function(m,k,l){i.set("purged",true)});j.fail(function(o,k,n){var l=_l("Unable to purge this dataset");var m=("Removal of datasets by users is not allowed in this Galaxy instance");if(o.responseJSON&&o.responseJSON.error){l=o.responseJSON.error}else{if(o.responseText.indexOf(m)!==-1){l=m}}o.responseText=l;i.trigger("error",i,o,h,_l(l),{error:l})})},searchAttributes:["name","file_ext","genome_build","misc_blurb","misc_info","annotation","tags"],searchAliases:{title:"name",format:"file_ext",database:"genome_build",blurb:"misc_blurb",description:"misc_blurb",info:"misc_info",tag:"tags"},searchAttribute:function(j,h){var i=this.get(j);if(!h||(i===undefined||i===null)){return false}if(_.isArray(i)){return this._searchArrayAttribute(i,h)}return(i.toString().toLowerCase().indexOf(h.toLowerCase())!==-1)},_searchArrayAttribute:function(i,h){h=h.toLowerCase();return _.any(i,function(j){return(j.toString().toLowerCase().indexOf(h.toLowerCase())!==-1)})},search:function(h){var i=this;return _.filter(this.searchAttributes,function(j){return i.searchAttribute(j,h)})},matches:function(i){var k="=",h=i.split(k);if(h.length>=2){var j=h[0];j=this.searchAliases[j]||j;return this.searchAttribute(j,h[1])}return !!this.search(i).length},matchesAll:function(i){var h=this;i=i.match(/(".*"|\w*=".*"|\S*)/g).filter(function(j){return !!j});return _.all(i,function(j){return h.matches(j)})},toString:function(){var h=this.get("id")||"";if(this.get("name")){h=this.get("hid")+' :"'+this.get("name")+'",'+h}return"HDA("+h+")"}});d.STATES={UPLOAD:"upload",QUEUED:"queued",RUNNING:"running",SETTING_METADATA:"setting_metadata",NEW:"new",EMPTY:"empty",OK:"ok",PAUSED:"paused",FAILED_METADATA:"failed_metadata",NOT_VIEWABLE:"noPermission",DISCARDED:"discarded",ERROR:"error"};d.READY_STATES=[d.STATES.NEW,d.STATES.OK,d.STATES.EMPTY,d.STATES.PAUSED,d.STATES.FAILED_METADATA,d.STATES.NOT_VIEWABLE,d.STATES.DISCARDED,d.STATES.ERROR];d.NOT_READY_STATES=[d.STATES.UPLOAD,d.STATES.QUEUED,d.STATES.RUNNING,d.STATES.SETTING_METADATA];var e=Backbone.Collection.extend(LoggableMixin).extend({model:d,urlRoot:galaxy_config.root+"api/histories",url:function(){return this.urlRoot+"/"+this.historyId+"/contents"},initialize:function(i,h){h=h||{};this.historyId=h.historyId},ids:function(){return this.map(function(h){return h.id})},notReady:function(){return this.filter(function(h){return !h.inReadyState()})},running:function(){var h=[];this.each(function(i){if(!i.inReadyState()){h.push(i.get("id"))}});return h},getByHid:function(h){return _.first(this.filter(function(i){return i.get("hid")===h}))},getVisible:function(h,k,j){j=j||[];var i=new e(this.filter(function(l){return l.isVisible(h,k)}));_.each(j,function(l){if(!_.isFunction(l)){return}i=new e(i.filter(l))});return i},fetchAllDetails:function(i){i=i||{};var h={details:"all"};i.data=(i.data)?(_.extend(i.data,h)):(h);return this.fetch(i)},matches:function(h){return this.filter(function(i){return i.matches(h)})},set:function(j,h){var i=this;j=_.map(j,function(l){var m=i.get(l.id);if(!m){return l}var k=m.toJSON();_.extend(k,l);return k});Backbone.Collection.prototype.set.call(this,j,h)},toString:function(){return(["HDACollection(",[this.historyId,this.length].join(),")"].join(""))}});return{HistoryDatasetAssociation:d,HDACollection:e}}); \ No newline at end of file diff -r 253f888144aaa4ae4eadaf15de493b3144fc5cd3 -r a39f452a7f657f3b43dce87d08e143ed6f02867e static/scripts/packed/mvc/history/history-panel.js --- a/static/scripts/packed/mvc/history/history-panel.js +++ b/static/scripts/packed/mvc/history/history-panel.js @@ -1,1 +1,1 @@ -define(["mvc/history/history-model","mvc/dataset/hda-base","mvc/dataset/hda-edit"],function(d,b,a){var c=Backbone.View.extend(LoggableMixin).extend({HDAView:a.HDAEditView,tagName:"div",className:"history-panel",fxSpeed:"fast",datasetsSelector:".datasets-list",emptyMsgSelector:".empty-history-message",msgsSelector:".message-container",initialize:function(e){e=e||{};if(e.logger){this.logger=e.logger}this.log(this+".initialize:",e);this._setUpListeners();this.hdaViews={};this.indicator=new LoadingIndicator(this.$el);if(this.model){this._setUpWebStorage(e.initiallyExpanded,e.show_deleted,e.show_hidden);this._setUpModelEventHandlers()}if(e.onready){e.onready.call(this)}},_setUpListeners:function(){this.on("error",function(f,i,e,h,g){this.errorHandler(f,i,e,h,g)});this.on("loading-history",function(){this.showLoadingIndicator("loading history...")});this.on("loading-done",function(){this.hideLoadingIndicator()});this.once("rendered",function(){this.trigger("rendered:initial",this);return false});this.on("switched-history current-history new-history",function(){if(_.isEmpty(this.hdaViews)){this.trigger("empty-history",this)}});if(this.logger){this.on("all",function(e){this.log(this+"",arguments)},this)}},errorHandler:function(g,j,f,i,h){var e=this._parseErrorMessage(g,j,f,i,h);if(j&&j.status===0&&j.readyState===0){}else{if(j&&j.status===502){}else{if(!this.$el.find(this.msgsSelector).is(":visible")){this.once("rendered",function(){this.displayMessage("error",e.message,e.details)})}else{this.displayMessage("error",e.message,e.details)}}}},_parseErrorMessage:function(h,l,g,k,j){var f=Galaxy.currUser,e={message:this._bePolite(k),details:{user:(f instanceof User)?(f.toJSON()):(f+""),source:(h instanceof Backbone.Model)?(h.toJSON()):(h+""),xhr:l,options:(l)?(_.omit(g,"xhr")):(g)}};_.extend(e.details,j||{});if(l&&_.isFunction(l.getAllResponseHeaders)){var i=l.getAllResponseHeaders();i=_.compact(i.split("\n"));i=_.map(i,function(m){return m.split(": ")});e.details.xhr.responseHeaders=_.object(i)}return e},_bePolite:function(e){e=e||_l("An error occurred while getting updates from the server");return e+". "+_l("Please contact a Galaxy administrator if the problem persists.")},loadCurrentHistory:function(f){var e=this;return this.loadHistoryWithHDADetails("current",f).then(function(h,g){e.trigger("current-history",e)})},switchToHistory:function(h,g){var e=this,f=function(){return jQuery.post(galaxy_config.root+"api/histories/"+h+"/set_as_current")};return this.loadHistoryWithHDADetails(h,g,f).then(function(j,i){e.trigger("switched-history",e)})},createNewHistory:function(g){if(!Galaxy||!Galaxy.currUser||Galaxy.currUser.isAnonymous()){this.displayMessage("error",_l("You must be logged in to create histories"));return $.when()}var e=this,f=function(){return jQuery.post(galaxy_config.root+"api/histories",{current:true})};return this.loadHistory(undefined,g,f).then(function(i,h){e.trigger("new-history",e)})},loadHistoryWithHDADetails:function(h,g,f,j){var e=this,i=function(k){return e.getExpandedHdaIds(k.id)};return this.loadHistory(h,g,f,j,i)},loadHistory:function(h,g,f,k,i){this.trigger("loading-history",this);g=g||{};var e=this;var j=d.History.getHistoryData(h,{historyFn:f,hdaFn:k,hdaDetailIds:g.initiallyExpanded||i});return this._loadHistoryFromXHR(j,g).fail(function(n,l,m){e.trigger("error",e,n,g,_l("An error was encountered while "+l),{historyId:h,history:m||{}})}).always(function(){e.trigger("loading-done",e)})},_loadHistoryFromXHR:function(g,f){var e=this;g.then(function(h,i){e.setModel(h,i,f)});g.fail(function(i,h){e.render()});return g},setModel:function(g,e,f){f=f||{};if(this.model){this.model.clearUpdateTimeout();this.stopListening(this.model);this.stopListening(this.model.hdas)}this.hdaViews={};if(Galaxy&&Galaxy.currUser){g.user=Galaxy.currUser.toJSON()}this.model=new d.History(g,e,f);this._setUpWebStorage(f.initiallyExpanded,f.show_deleted,f.show_hidden);this._setUpModelEventHandlers();this.trigger("new-model",this);this.render();return this},refreshHdas:function(f,e){if(this.model){return this.model.refresh(f,e)}return $.when()},_setUpWebStorage:function(f,e,g){this.storage=new PersistentStorage(this._getStorageKey(this.model.get("id")),{expandedHdas:{},show_deleted:false,show_hidden:false});this.log(this+" (prev) storage:",JSON.stringify(this.storage.get(),null,2));if(f){this.storage.set("exandedHdas",f)}if((e===true)||(e===false)){this.storage.set("show_deleted",e)}if((g===true)||(g===false)){this.storage.set("show_hidden",g)}this.show_deleted=this.storage.get("show_deleted");this.show_hidden=this.storage.get("show_hidden");this.trigger("new-storage",this.storage,this);this.log(this+" (init'd) storage:",this.storage.get())},_getStorageKey:function(e){if(!e){throw new Error("_getStorageKey needs valid id: "+e)}return("history:"+e)},clearWebStorage:function(){for(var e in sessionStorage){if(e.indexOf("history:")===0){sessionStorage.removeItem(e)}}},getStoredOptions:function(f){if(!f||f==="current"){return(this.storage)?(this.storage.get()):({})}var e=sessionStorage.getItem(this._getStorageKey(f));return(e===null)?({}):(JSON.parse(e))},getExpandedHdaIds:function(e){var f=this.getStoredOptions(e).expandedHdas;return((_.isEmpty(f))?([]):(_.keys(f)))},_setUpModelEventHandlers:function(){this.model.on("error error:hdas",function(f,h,e,g){this.errorHandler(f,h,e,g)},this);this.model.on("change:nice_size",this.updateHistoryDiskSize,this);if(Galaxy&&Galaxy.quotaMeter){this.listenTo(this.model,"change:nice_size",function(){Galaxy.quotaMeter.update()})}this.model.hdas.on("add",this.addHdaView,this);this.model.hdas.on("change:deleted",this.handleHdaDeletionChange,this);this.model.hdas.on("change:visible",this.handleHdaVisibleChange,this);this.model.hdas.on("change:purged",function(e){this.model.fetch()},this);this.model.hdas.on("state:ready",function(f,g,e){if((!f.get("visible"))&&(!this.storage.get("show_hidden"))){this.removeHdaView(this.hdaViews[f.id])}},this)},addHdaView:function(h){this.log("add."+this,h);var f=this;if(!h.isVisible(this.storage.get("show_deleted"),this.storage.get("show_hidden"))){return}$({}).queue([function g(j){var i=f.$el.find(f.emptyMsgSelector);if(i.is(":visible")){i.fadeOut(f.fxSpeed,j)}else{j()}},function e(j){f.scrollToTop();var i=f.$el.find(f.datasetsSelector);f.createHdaView(h).$el.hide().prependTo(i).slideDown(f.fxSpeed)}])},createHdaView:function(g){var f=g.get("id"),e=this.storage.get("expandedHdas").get(f),h=new this.HDAView({model:g,expanded:e,hasUser:this.model.hasUser(),logger:this.logger});this._setUpHdaListeners(h);this.hdaViews[f]=h;return h.render()},_setUpHdaListeners:function(f){var e=this;f.on("body-expanded",function(g){e.storage.get("expandedHdas").set(g,true)});f.on("body-collapsed",function(g){e.storage.get("expandedHdas").deleteKey(g)});f.on("error",function(h,j,g,i){e.errorHandler(h,j,g,i)})},handleHdaDeletionChange:function(e){if(e.get("deleted")&&!this.storage.get("show_deleted")){this.removeHdaView(this.hdaViews[e.id])}},handleHdaVisibleChange:function(e){if(e.hidden()&&!this.storage.get("show_hidden")){this.removeHdaView(this.hdaViews[e.id])}},removeHdaView:function(f){if(!f){return}var e=this;f.$el.fadeOut(e.fxSpeed,function(){f.off();f.remove();delete e.hdaViews[f.model.id];if(_.isEmpty(e.hdaViews)){e.$el.find(e.emptyMsgSelector).fadeIn(e.fxSpeed,function(){e.trigger("empty-history",e)})}})},render:function(g){var e=this,f;if(this.model){f=this.renderModel()}else{f=this.renderWithoutModel()}$(e).queue("fx",[function(h){if(e.$el.is(":visible")){e.$el.fadeOut(e.fxSpeed,h)}else{h()}},function(h){e.$el.empty();if(f){e.$el.append(f.children())}e.$el.fadeIn(e.fxSpeed,h)},function(h){if(g){g.call(this)}e.trigger("rendered",this)}]);return this},renderWithoutModel:function(){var e=$("<div/>"),f=$("<div/>").addClass("message-container").css({"margin-left":"4px","margin-right":"4px"});return e.append(f)},renderModel:function(){var e=$("<div/>");if(!Galaxy||!Galaxy.currUser||Galaxy.currUser.isAnonymous()){e.append(c.templates.anonHistoryPanel(this.model.toJSON()))}else{e.append(c.templates.historyPanel(this.model.toJSON()));this._renderTags(e);this._renderAnnotation(e)}this._setUpBehaviours(e);if(!this.model.hdas.length||!this.renderHdas(e.find(this.datasetsSelector))){e.find(this.emptyMsgSelector).show()}return e},_renderTags:function(e){this.tagsEditor=new TagsEditor({model:this.model,el:e.find(".history-controls .tags-display"),onshowFirstTime:function(){this.render()},$activator:faIconButton({title:_l("Edit history tags"),classes:"history-tag-btn",faIcon:"fa-tags"}).appendTo(e.find(".history-secondary-actions"))})},_renderAnnotation:function(e){this.annotationEditor=new AnnotationEditor({model:this.model,el:e.find(".history-controls .annotation-display"),onshowFirstTime:function(){this.render()},$activator:faIconButton({title:_l("Edit history tags"),classes:"history-annotate-btn",faIcon:"fa-comment"}).appendTo(e.find(".history-secondary-actions"))})},_setUpBehaviours:function(e){e=e||this.$el;e.find("[title]").tooltip({placement:"bottom"});if(!this.model||!Galaxy.currUser||Galaxy.currUser.isAnonymous()){return}var f=this;e.find(".history-name").make_text_editable({on_finish:function(g){e.find(".history-name").text(g);f.model.save({name:g}).fail(function(){e.find(".history-name").text(f.model.previous("name"))})}})},renderHdas:function(f){this.hdaViews={};var e=this,g=this.model.hdas.getVisible(this.storage.get("show_deleted"),this.storage.get("show_hidden"));_.each(g,function(h){f.prepend(e.createHdaView(h).$el)});return g.length},events:{"click .message-container":"clearMessages"},updateHistoryDiskSize:function(){this.$el.find(".history-size").text(this.model.get("nice_size"))},collapseAllHdaBodies:function(){_.each(this.hdaViews,function(e){e.toggleBodyVisibility(null,false)});this.storage.set("expandedHdas",{})},toggleShowDeleted:function(){this.storage.set("show_deleted",!this.storage.get("show_deleted"));this.render();return this.storage.get("show_deleted")},toggleShowHidden:function(){this.storage.set("show_hidden",!this.storage.get("show_hidden"));this.render();return this.storage.get("show_hidden")},showSelect:function(e){_.each(this.hdaViews,function(f){f.showSelect(e)})},hideSelect:function(e){_.each(this.hdaViews,function(f){f.hideSelect(e)})},getSelectedHdaViews:function(){return _.filter(this.hdaViews,function(e){return e.selected})},showLoadingIndicator:function(f,e,g){e=(e!==undefined)?(e):(this.fxSpeed);if(!this.indicator){this.indicator=new LoadingIndicator(this.$el,this.$el.parent())}if(!this.$el.is(":visible")){this.indicator.show(0,g)}else{this.$el.fadeOut(e);this.indicator.show(f,e,g)}},hideLoadingIndicator:function(e,f){e=(e!==undefined)?(e):(this.fxSpeed);if(this.indicator){this.indicator.hide(e,f)}},displayMessage:function(j,k,i){var g=this;this.scrollToTop();var h=this.$el.find(this.msgsSelector),e=$("<div/>").addClass(j+"message").html(k);if(!_.isEmpty(i)){var f=$('<a href="javascript:void(0)">Details</a>').click(function(){Galaxy.modal.show(g.messageToModalOptions(j,k,i));return false});e.append(" ",f)}return h.html(e)},messageToModalOptions:function(i,k,h){var e=this,j=$("<div/>"),g={title:"Details"};function f(l){l=_.omit(l,_.functions(l));return["<table>",_.map(l,function(n,m){n=(_.isObject(n))?(f(n)):(n);return'<tr><td style="vertical-align: top; color: grey">'+m+'</td><td style="padding-left: 8px">'+n+"</td></tr>"}).join(""),"</table>"].join("")}if(_.isObject(h)){g.body=j.append(f(h))}else{g.body=j.html(h)}g.buttons={Ok:function(){Galaxy.modal.hide();e.clearMessages()}};return g},clearMessages:function(){var e=this.$el.find(this.msgsSelector);e.empty()},scrollPosition:function(){return this.$el.parent().scrollTop()},scrollTo:function(e){this.$el.parent().scrollTop(e)},scrollToTop:function(){this.$el.parent().scrollTop(0);return this},scrollIntoView:function(f,g){if(!g){this.$el.parent().parent().scrollTop(f);return this}var e=window,h=this.$el.parent().parent(),j=$(e).innerHeight(),i=(j/2)-(g/2);$(h).scrollTop(f-i);return this},scrollToId:function(f){if((!f)||(!this.hdaViews[f])){return this}var e=this.hdaViews[f].$el;this.scrollIntoView(e.offset().top,e.outerHeight());return this},scrollToHid:function(e){var f=this.model.hdas.getByHid(e);if(!f){return this}return this.scrollToId(f.id)},connectToQuotaMeter:function(e){if(!e){return this}this.listenTo(e,"quota:over",this.showQuotaMessage);this.listenTo(e,"quota:under",this.hideQuotaMessage);this.on("rendered rendered:initial",function(){if(e&&e.isOverQuota()){this.showQuotaMessage()}});return this},showQuotaMessage:function(){var e=this.$el.find(".quota-message");if(e.is(":hidden")){e.slideDown(this.fxSpeed)}},hideQuotaMessage:function(){var e=this.$el.find(".quota-message");if(!e.is(":hidden")){e.slideUp(this.fxSpeed)}},connectToOptionsMenu:function(e){if(!e){return this}this.on("new-storage",function(g,f){if(e&&g){e.findItemByHtml(_l("Include Deleted Datasets")).checked=g.get("show_deleted");e.findItemByHtml(_l("Include Hidden Datasets")).checked=g.get("show_hidden")}});return this},toString:function(){return"HistoryPanel("+((this.model)?(this.model.get("name")):(""))+")"}});c.templates={historyPanel:Handlebars.templates["template-history-historyPanel"],anonHistoryPanel:Handlebars.templates["template-history-historyPanel-anon"]};return{HistoryPanel:c}}); \ No newline at end of file +define(["mvc/history/history-model","mvc/dataset/hda-base","mvc/dataset/hda-edit"],function(d,b,a){var c=Backbone.View.extend(LoggableMixin).extend({HDAView:a.HDAEditView,tagName:"div",className:"history-panel",fxSpeed:"fast",datasetsSelector:".datasets-list",emptyMsgSelector:".empty-history-message",msgsSelector:".message-container",initialize:function(e){e=e||{};if(e.logger){this.logger=e.logger}this.log(this+".initialize:",e);this._setUpListeners();this.hdaViews={};this.indicator=new LoadingIndicator(this.$el);this.filters=[];if(this.model){this._setUpWebStorage(e.initiallyExpanded,e.show_deleted,e.show_hidden);this._setUpModelEventHandlers()}if(e.onready){e.onready.call(this)}},_setUpListeners:function(){this.on("error",function(f,i,e,h,g){this.errorHandler(f,i,e,h,g)});this.on("loading-history",function(){this.showLoadingIndicator("loading history...")});this.on("loading-done",function(){this.hideLoadingIndicator()});this.once("rendered",function(){this.trigger("rendered:initial",this);return false});this.on("switched-history current-history new-history",function(){if(_.isEmpty(this.hdaViews)){this.trigger("empty-history",this)}});if(this.logger){this.on("all",function(e){this.log(this+"",arguments)},this)}},errorHandler:function(g,j,f,i,h){var e=this._parseErrorMessage(g,j,f,i,h);if(j&&j.status===0&&j.readyState===0){}else{if(j&&j.status===502){}else{if(!this.$el.find(this.msgsSelector).is(":visible")){this.once("rendered",function(){this.displayMessage("error",e.message,e.details)})}else{this.displayMessage("error",e.message,e.details)}}}},_parseErrorMessage:function(h,l,g,k,j){var f=Galaxy.currUser,e={message:this._bePolite(k),details:{user:(f instanceof User)?(f.toJSON()):(f+""),source:(h instanceof Backbone.Model)?(h.toJSON()):(h+""),xhr:l,options:(l)?(_.omit(g,"xhr")):(g)}};_.extend(e.details,j||{});if(l&&_.isFunction(l.getAllResponseHeaders)){var i=l.getAllResponseHeaders();i=_.compact(i.split("\n"));i=_.map(i,function(m){return m.split(": ")});e.details.xhr.responseHeaders=_.object(i)}return e},_bePolite:function(e){e=e||_l("An error occurred while getting updates from the server");return e+". "+_l("Please contact a Galaxy administrator if the problem persists.")},loadCurrentHistory:function(f){var e=this;return this.loadHistoryWithHDADetails("current",f).then(function(h,g){e.trigger("current-history",e)})},switchToHistory:function(h,g){var e=this,f=function(){return jQuery.post(galaxy_config.root+"api/histories/"+h+"/set_as_current")};return this.loadHistoryWithHDADetails(h,g,f).then(function(j,i){e.trigger("switched-history",e)})},createNewHistory:function(g){if(!Galaxy||!Galaxy.currUser||Galaxy.currUser.isAnonymous()){this.displayMessage("error",_l("You must be logged in to create histories"));return $.when()}var e=this,f=function(){return jQuery.post(galaxy_config.root+"api/histories",{current:true})};return this.loadHistory(undefined,g,f).then(function(i,h){e.trigger("new-history",e)})},loadHistoryWithHDADetails:function(h,g,f,j){var e=this,i=function(k){return e.getExpandedHdaIds(k.id)};return this.loadHistory(h,g,f,j,i)},loadHistory:function(h,g,f,k,i){this.trigger("loading-history",this);g=g||{};var e=this;var j=d.History.getHistoryData(h,{historyFn:f,hdaFn:k,hdaDetailIds:g.initiallyExpanded||i});return this._loadHistoryFromXHR(j,g).fail(function(n,l,m){e.trigger("error",e,n,g,_l("An error was encountered while "+l),{historyId:h,history:m||{}})}).always(function(){e.trigger("loading-done",e)})},_loadHistoryFromXHR:function(g,f){var e=this;g.then(function(h,i){e.setModel(h,i,f)});g.fail(function(i,h){e.render()});return g},setModel:function(g,e,f){f=f||{};if(this.model){this.model.clearUpdateTimeout();this.stopListening(this.model);this.stopListening(this.model.hdas)}this.hdaViews={};if(Galaxy&&Galaxy.currUser){g.user=Galaxy.currUser.toJSON()}this.model=new d.History(g,e,f);this._setUpWebStorage(f.initiallyExpanded,f.show_deleted,f.show_hidden);this._setUpModelEventHandlers();this.trigger("new-model",this);this.render();return this},refreshHdas:function(f,e){if(this.model){return this.model.refresh(f,e)}return $.when()},_setUpWebStorage:function(f,e,g){this.storage=new PersistentStorage(this._getStorageKey(this.model.get("id")),{expandedHdas:{},show_deleted:false,show_hidden:false});this.log(this+" (prev) storage:",JSON.stringify(this.storage.get(),null,2));if(f){this.storage.set("exandedHdas",f)}if((e===true)||(e===false)){this.storage.set("show_deleted",e)}if((g===true)||(g===false)){this.storage.set("show_hidden",g)}this.show_deleted=this.storage.get("show_deleted");this.show_hidden=this.storage.get("show_hidden");this.trigger("new-storage",this.storage,this);this.log(this+" (init'd) storage:",this.storage.get())},_getStorageKey:function(e){if(!e){throw new Error("_getStorageKey needs valid id: "+e)}return("history:"+e)},clearWebStorage:function(){for(var e in sessionStorage){if(e.indexOf("history:")===0){sessionStorage.removeItem(e)}}},getStoredOptions:function(f){if(!f||f==="current"){return(this.storage)?(this.storage.get()):({})}var e=sessionStorage.getItem(this._getStorageKey(f));return(e===null)?({}):(JSON.parse(e))},getExpandedHdaIds:function(e){var f=this.getStoredOptions(e).expandedHdas;return((_.isEmpty(f))?([]):(_.keys(f)))},_setUpModelEventHandlers:function(){this.model.on("error error:hdas",function(f,h,e,g){this.errorHandler(f,h,e,g)},this);this.model.on("change:nice_size",this.updateHistoryDiskSize,this);if(Galaxy&&Galaxy.quotaMeter){this.listenTo(this.model,"change:nice_size",function(){Galaxy.quotaMeter.update()})}this.model.hdas.on("add",this.addHdaView,this);this.model.hdas.on("change:deleted",this.handleHdaDeletionChange,this);this.model.hdas.on("change:visible",this.handleHdaVisibleChange,this);this.model.hdas.on("change:purged",function(e){this.model.fetch()},this);this.model.hdas.on("state:ready",function(f,g,e){if((!f.get("visible"))&&(!this.storage.get("show_hidden"))){this.removeHdaView(this.hdaViews[f.id])}},this)},addHdaView:function(h){this.log("add."+this,h);var f=this;if(!h.isVisible(this.storage.get("show_deleted"),this.storage.get("show_hidden"))){return}$({}).queue([function g(j){var i=f.$el.find(f.emptyMsgSelector);if(i.is(":visible")){i.fadeOut(f.fxSpeed,j)}else{j()}},function e(j){f.scrollToTop();var i=f.$el.find(f.datasetsSelector);f.createHdaView(h).$el.hide().prependTo(i).slideDown(f.fxSpeed)}])},createHdaView:function(g){var f=g.get("id"),e=this.storage.get("expandedHdas").get(f),h=new this.HDAView({model:g,expanded:e,hasUser:this.model.hasUser(),logger:this.logger});this._setUpHdaListeners(h);this.hdaViews[f]=h;return h.render()},_setUpHdaListeners:function(f){var e=this;f.on("body-expanded",function(g){e.storage.get("expandedHdas").set(g,true)});f.on("body-collapsed",function(g){e.storage.get("expandedHdas").deleteKey(g)});f.on("error",function(h,j,g,i){e.errorHandler(h,j,g,i)})},handleHdaDeletionChange:function(e){if(e.get("deleted")&&!this.storage.get("show_deleted")){this.removeHdaView(this.hdaViews[e.id])}},handleHdaVisibleChange:function(e){if(e.hidden()&&!this.storage.get("show_hidden")){this.removeHdaView(this.hdaViews[e.id])}},removeHdaView:function(f){if(!f){return}var e=this;f.$el.fadeOut(e.fxSpeed,function(){f.off();f.remove();delete e.hdaViews[f.model.id];if(_.isEmpty(e.hdaViews)){e.$el.find(e.emptyMsgSelector).fadeIn(e.fxSpeed,function(){e.trigger("empty-history",e)})}})},render:function(g,h){g=(g===undefined)?(this.fxSpeed):(g);var e=this,f;if(this.model){f=this.renderModel()}else{f=this.renderWithoutModel()}$(e).queue("fx",[function(i){if(g&&e.$el.is(":visible")){e.$el.fadeOut(g,i)}else{i()}},function(i){e.$el.empty();if(f){e.$el.append(f.children())}i()},function(i){if(g&&!e.$el.is(":visible")){e.$el.fadeIn(g,i)}else{i()}},function(i){if(h){h.call(this)}e.trigger("rendered",this);i()}]);return this},renderWithoutModel:function(){var e=$("<div/>"),f=$("<div/>").addClass("message-container").css({"margin-left":"4px","margin-right":"4px"});return e.append(f)},renderModel:function(){var e=$("<div/>");if(!Galaxy||!Galaxy.currUser||Galaxy.currUser.isAnonymous()){e.append(c.templates.anonHistoryPanel(this.model.toJSON()))}else{e.append(c.templates.historyPanel(this.model.toJSON()));this._renderTags(e);this._renderAnnotation(e)}this._setUpBehaviours(e);this.renderHdas(e);return e},_renderTags:function(e){this.tagsEditor=new TagsEditor({model:this.model,el:e.find(".history-controls .tags-display"),onshowFirstTime:function(){this.render()},$activator:faIconButton({title:_l("Edit history tags"),classes:"history-tag-btn",faIcon:"fa-tags"}).appendTo(e.find(".history-secondary-actions"))})},_renderAnnotation:function(e){this.annotationEditor=new AnnotationEditor({model:this.model,el:e.find(".history-controls .annotation-display"),onshowFirstTime:function(){this.render()},$activator:faIconButton({title:_l("Edit history tags"),classes:"history-annotate-btn",faIcon:"fa-comment"}).appendTo(e.find(".history-secondary-actions"))})},_setUpBehaviours:function(e){e=e||this.$el;e.find("[title]").tooltip({placement:"bottom"});if(!this.model||!Galaxy.currUser||Galaxy.currUser.isAnonymous()){return}var f=this;e.find(".history-name").make_text_editable({on_finish:function(g){e.find(".history-name").text(g);f.model.save({name:g}).fail(function(){e.find(".history-name").text(f.model.previous("name"))})}})},renderHdas:function(g){g=g||this.$el;this.hdaViews={};var f=this,e=g.find(this.datasetsSelector),h=this.model.hdas.getVisible(this.storage.get("show_deleted"),this.storage.get("show_hidden"),this.filters);e.empty();if(h.length){h.each(function(i){e.prepend(f.createHdaView(i).$el)});g.find(this.emptyMsgSelector).hide()}else{g.find(this.emptyMsgSelector).show()}return this.hdaViews},events:{"click .message-container":"clearMessages"},updateHistoryDiskSize:function(){this.$el.find(".history-size").text(this.model.get("nice_size"))},collapseAllHdaBodies:function(){_.each(this.hdaViews,function(e){e.toggleBodyVisibility(null,false)});this.storage.set("expandedHdas",{})},toggleShowDeleted:function(){this.storage.set("show_deleted",!this.storage.get("show_deleted"));this.render();return this.storage.get("show_deleted")},toggleShowHidden:function(){this.storage.set("show_hidden",!this.storage.get("show_hidden"));this.render();return this.storage.get("show_hidden")},renderSearchControls:function(){var e=this;e.model.hdas.fetchAllDetails({silent:true});function g(h){e.searchFor=h;e.filters=[function(i){return i.matchesAll(e.searchFor)}];e.trigger("search:searching",h,e);e.renderHdas()}function f(){e.searchFor="";e.filters=[];e.trigger("search:clear",e);e.renderHdas()}return searchInput({initialVal:e.searchFor,name:"history-search",placeholder:"search datasets",classes:"history-search",onsearch:g,onclear:f}).addClass("history-search-controls").css("padding","0px 0px 8px 0px")},toggleSearchControls:function(){var e=this.$el.find(".history-search-controls");if(!e.size()){e=this.renderSearchControls().hide();this.$el.find(".history-title").before(e)}e.slideToggle(this.fxSpeed)},showSelect:function(e){_.each(this.hdaViews,function(f){f.showSelect(e)})},hideSelect:function(e){_.each(this.hdaViews,function(f){f.hideSelect(e)})},getSelectedHdaViews:function(){return _.filter(this.hdaViews,function(e){return e.selected})},showLoadingIndicator:function(f,e,g){e=(e!==undefined)?(e):(this.fxSpeed);if(!this.indicator){this.indicator=new LoadingIndicator(this.$el,this.$el.parent())}if(!this.$el.is(":visible")){this.indicator.show(0,g)}else{this.$el.fadeOut(e);this.indicator.show(f,e,g)}},hideLoadingIndicator:function(e,f){e=(e!==undefined)?(e):(this.fxSpeed);if(this.indicator){this.indicator.hide(e,f)}},displayMessage:function(j,k,i){var g=this;this.scrollToTop();var h=this.$el.find(this.msgsSelector),e=$("<div/>").addClass(j+"message").html(k);if(!_.isEmpty(i)){var f=$('<a href="javascript:void(0)">Details</a>').click(function(){Galaxy.modal.show(g.messageToModalOptions(j,k,i));return false});e.append(" ",f)}return h.html(e)},messageToModalOptions:function(i,k,h){var e=this,j=$("<div/>"),g={title:"Details"};function f(l){l=_.omit(l,_.functions(l));return["<table>",_.map(l,function(n,m){n=(_.isObject(n))?(f(n)):(n);return'<tr><td style="vertical-align: top; color: grey">'+m+'</td><td style="padding-left: 8px">'+n+"</td></tr>"}).join(""),"</table>"].join("")}if(_.isObject(h)){g.body=j.append(f(h))}else{g.body=j.html(h)}g.buttons={Ok:function(){Galaxy.modal.hide();e.clearMessages()}};return g},clearMessages:function(){var e=this.$el.find(this.msgsSelector);e.empty()},scrollPosition:function(){return this.$el.parent().scrollTop()},scrollTo:function(e){this.$el.parent().scrollTop(e)},scrollToTop:function(){this.$el.parent().scrollTop(0);return this},scrollIntoView:function(f,g){if(!g){this.$el.parent().parent().scrollTop(f);return this}var e=window,h=this.$el.parent().parent(),j=$(e).innerHeight(),i=(j/2)-(g/2);$(h).scrollTop(f-i);return this},scrollToId:function(f){if((!f)||(!this.hdaViews[f])){return this}var e=this.hdaViews[f].$el;this.scrollIntoView(e.offset().top,e.outerHeight());return this},scrollToHid:function(e){var f=this.model.hdas.getByHid(e);if(!f){return this}return this.scrollToId(f.id)},connectToQuotaMeter:function(e){if(!e){return this}this.listenTo(e,"quota:over",this.showQuotaMessage);this.listenTo(e,"quota:under",this.hideQuotaMessage);this.on("rendered rendered:initial",function(){if(e&&e.isOverQuota()){this.showQuotaMessage()}});return this},showQuotaMessage:function(){var e=this.$el.find(".quota-message");if(e.is(":hidden")){e.slideDown(this.fxSpeed)}},hideQuotaMessage:function(){var e=this.$el.find(".quota-message");if(!e.is(":hidden")){e.slideUp(this.fxSpeed)}},connectToOptionsMenu:function(e){if(!e){return this}this.on("new-storage",function(g,f){if(e&&g){e.findItemByHtml(_l("Include Deleted Datasets")).checked=g.get("show_deleted");e.findItemByHtml(_l("Include Hidden Datasets")).checked=g.get("show_hidden")}});return this},toString:function(){return"HistoryPanel("+((this.model)?(this.model.get("name")):(""))+")"}});c.templates={historyPanel:Handlebars.templates["template-history-historyPanel"],anonHistoryPanel:Handlebars.templates["template-history-historyPanel-anon"]};return{HistoryPanel:c}}); \ No newline at end of file diff -r 253f888144aaa4ae4eadaf15de493b3144fc5cd3 -r a39f452a7f657f3b43dce87d08e143ed6f02867e static/scripts/packed/mvc/ui.js --- a/static/scripts/packed/mvc/ui.js +++ b/static/scripts/packed/mvc/ui.js @@ -1,1 +1,1 @@ -var IconButton=Backbone.Model.extend({defaults:{title:"",icon_class:"",on_click:null,menu_options:null,is_menu_button:true,id:null,href:null,target:null,enabled:true,visible:true,tooltip_config:{}}});var IconButtonView=Backbone.View.extend({initialize:function(){this.model.attributes.tooltip_config={placement:"bottom"};this.model.bind("change",this.render,this)},render:function(){this.$el.tooltip("hide");var a=this.template(this.model.toJSON());a.tooltip(this.model.get("tooltip_config"));this.$el.replaceWith(a);this.setElement(a);return this},events:{click:"click"},click:function(a){if(_.isFunction(this.model.get("on_click"))){this.model.get("on_click")(a);return false}return true},template:function(b){var a='title="'+b.title+'" class="icon-button';if(b.is_menu_button){a+=" menu-button"}a+=" "+b.icon_class;if(!b.enabled){a+="_disabled"}a+='"';if(b.id){a+=' id="'+b.id+'"'}a+=' href="'+b.href+'"';if(b.target){a+=' target="'+b.target+'"'}if(!b.visible){a+=' style="display: none;"'}if(b.enabled){a="<a "+a+"/>"}else{a="<span "+a+"/>"}return $(a)}});var IconButtonCollection=Backbone.Collection.extend({model:IconButton});var IconButtonMenuView=Backbone.View.extend({tagName:"div",initialize:function(){this.render()},render:function(){var a=this;this.collection.each(function(d){var b=$("<a/>").attr("href","javascript:void(0)").attr("title",d.attributes.title).addClass("icon-button menu-button").addClass(d.attributes.icon_class).appendTo(a.$el).click(d.attributes.on_click);if(d.attributes.tooltip_config){b.tooltip(d.attributes.tooltip_config)}var c=d.get("options");if(c){make_popupmenu(b,c)}});return this}});var create_icon_buttons_menu=function(b,a){if(!a){a={}}var c=new IconButtonCollection(_.map(b,function(d){return new IconButton(_.extend(d,a))}));return new IconButtonMenuView({collection:c})};var Grid=Backbone.Collection.extend({});var GridView=Backbone.View.extend({});var PopupMenu=Backbone.View.extend({initialize:function(b,a){this.$button=b||$("<div/>");this.options=a||[];var c=this;this.$button.click(function(d){c._renderAndShow(d);return false})},_renderAndShow:function(a){this.render();this.$el.appendTo("body");this.$el.css(this._getShownPosition(a));this._setUpCloseBehavior();this.$el.show()},render:function(){this.$el.addClass("popmenu-wrapper").hide().css({position:"absolute"}).html(this.template(this.$button.attr("id"),this.options));if(this.options.length){var a=this;this.$el.find("li").each(function(c,b){var d=a.options[c];if(d.func){$(this).children("a.popupmenu-option").click(function(e){d.func.call(a,e,d)})}})}return this},template:function(b,a){return['<ul id="',b,'-menu" class="dropdown-menu">',this._templateOptions(a),"</ul>"].join("")},_templateOptions:function(a){if(!a.length){return"<li>(no options)</li>"}return _.map(a,function(d){if(d.divider){return'<li class="divider"></li>'}else{if(d.header){return['<li class="head"><a href="javascript:void(0);">',d.html,"</a></li>"].join("")}}var c=d.href||"javascript:void(0);",e=(d.target)?(' target="'+d.target+'"'):(""),b=(d.checked)?('<span class="fa fa-check"></span>'):("");return['<li><a class="popupmenu-option" href="',c,'"',e,">",b,d.html,"</a></li>"].join("")}).join("")},_getShownPosition:function(b){var c=this.$el.width();var a=b.pageX-c/2;a=Math.min(a,$(document).scrollLeft()+$(window).width()-c-5);a=Math.max(a,$(document).scrollLeft()+5);return{top:b.pageY,left:a}},_setUpCloseBehavior:function(){var b=this;var a=function(c){c.one("click.close_popup",function(){b.remove()})};a($(window.document));a($(window.top.document));_.each(window.top.frames,function(c){a($(c.document))})},addItem:function(b,a){a=(a>=0)?a:this.options.length;this.options.splice(a,0,b);return this},removeItem:function(a){if(a>=0){this.options.splice(a,1)}return this},findIndexByHtml:function(b){for(var a=0;a<this.options.length;a++){if(_.has(this.options[a],"html")&&(this.options[a].html===b)){return a}}return null},findItemByHtml:function(a){return this.options[(this.findIndexByHtml(a))]},toString:function(){return"PopupMenu"}});PopupMenu.make_popupmenu=function(b,c){var a=[];_.each(c,function(f,d){var e={html:d};if(f===null){e.header=true}else{if(jQuery.type(f)==="function"){e.func=f}}a.push(e)});return new PopupMenu($(b),a)};PopupMenu.convertLinksToOptions=function(c,a){c=$(c);a=a||"a";var b=[];c.find(a).each(function(g,e){var f={},d=$(g);f.html=d.text();if(d.attr("href")){var j=d.attr("href"),k=d.attr("target"),h=d.attr("confirm");f.func=function(){if((h)&&(!confirm(h))){return}switch(k){case"_parent":window.parent.location=j;break;case"_top":window.top.location=j;break;default:window.location=j}}}b.push(f)});return b};PopupMenu.fromExistingDom=function(d,c,a){d=$(d);c=$(c);var b=PopupMenu.convertLinksToOptions(c,a);c.remove();return new PopupMenu(d,b)};PopupMenu.make_popup_menus=function(c,b,d){c=c||document;b=b||"div[popupmenu]";d=d||function(e,f){return"#"+e.attr("popupmenu")};var a=[];$(c).find(b).each(function(){var e=$(this),f=$(c).find(d(e,c));a.push(PopupMenu.fromDom(f,e));f.addClass("popup")});return a};var faIconButton=function(a){a=a||{};a.tooltipConfig=a.tooltipConfig||{placement:"bottom"};a.classes=["icon-btn"].concat(a.classes||[]);if(a.disabled){a.classes.push("disabled")}var b=['<a class="',a.classes.join(" "),'"',((a.title)?(' title="'+a.title+'"'):("")),((a.target)?(' target="'+a.target+'"'):("")),' href="',((a.href)?(a.href):("javascript:void(0);")),'">','<span class="fa ',a.faIcon,'"></span>',"</a>"].join("");var c=$(b).tooltip(a.tooltipConfig);if(_.isFunction(a.onclick)){c.click(a.onclick)}return c}; \ No newline at end of file +var IconButton=Backbone.Model.extend({defaults:{title:"",icon_class:"",on_click:null,menu_options:null,is_menu_button:true,id:null,href:null,target:null,enabled:true,visible:true,tooltip_config:{}}});var IconButtonView=Backbone.View.extend({initialize:function(){this.model.attributes.tooltip_config={placement:"bottom"};this.model.bind("change",this.render,this)},render:function(){this.$el.tooltip("hide");var a=this.template(this.model.toJSON());a.tooltip(this.model.get("tooltip_config"));this.$el.replaceWith(a);this.setElement(a);return this},events:{click:"click"},click:function(a){if(_.isFunction(this.model.get("on_click"))){this.model.get("on_click")(a);return false}return true},template:function(b){var a='title="'+b.title+'" class="icon-button';if(b.is_menu_button){a+=" menu-button"}a+=" "+b.icon_class;if(!b.enabled){a+="_disabled"}a+='"';if(b.id){a+=' id="'+b.id+'"'}a+=' href="'+b.href+'"';if(b.target){a+=' target="'+b.target+'"'}if(!b.visible){a+=' style="display: none;"'}if(b.enabled){a="<a "+a+"/>"}else{a="<span "+a+"/>"}return $(a)}});var IconButtonCollection=Backbone.Collection.extend({model:IconButton});var IconButtonMenuView=Backbone.View.extend({tagName:"div",initialize:function(){this.render()},render:function(){var a=this;this.collection.each(function(d){var b=$("<a/>").attr("href","javascript:void(0)").attr("title",d.attributes.title).addClass("icon-button menu-button").addClass(d.attributes.icon_class).appendTo(a.$el).click(d.attributes.on_click);if(d.attributes.tooltip_config){b.tooltip(d.attributes.tooltip_config)}var c=d.get("options");if(c){make_popupmenu(b,c)}});return this}});var create_icon_buttons_menu=function(b,a){if(!a){a={}}var c=new IconButtonCollection(_.map(b,function(d){return new IconButton(_.extend(d,a))}));return new IconButtonMenuView({collection:c})};var Grid=Backbone.Collection.extend({});var GridView=Backbone.View.extend({});var PopupMenu=Backbone.View.extend({initialize:function(b,a){this.$button=b||$("<div/>");this.options=a||[];var c=this;this.$button.click(function(d){c._renderAndShow(d);return false})},_renderAndShow:function(a){this.render();this.$el.appendTo("body");this.$el.css(this._getShownPosition(a));this._setUpCloseBehavior();this.$el.show()},render:function(){this.$el.addClass("popmenu-wrapper").hide().css({position:"absolute"}).html(this.template(this.$button.attr("id"),this.options));if(this.options.length){var a=this;this.$el.find("li").each(function(c,b){var d=a.options[c];if(d.func){$(this).children("a.popupmenu-option").click(function(e){d.func.call(a,e,d)})}})}return this},template:function(b,a){return['<ul id="',b,'-menu" class="dropdown-menu">',this._templateOptions(a),"</ul>"].join("")},_templateOptions:function(a){if(!a.length){return"<li>(no options)</li>"}return _.map(a,function(d){if(d.divider){return'<li class="divider"></li>'}else{if(d.header){return['<li class="head"><a href="javascript:void(0);">',d.html,"</a></li>"].join("")}}var c=d.href||"javascript:void(0);",e=(d.target)?(' target="'+d.target+'"'):(""),b=(d.checked)?('<span class="fa fa-check"></span>'):("");return['<li><a class="popupmenu-option" href="',c,'"',e,">",b,d.html,"</a></li>"].join("")}).join("")},_getShownPosition:function(b){var c=this.$el.width();var a=b.pageX-c/2;a=Math.min(a,$(document).scrollLeft()+$(window).width()-c-5);a=Math.max(a,$(document).scrollLeft()+5);return{top:b.pageY,left:a}},_setUpCloseBehavior:function(){var b=this;var a=function(c){c.one("click.close_popup",function(){b.remove()})};a($(window.document));a($(window.top.document));_.each(window.top.frames,function(c){a($(c.document))})},addItem:function(b,a){a=(a>=0)?a:this.options.length;this.options.splice(a,0,b);return this},removeItem:function(a){if(a>=0){this.options.splice(a,1)}return this},findIndexByHtml:function(b){for(var a=0;a<this.options.length;a++){if(_.has(this.options[a],"html")&&(this.options[a].html===b)){return a}}return null},findItemByHtml:function(a){return this.options[(this.findIndexByHtml(a))]},toString:function(){return"PopupMenu"}});PopupMenu.make_popupmenu=function(b,c){var a=[];_.each(c,function(f,d){var e={html:d};if(f===null){e.header=true}else{if(jQuery.type(f)==="function"){e.func=f}}a.push(e)});return new PopupMenu($(b),a)};PopupMenu.convertLinksToOptions=function(c,a){c=$(c);a=a||"a";var b=[];c.find(a).each(function(g,e){var f={},d=$(g);f.html=d.text();if(d.attr("href")){var j=d.attr("href"),k=d.attr("target"),h=d.attr("confirm");f.func=function(){if((h)&&(!confirm(h))){return}switch(k){case"_parent":window.parent.location=j;break;case"_top":window.top.location=j;break;default:window.location=j}}}b.push(f)});return b};PopupMenu.fromExistingDom=function(d,c,a){d=$(d);c=$(c);var b=PopupMenu.convertLinksToOptions(c,a);c.remove();return new PopupMenu(d,b)};PopupMenu.make_popup_menus=function(c,b,d){c=c||document;b=b||"div[popupmenu]";d=d||function(e,f){return"#"+e.attr("popupmenu")};var a=[];$(c).find(b).each(function(){var e=$(this),f=$(c).find(d(e,c));a.push(PopupMenu.fromDom(f,e));f.addClass("popup")});return a};var faIconButton=function(a){a=a||{};a.tooltipConfig=a.tooltipConfig||{placement:"bottom"};a.classes=["icon-btn"].concat(a.classes||[]);if(a.disabled){a.classes.push("disabled")}var b=['<a class="',a.classes.join(" "),'"',((a.title)?(' title="'+a.title+'"'):("")),((a.target)?(' target="'+a.target+'"'):("")),' href="',((a.href)?(a.href):("javascript:void(0);")),'">','<span class="fa ',a.faIcon,'"></span>',"</a>"].join("");var c=$(b).tooltip(a.tooltipConfig);if(_.isFunction(a.onclick)){c.click(a.onclick)}return c};var searchInput=function(k){var a=27,h=13,i=$("<div/>"),b={initialVal:"",name:"search",placeholder:"search",classes:"",onclear:function(){},onsearch:function(l){},minSearchLen:0,escWillClear:true,oninit:function(){}};if(jQuery.type(k)==="object"){k=jQuery.extend(true,b,k)}function d(l){var m=$(this).parent().children("input");m.val("");m.trigger("clear:searchInput");k.onclear()}function j(m,l){$(this).trigger("search:searchInput",l);k.onsearch(l)}function c(){return['<input type="text" name="',k.name,'" placeholder="',k.placeholder,'" ','class="search-query ',k.classes,'" ',"/>"].join("")}function g(){return $(c()).css({width:"100%","padding-right":"24px"}).focus(function(l){$(this).select()}).keyup(function(m){if(m.which===a&&k.escWillClear){d.call(this,m)}else{var l=$(this).val();if((m.which===h)||(k.minSearchLen&&l.length>=k.minSearchLen)){j.call(this,m,l)}else{if(!l.length){d.call(this,m)}}}}).val(k.initialVal)}function f(){return'<span class="search-clear fa fa-times-circle"></span>'}function e(){return $(f()).css({position:"absolute",right:"15px","font-size":"1.4em","line-height":"23px",color:"grey"}).click(function(l){d.call(this,l)})}return i.append([g(),e()])}; \ No newline at end of file diff -r 253f888144aaa4ae4eadaf15de493b3144fc5cd3 -r a39f452a7f657f3b43dce87d08e143ed6f02867e static/style/blue/base.css --- a/static/style/blue/base.css +++ b/static/style/blue/base.css @@ -1363,7 +1363,7 @@ input[type="submit"].btn-primary,button.btn-primary{color:#fff;background-color:#5f6990;border-color:#0e0f15}input[type="submit"].btn-primary:hover,button.btn-primary:hover,input[type="submit"].btn-primary:focus,button.btn-primary:focus,input[type="submit"].btn-primary:active,button.btn-primary:active,input[type="submit"].btn-primary.active,button.btn-primary.active,.open .dropdown-toggleinput[type="submit"].btn-primary,.open .dropdown-togglebutton.btn-primary{color:#fff;background-color:#4e5777;border-color:#000} input[type="submit"].btn-primary:active,button.btn-primary:active,input[type="submit"].btn-primary.active,button.btn-primary.active,.open .dropdown-toggleinput[type="submit"].btn-primary,.open .dropdown-togglebutton.btn-primary{background-image:none} input[type="submit"].btn-primary.disabled,button.btn-primary.disabled,input[type="submit"].btn-primary[disabled],button.btn-primary[disabled],fieldset[disabled] input[type="submit"].btn-primary,fieldset[disabled] button.btn-primary,input[type="submit"].btn-primary.disabled:hover,button.btn-primary.disabled:hover,input[type="submit"].btn-primary[disabled]:hover,button.btn-primary[disabled]:hover,fieldset[disabled] input[type="submit"].btn-primary:hover,fieldset[disabled] button.btn-primary:hover,input[type="submit"].btn-primary.disabled:focus,button.btn-primary.disabled:focus,input[type="submit"].btn-primary[disabled]:focus,button.btn-primary[disabled]:focus,fieldset[disabled] input[type="submit"].btn-primary:focus,fieldset[disabled] button.btn-primary:focus,input[type="submit"].btn-primary.disabled:active,button.btn-primary.disabled:active,input[type="submit"].btn-primary[disabled]:active,button.btn-primary[disabled]:active,fieldset[disabled] input[type="submit"].btn-primary:active,fieldset[disabled] button.btn-primary:active,input[type="submit"].btn-primary.disabled.active,button.btn-primary.disabled.active,input[type="submit"].btn-primary[disabled].active,button.btn-primary[disabled].active,fieldset[disabled] input[type="submit"].btn-primary.active,fieldset[disabled] button.btn-primary.active{background-color:#5f6990;border-color:#0e0f15} -.search-query{display:inline-block;padding:4px;font-size:12px;line-height:1.428571429;color:#555;border:1px solid #aaa;padding-left:14px !important;padding-right:14px !important;margin-bottom:0;-webkit-border-radius:14px;-moz-border-radius:14px;border-radius:14px;max-width:auto} +.search-query{display:inline-block;padding:4px;font-size:12px;line-height:1.428571429;color:#555;border:1px solid #aaa;padding-left:14px !important;padding-right:14px;margin-bottom:0;-webkit-border-radius:14px;-moz-border-radius:14px;border-radius:14px;max-width:auto} .search-query:focus{border-color:rgba(24,132,218,0.8);-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6);outline:0;outline:thin dotted \9;} .search-spinner{position:absolute;display:none;right:6px;top:9px} #search-clear-btn{position:absolute;right:6px;top:5px;display:block;font-size:1.4em;text-decoration:none;color:#888;font-family:FontAwesome;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}#search-clear-btn:before{content:"\f057"} diff -r 253f888144aaa4ae4eadaf15de493b3144fc5cd3 -r a39f452a7f657f3b43dce87d08e143ed6f02867e static/style/src/less/base.less --- a/static/style/src/less/base.less +++ b/static/style/src/less/base.less @@ -721,7 +721,7 @@ color: @gray; border: 1px solid @input-border; padding-left: 14px !important; - padding-right: 14px !important; + padding-right: 14px; margin-bottom: 0; // remove the default margin on all inputs .border-radius(14px); max-width: auto; Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Functional tests for tool dependency XML verification. Functional tests for detecting missing env.sh files due to tool dependency installation errors.
by commits-noreply＠bitbucket.org 21 Nov '13

21 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/253f888144aa/ Changeset: 253f888144aa User: Dave Bouvier Date: 2013-11-21 17:12:13 Summary: Functional tests for tool dependency XML verification. Functional tests for detecting missing env.sh files due to tool dependency installation errors. Affected #: 6 files diff -r d448ed76962c8b36fde83e5cd8dec91e97c9d812 -r 253f888144aaa4ae4eadaf15de493b3144fc5cd3 test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -1,6 +1,15 @@ +import common +import string +import os +import re +import test_db_util +import simplejson +import logging +import time +import tempfile +import tarfile import galaxy.webapps.tool_shed.util.hgweb_config import galaxy.model as galaxy_model -import common, string, os, re, test_db_util, simplejson, logging, time import galaxy.util as util from tool_shed.util import shed_util_common as suc from base.twilltestcase import tc, from_json_string, TwillTestCase, security, urllib @@ -44,7 +53,7 @@ url = '/repository_review/create_component?operation=create' self.visit_url( url ) self.submit_form( 1, 'create_component_button', **kwd ) - + def browse_category( self, category, strings_displayed=[], strings_not_displayed=[] ): url = '/repository/browse_valid_categories?sort=name&operation=valid_repositories_by_category&id=%s' % \ self.security.encode_id( category.id ) @@ -479,7 +488,17 @@ else: string = string.replace( character, replacement ) return string - + + def export_capsule( self, repository ): + url = '/repository/export?repository_id=%s&changeset_revision=%s' % \ + ( self.security.encode_id( repository.id ), self.get_repository_tip( repository ) ) + self.visit_url( url ) + self.submit_form( 'export_repository', 'export_repository_button' ) + fd, capsule_filename = tempfile.mkstemp() + os.close( fd ) + file( capsule_filename, 'w' ).write( self.last_page() ) + return capsule_filename + def fill_review_form( self, review_contents_dict, strings_displayed=[], strings_not_displayed=[] ): kwd = dict() changed = False @@ -727,7 +746,17 @@ tc.fv( "user_access", "allow_push", '+%s' % username ) tc.submit( 'user_access_button' ) self.check_for_strings( post_submit_strings_displayed, post_submit_strings_not_displayed ) - + + def import_capsule( self, filename, strings_displayed=[], strings_not_displayed=[], + strings_displayed_after_submit=[], strings_not_displayed_after_submit=[] ): + url = '/repository/upload_capsule' + self.visit_url( url ) + tc.formfile( 'upload_capsule', 'file_data', filename ) + tc.submit( 'upload_capsule_button' ) + self.check_for_strings( strings_displayed, strings_not_displayed ) + self.submit_form( 'import_capsule', 'import_capsule_button' ) + self.check_for_strings( strings_displayed_after_submit, strings_not_displayed_after_submit ) + def import_workflow( self, repository, workflow_name, strings_displayed=[], strings_not_displayed=[] ): url = '/admin_toolshed/import_workflow?repository_id=%s&workflow_name=%s' % \ ( self.security.encode_id( repository.id ), tool_shed_encode( workflow_name ) ) diff -r d448ed76962c8b36fde83e5cd8dec91e97c9d812 -r 253f888144aaa4ae4eadaf15de493b3144fc5cd3 test/tool_shed/functional/test_0480_tool_dependency_xml_verification.py --- /dev/null +++ b/test/tool_shed/functional/test_0480_tool_dependency_xml_verification.py @@ -0,0 +1,75 @@ +from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os +import tool_shed.base.test_db_util as test_db_util + +import logging +log = logging.getLogger( __name__ ) + +category_name = 'Test 0480 Tool dependency definition validation' +category_description = 'Test script 0480 for validating tool dependency definitions.' +repository_name = 'package_invalid_tool_dependency_xml_1_0_0' +repository_description = "Contains a tool dependency definition that should return an error." +repository_long_description = "This repository is in the test suite 0480" + +''' + +1. Create a repository package_invalid_tool_dependency_xml_1_0_0 +2. Upload a tool_dependencies.xml file to the repository with no <actions> tags around the <action> tags. +3. Verify error message is displayed. + +''' + +class TestDependencyDefinitionValidation( ShedTwillTestCase ): + '''Test the tool shed's tool dependency XML validation.''' + + def test_0000_initiate_users_and_category( self ): + """Create necessary user accounts and login as an admin user.""" + self.logout() + self.login( email=common.admin_email, username=common.admin_username ) + admin_user = test_db_util.get_user( common.admin_email ) + assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email + admin_user_private_role = test_db_util.get_private_role( admin_user ) + self.create_category( name=category_name, description=category_description ) + self.logout() + self.login( email=common.test_user_2_email, username=common.test_user_2_name ) + test_user_2 = test_db_util.get_user( common.test_user_2_email ) + assert test_user_2 is not None, 'Problem retrieving user with email %s from the database' % common.test_user_2_email + test_user_2_private_role = test_db_util.get_private_role( test_user_2 ) + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + test_user_1 = test_db_util.get_user( common.test_user_1_email ) + assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % common.test_user_1_email + test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + + def test_0005_create_tool_dependency_repository( self ): + '''Create and populate package_invalid_tool_dependency_xml_1_0_0.''' + ''' + This is step 1 - Create a repository package_invalid_tool_dependency_xml_1_0_0. + + Create a repository named package_invalid_tool_dependency_xml_1_0_0 that will contain only a single file named tool_dependencies.xml. + ''' + category = test_db_util.get_category_by_name( category_name ) + repository = self.get_or_create_repository( name=repository_name, + description=repository_description, + long_description=repository_long_description, + owner=common.test_user_1_name, + category_id=self.security.encode_id( category.id ), + strings_displayed=[] ) + self.upload_file( repository, + filename='0480_files/tool_dependencies.xml', + filepath=None, + valid_tools_only=False, + uncompress_file=False, + remove_repo_files_not_in_tar=False, + commit_message='Populate package_invalid_tool_dependency_xml_1_0_0 with an improperly defined tool dependency.', + strings_displayed=[ 'package cannot be installed because', 'missing either an <actions> tag set' ], + strings_not_displayed=[] ) + + def test_0010_populate_tool_dependency_repository( self ): + '''Verify package_invalid_tool_dependency_xml_1_0_0.''' + ''' + This is step 3 - Verify repository. The uploaded tool dependency XML should not have resulted in a new changeset. + ''' + repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + assert self.repository_is_new( repository ), 'Uploading an incorrectly defined tool_dependencies.xml resulted in a changeset being generated.' + + \ No newline at end of file diff -r d448ed76962c8b36fde83e5cd8dec91e97c9d812 -r 253f888144aaa4ae4eadaf15de493b3144fc5cd3 test/tool_shed/functional/test_1440_missing_env_sh_files.py --- /dev/null +++ b/test/tool_shed/functional/test_1440_missing_env_sh_files.py @@ -0,0 +1,130 @@ +from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os +import tool_shed.base.test_db_util as test_db_util + +import logging +log = logging.getLogger( __name__ ) + +category_name = 'Test 1440 Tool dependency missing env.sh' +category_description = 'Test script 1440 for detection of missing environment settings.' +package_repository_name = 'package_env_sh_1_0_1440' +tool_repository_name = 'filter_1440' +package_repository_description = 'Repository that should result in an env.sh file, but does not.' +tool_repository_description = 'Galaxy filtering tool.' +package_repository_long_description = '%s: %s' % ( package_repository_name, package_repository_description ) +tool_repository_long_description = '%s: %s' % ( tool_repository_name, tool_repository_description ) + +''' +1. Create a tool dependency type repository that reliably fails to install successfully. This repository should define + an action that would have created an env.sh file on success, resulting in an env.sh file that should exist, but is missing. + +2. Create a repository that defines a complex repository dependency in the repository created in step 1, with prior_install_required + and set_environment_for_install. + +3. Attempt to install the second repository into a galaxy instance, verify that it is installed but missing tool dependencies. + +''' + + +class TestMissingEnvSh( ShedTwillTestCase ): + '''Test installing a repository that should create an env.sh file, but does not.''' + + def test_0000_initiate_users_and_category( self ): + """Create necessary user accounts and login as an admin user.""" + self.logout() + self.login( email=common.admin_email, username=common.admin_username ) + admin_user = test_db_util.get_user( common.admin_email ) + assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email + admin_user_private_role = test_db_util.get_private_role( admin_user ) + self.create_category( name=category_name, description=category_description ) + self.logout() + self.login( email=common.test_user_2_email, username=common.test_user_2_name ) + test_user_2 = test_db_util.get_user( common.test_user_2_email ) + assert test_user_2 is not None, 'Problem retrieving user with email %s from the database' % common.test_user_2_email + test_user_2_private_role = test_db_util.get_private_role( test_user_2 ) + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + test_user_1 = test_db_util.get_user( common.test_user_1_email ) + assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % common.test_user_1_email + test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + + def test_0005_create_package_repository( self ): + '''Create and populate package_env_sh_1_0_1440.''' + ''' + This is step 1 - Create repository package_env_sh_1_0_1440. + + Create and populate a repository that is designed to fail a tool dependency installation. This tool dependency should + also define one or more environment variables. + ''' + category = test_db_util.get_category_by_name( category_name ) + repository = self.get_or_create_repository( name=package_repository_name, + description=package_repository_description, + long_description=package_repository_long_description, + owner=common.test_user_1_name, + category_id=self.security.encode_id( category.id ), + strings_displayed=[] ) + # Upload the edited tool dependency definition to the package_lapack_3_4_1440 repository. + self.upload_file( repository, + filename='1440_files/dependency_definition/tool_dependencies.xml', + filepath=None, + valid_tools_only=True, + uncompress_file=False, + remove_repo_files_not_in_tar=False, + commit_message='Populate package_env_sh_1_0_1440 with a broken tool dependency definition.', + strings_displayed=[], + strings_not_displayed=[] ) + + def test_0010_create_filter_repository( self ): + '''Create and populate filter_1440.''' + ''' + This is step 2 - Create a repository that defines a complex repository dependency on the repository created in + step 1, with prior_install_required and set_environment_for_install. + ''' + category = test_db_util.get_category_by_name( category_name ) + repository = self.get_or_create_repository( name=tool_repository_name, + description=tool_repository_description, + long_description=tool_repository_long_description, + owner=common.test_user_1_name, + category_id=self.security.encode_id( category.id ), + strings_displayed=[] ) + # Upload the edited tool dependency definition to the package_lapack_3_4_1440 repository. + self.upload_file( repository, + filename='filtering/filtering_2.2.0.tar', + filepath=None, + valid_tools_only=True, + uncompress_file=False, + remove_repo_files_not_in_tar=False, + commit_message='Populate filter_1440 with the filtering tool.', + strings_displayed=[], + strings_not_displayed=[] ) + self.upload_file( repository, + filename='1440_files/complex_dependency/tool_dependencies.xml', + filepath=None, + valid_tools_only=True, + uncompress_file=False, + remove_repo_files_not_in_tar=False, + commit_message='Populate filter_1440 with a dependency on package_env_sh_1_0_1440.', + strings_displayed=[], + strings_not_displayed=[] ) + + def test_0015_install_filter_repository( self ): + '''Install the filter_1440 repository to galaxy.''' + ''' + This is step 3 - Attempt to install the second repository into a galaxy instance, verify that it is installed but + missing tool dependencies. + ''' + self.galaxy_logout() + self.galaxy_login( email=common.admin_email, username=common.admin_username ) + post_submit_strings_displayed = [ 'filter_1440', 'package_env_sh_1_0_1440' ] + self.install_repository( 'filter_1440', + common.test_user_1_name, + category_name, + install_tool_dependencies=True, + post_submit_strings_displayed=post_submit_strings_displayed ) + + def test_0020_verify_missing_tool_dependency( self ): + '''Verify that the filter_1440 repository is installed and missing tool dependencies.''' + repository = test_db_util.get_installed_repository_by_name_owner( 'filter_1440', common.test_user_1_name ) + strings_displayed = [ 'Missing tool dependencies' ] + self.display_installed_repository_manage_page( repository, strings_displayed=strings_displayed ) + assert len( repository.missing_tool_dependencies ) == 1, 'filter_1440 should have a missing tool dependency, but does not.' + diff -r d448ed76962c8b36fde83e5cd8dec91e97c9d812 -r 253f888144aaa4ae4eadaf15de493b3144fc5cd3 test/tool_shed/test_data/0480_files/tool_dependencies.xml --- /dev/null +++ b/test/tool_shed/test_data/0480_files/tool_dependencies.xml @@ -0,0 +1,14 @@ +<?xml version="1.0"?> +<tool_dependency> + <package name="invalid_tool_dependency_xml" version="1.0.0"> + <install version="1.0"> + <action type="download_file">file://./</action> + <action type="shell_command">/bin/true</action> + <action type="set_environment"> + <environment_variable action="set_to" name="INSTALL_DIR">$INSTALL_DIR</environment_variable> + </action> + </install> + <readme> + </readme> + </package> +</tool_dependency> \ No newline at end of file diff -r d448ed76962c8b36fde83e5cd8dec91e97c9d812 -r 253f888144aaa4ae4eadaf15de493b3144fc5cd3 test/tool_shed/test_data/1440_files/complex_dependency/tool_dependencies.xml --- /dev/null +++ b/test/tool_shed/test_data/1440_files/complex_dependency/tool_dependencies.xml @@ -0,0 +1,6 @@ +<?xml version='1.0' encoding='utf-8'?> +<tool_dependency> + <package name="failure" version="1.0.0"> + <repository name="package_env_sh_1_0_1440" owner="user1" /> + </package> +</tool_dependency> \ No newline at end of file diff -r d448ed76962c8b36fde83e5cd8dec91e97c9d812 -r 253f888144aaa4ae4eadaf15de493b3144fc5cd3 test/tool_shed/test_data/1440_files/dependency_definition/tool_dependencies.xml --- /dev/null +++ b/test/tool_shed/test_data/1440_files/dependency_definition/tool_dependencies.xml @@ -0,0 +1,14 @@ +<?xml version='1.0' encoding='utf-8'?> +<tool_dependency> + <package name="failure" version="1.0.0"> + <install version="1.0"> + <actions> + <action type="shell_command">false</action> + <action type="set_environment"> + <environment_variable name="TEST_SUCCEEDED" action="prepend_to">true</environment_variable> + </action> + </actions> + </install> + <readme></readme> + </package> +</tool_dependency> \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Code cleanup and minor fixes in prep scripts for tool shed nightly test runs and its functional test framework.
by commits-noreply＠bitbucket.org 21 Nov '13

21 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d448ed76962c/ Changeset: d448ed76962c User: greg Date: 2013-11-21 16:28:22 Summary: Code cleanup and minor fixes in prep scripts for tool shed nightly test runs and its functional test framework. Affected #: 3 files diff -r e982fcaaaab558e6d4ecb3750d2ec721d5fd861d -r d448ed76962c8b36fde83e5cd8dec91e97c9d812 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -1,91 +1,69 @@ #!/usr/bin/env python +import ConfigParser +import logging +import os +import shutil +import sys +import tempfile +import time -import os, sys, logging, tempfile +from datetime import datetime +from optparse import OptionParser +from time import strftime new_path = [ os.path.join( os.getcwd(), "lib" ), os.path.join( os.getcwd(), "test" ) ] -new_path.extend( sys.path[1:] ) +new_path.extend( sys.path[ 1: ] ) sys.path = new_path +from galaxy import eggs +eggs.require( "SQLAlchemy >= 0.4" ) +eggs.require( 'mercurial' ) + +import galaxy.webapps.tool_shed.config as tool_shed_config +import galaxy.webapps.tool_shed.model.mapping + +from base.util import get_test_environment +from base.util import get_database_version +from base.util import get_repository_current_revision +from galaxy.model.orm import and_, not_, select +from mercurial import hg +from mercurial import ui +from mercurial import __version__ +from tool_shed.util.shed_util_common import clone_repository +from tool_shed.util.shed_util_common import get_configured_ui + log = logging.getLogger() log.setLevel( 10 ) log.addHandler( logging.StreamHandler( sys.stdout ) ) -from galaxy import eggs -import pkg_resources -eggs.require( "SQLAlchemy >= 0.4" ) -eggs.require( 'mercurial' ) -from mercurial import hg, ui, commands, __version__ +assert sys.version_info[ :2 ] >= ( 2, 6 ) -import time, ConfigParser, shutil -from datetime import datetime, timedelta -from time import strftime -from optparse import OptionParser -import galaxy.webapps.tool_shed.config as tool_shed_config -import galaxy.webapps.tool_shed.model.mapping -import sqlalchemy as sa -from galaxy.model.orm import and_, not_, select -from galaxy.util.json import from_json_string, to_json_string -from galaxy.web import url_for -from galaxy.tools import parameters -from tool_shed.util.shed_util_common import clone_repository, get_configured_ui - -from base.util import get_test_environment, get_database_version, get_repository_current_revision - -assert sys.version_info[:2] >= ( 2, 4 ) - -def main(): - '''Script that checks repositories to see if the tools contained within them have functional tests defined.''' - parser = OptionParser() - parser.add_option( "-i", "--info_only", action="store_true", dest="info_only", help="info about the requested action", default=False ) - parser.add_option( "-s", - "--section", - action="store", - dest="section", - default='server:main', - help="which .ini file section to extract the host and port from" ) - parser.add_option( - "-v", "--verbose", - action="count", dest="verbosity", - default=1, - help="Control the amount of detail in the log output.") - parser.add_option( - "--verbosity", action="store", dest="verbosity", - metavar='VERBOSITY', - type="int", help="Control the amount of detail in the log output. --verbosity=1 is " - "the same as -v") - ( options, args ) = parser.parse_args() - try: - ini_file = args[0] - except IndexError: - print "Usage: python %s <tool shed .ini file> [options]" % sys.argv[ 0 ] - exit( 127 ) - config_parser = ConfigParser.ConfigParser( {'here':os.getcwd()} ) - config_parser.read( ini_file ) - config_dict = {} - for key, value in config_parser.items( "app:main" ): - config_dict[key] = value - config = tool_shed_config.Configuration( **config_dict ) - - config_section = options.section - now = strftime( "%Y-%m-%d %H:%M:%S" ) - print "#############################################################################" - print "# %s - Checking repositories for tools with functional tests." % now - print "# This tool shed is configured to listen on %s:%s." % ( config_parser.get( config_section, 'host' ), config_parser.get( config_section, 'port' ) ) - app = FlagRepositoriesApplication( config ) - - if options.info_only: - print "# Displaying info only ( --info_only )" - if options.verbosity: - print "# Displaying extra information ( --verbosity = %d )" % options.verbosity - - check_and_flag_repositories( app, info_only=options.info_only, verbosity=options.verbosity ) +class FlagRepositoriesApplication( object ): + """Encapsulates the state of a Universe application""" + def __init__( self, config ): + if config.database_connection is False: + config.database_connection = "sqlite:///%s?isolation_level=IMMEDIATE" % config.database + # Setup the database engine and ORM + self.model = galaxy.webapps.tool_shed.model.mapping.init( config.file_path, config.database_connection, engine_options={}, create_tables=False ) + self.hgweb_config_manager = self.model.hgweb_config_manager + self.hgweb_config_manager.hgweb_config_dir = config.hgweb_config_dir + print "# Using configured hgweb.config file: ", self.hgweb_config_manager.hgweb_config + @property + def sa_session( self ): + """ + Returns a SQLAlchemy session -- currently just gets the current + session from the threadlocal session context, but this is provided + to allow migration toward a more SQLAlchemy 0.4 style of use. + """ + return self.model.context.current + def shutdown( self ): + pass def check_and_flag_repositories( app, info_only=False, verbosity=1 ): - ''' + """ This method will iterate through all records in the repository_metadata table, checking each one for tool metadata, - then checking the tool metadata for tests. - Each tool's metadata should look something like: + then checking the tool metadata for tests. Each tool's metadata should look something like: { "add_to_tool_panel": true, "description": "", @@ -111,94 +89,11 @@ "version": "1.2.3", "version_string_cmd": null } - If the "tests" attribute is missing or empty, this script will mark the metadata record (which is specific to a changeset revision of a repository) not to be tested. If each "tools" attribute has at least one valid "tests" entry, this script will do nothing, and leave it available for the install and test repositories script to process. If the tested changeset revision does not have a test-data directory, this script will also mark the revision not to be tested. - - TODO: Update this dict structure with the recently added components. - - If any error is encountered, the script will update the repository_metadata.tool_test_results attribute following this structure: - { - "test_environment": - { - "galaxy_revision": "9001:abcd1234", - "galaxy_database_version": "114", - "tool_shed_revision": "9001:abcd1234", - "tool_shed_mercurial_version": "2.3.1", - "tool_shed_database_version": "17", - "python_version": "2.7.2", - "architecture": "x86_64", - "system": "Darwin 12.2.0" - }, - "passed_tests": - [ - { - "test_id": "The test ID, generated by twill", - "tool_id": "The tool ID that was tested", - "tool_version": "The tool version that was tested", - }, - ] - "failed_tests": - [ - { - "test_id": "The test ID, generated by twill", - "tool_id": "The tool ID that was tested", - "tool_version": "The tool version that was tested", - "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." - "traceback": "The captured traceback." - }, - ] - "installation_errors": - { - 'tool_dependencies': - [ - { - 'type': 'Type of tool dependency, e.g. package, set_environment, etc.', - 'name': 'Name of the tool dependency.', - 'version': 'Version if this is a package, otherwise blank.', - 'error_message': 'The error message returned when installation was attempted.', - }, - ], - 'repository_dependencies': - [ - { - 'tool_shed': 'The tool shed that this repository was installed from.', - 'name': 'The name of the repository that failed to install.', - 'owner': 'Owner of the failed repository.', - 'changeset_revision': 'Changeset revision of the failed repository.', - 'error_message': 'The error message that was returned when the repository failed to install.', - }, - ], - 'current_repository': - [ - { - 'tool_shed': 'The tool shed that this repository was installed from.', - 'name': 'The name of the repository that failed to install.', - 'owner': 'Owner of the failed repository.', - 'changeset_revision': 'Changeset revision of the failed repository.', - 'error_message': 'The error message that was returned when the repository failed to install.', - }, - ], - { - "name": "The name of the repository.", - "owner": "The owner of the repository.", - "changeset_revision": "The changeset revision of the repository.", - "error_message": "The message stored in tool_dependency.error_message." - }, - } - "missing_test_components": - [ - { - "tool_id": "The tool ID that missing components.", - "tool_version": "The version of the tool." - "tool_guid": "The guid of the tool." - "missing_components": "Which components are missing, e.g. the test data filename, or the test-data directory." - }, - ] - } - ''' + """ start = time.time() skip_metadata_ids = [] checked_repository_ids = [] @@ -214,49 +109,68 @@ # Get the list of metadata records to check for functional tests and test data. Limit this to records that have not been flagged do_not_test, # since there's no need to check them again if they won't be tested anyway. Also filter out changeset revisions that are not downloadable, # because it's redundant to test a revision that a user can't install. - for metadata_record in app.sa_session.query( app.model.RepositoryMetadata ) \ - .filter( and_( app.model.RepositoryMetadata.table.c.downloadable == True, - app.model.RepositoryMetadata.table.c.includes_tools == True, - app.model.RepositoryMetadata.table.c.do_not_test == False, - not_( app.model.RepositoryMetadata.table.c.id.in_( skip_metadata_ids ) ) ) ): + for repository_metadata in app.sa_session.query( app.model.RepositoryMetadata ) \ + .filter( and_( app.model.RepositoryMetadata.table.c.downloadable == True, + app.model.RepositoryMetadata.table.c.includes_tools == True, + app.model.RepositoryMetadata.table.c.do_not_test == False, + not_( app.model.RepositoryMetadata.table.c.id.in_( skip_metadata_ids ) ) ) ): records_checked += 1 - # Initialize the repository_status dict with the test environment, but leave the test_errors empty. - repository_status = {} - if metadata_record.tool_test_results: - repository_status = metadata_record.tool_test_results - # Clear any old invalid tests for this metadata revision, since this could lead to duplication of invalid test rows, - # or tests incorrectly labeled as invalid. - missing_test_components = [] - if 'test_environment' in repository_status: - repository_status[ 'test_environment' ] = get_test_environment( repository_status[ 'test_environment' ] ) + # Create the repository_status dictionary, using the dictionary from the previous test run if available. + if repository_metadata.tool_test_results: + repository_status = repository_metadata.tool_test_results else: - repository_status[ 'test_environment' ] = get_test_environment() - repository_status[ 'test_environment' ][ 'tool_shed_database_version' ] = get_database_version( app ) - repository_status[ 'test_environment' ][ 'tool_shed_mercurial_version' ] = __version__.version - repository_status[ 'test_environment' ][ 'tool_shed_revision' ] = get_repository_current_revision( os.getcwd() ) - name = metadata_record.repository.name - owner = metadata_record.repository.user.username - changeset_revision = str( metadata_record.changeset_revision ) - if metadata_record.repository.id not in checked_repository_ids: - checked_repository_ids.append( metadata_record.repository.id ) + repository_status = {} + # Initialize the repository_status dictionary with the information about the current test environment. + last_test_environment = repository_status.get( 'test_environment', None ) + if last_test_environment is None: + test_environment = get_test_environment() + else: + test_environment = get_test_environment( last_test_environment ) + test_environment[ 'tool_shed_database_version' ] = get_database_version( app ) + test_environment[ 'tool_shed_mercurial_version' ] = __version__.version + test_environment[ 'tool_shed_revision' ] = get_repository_current_revision( os.getcwd() ) + repository_status[ 'test_environment' ] = test_environment + # Check the next repository revision. + changeset_revision = str( repository_metadata.changeset_revision ) + name = repository.name + owner = repository.user.username + metadata = repository_metadata.metadata + repository = repository_metadata.repository + if repository.id not in checked_repository_ids: + checked_repository_ids.append( repository.id ) if verbosity >= 1: print '# -------------------------------------------------------------------------------------------' print '# Now checking revision %s of %s, owned by %s.' % ( changeset_revision, name, owner ) # If this changeset revision has no tools, we don't need to do anything here, the install and test script has a filter for returning # only repositories that contain tools. - if 'tools' not in metadata_record.metadata: - continue - else: + tool_dicts = repository_metadata.get( 'tools', None ) + """ + "{"tools": + [{"add_to_tool_panel": true, + "description": "data on any column using simple expressions", + "guid": "localhost:9009/repos/iuc/filtering_with_empty_test_tags/Filter1/1.1.0", + "id": "Filter1", + "name": "Filter", + "requirements": [], + "tests": + [{"inputs": [], "name": "Test-1", "outputs": [], "required_files": []}, + {"inputs": [], "name": "Test-2", "outputs": [], "required_files": []}], + "tool_config": "database/community_files/000/repo_6/filtering.xml", + "tool_type": "default", + "version": "1.1.0", + "version_string_cmd": null}]}" + """ + if tool_dicts is not None: has_test_data = False testable_revision_found = False # Clone the repository up to the changeset revision we're checking. - repo_dir = metadata_record.repository.repo_path( app ) + repo_dir = repository_metadata.repository.repo_path( app ) repo = hg.repository( get_configured_ui(), repo_dir ) work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-cafr" ) cloned_ok, error_message = clone_repository( repo_dir, work_dir, changeset_revision ) if cloned_ok: # Iterate through all the directories in the cloned changeset revision and determine whether there's a - # directory named test-data. If this directory is not present, update the metadata record for the changeset + # directory named test-data. If this directory is not present update the metadata record for the changeset # revision we're checking. for root, dirs, files in os.walk( work_dir ): if '.hg' in dirs: @@ -266,18 +180,18 @@ test_data_path = os.path.join( root, dirs[ dirs.index( 'test-data' ) ] ) break if verbosity >= 1: - if not has_test_data: + if has_test_data: + print '# Test data directory found in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) + else: print '# Test data directory missing in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) - else: - print '# Test data directory found in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) print '# Checking for functional tests in changeset revision %s of %s, owned by %s.' % \ ( changeset_revision, name, owner ) - # Loop through all the tools in this metadata record, checking each one for defined functional tests. - for tool_metadata in metadata_record.metadata[ 'tools' ]: + # Inspect each tool_dict for defined functional tests. + for tool_dict in tool_dicts: tool_count += 1 - tool_id = tool_metadata[ 'id' ] - tool_version = tool_metadata[ 'version' ] - tool_guid = tool_metadata[ 'guid' ] + tool_id = tool_dict[ 'id' ] + tool_version = tool_dict[ 'version' ] + tool_guid = tool_dict[ 'guid' ] if verbosity >= 2: print "# Checking tool ID '%s' in changeset revision %s of %s." % \ ( tool_id, changeset_revision, name ) @@ -285,7 +199,7 @@ # not if it's missing or undefined. Filtering out those repositories at this step will reduce the number of "false negatives" the # automated functional test framework produces. tool_has_tests = False - defined_test_dicts = tool_metadata.get( 'tests', None ) + defined_test_dicts = tool_dict.get( 'tests', None ) if defined_test_dicts is not None: # We need to inspect the <test> tags because the following tags... # <tests> @@ -321,7 +235,7 @@ missing_test_files = [] has_test_files = False if tool_has_tests and has_test_data: - missing_test_files = check_for_missing_test_files( tool_metadata[ 'tests' ], test_data_path ) + missing_test_files = check_for_missing_test_files( defined_test_dicts, test_data_path ) if missing_test_files: if verbosity >= 2: print "# Tool ID '%s' in changeset revision %s of %s is missing one or more required test files: %s" % \ @@ -337,90 +251,7 @@ if missing_test_files: failure_reason += 'One or more test files are missing for tool %s: %s' % ( tool_id, ', '.join( missing_test_files ) ) problem_found = True - test_errors = dict( tool_id=tool_id, tool_version=tool_version, tool_guid=tool_guid, - missing_components=failure_reason ) - # The repository_metadata.tool_test_results attribute should always have the following structure: - # { - # "test_environment": - # { - # "galaxy_revision": "9001:abcd1234", - # "galaxy_database_version": "114", - # "tool_shed_revision": "9001:abcd1234", - # "tool_shed_mercurial_version": "2.3.1", - # "tool_shed_database_version": "17", - # "python_version": "2.7.2", - # "architecture": "x86_64", - # "system": "Darwin 12.2.0" - # }, - # "passed_tests": - # [ - # { - # "test_id": "The test ID, generated by twill", - # "tool_id": "The tool ID that was tested", - # "tool_version": "The tool version that was tested", - # }, - # ] - # "failed_tests": - # [ - # { - # "test_id": "The test ID, generated by twill", - # "tool_id": "The tool ID that was tested", - # "tool_version": "The tool version that was tested", - # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." - # "traceback": "The captured traceback." - # }, - # ] - # "installation_errors": - # { - # 'tool_dependencies': - # [ - # { - # 'type': 'Type of tool dependency, e.g. package, set_environment, etc.', - # 'name': 'Name of the tool dependency.', - # 'version': 'Version if this is a package, otherwise blank.', - # 'error_message': 'The error message returned when installation was attempted.', - # }, - # ], - # 'repository_dependencies': - # [ - # { - # 'tool_shed': 'The tool shed that this repository was installed from.', - # 'name': 'The name of the repository that failed to install.', - # 'owner': 'Owner of the failed repository.', - # 'changeset_revision': 'Changeset revision of the failed repository.', - # 'error_message': 'The error message that was returned when the repository failed to install.', - # }, - # ], - # 'current_repository': - # [ - # { - # 'tool_shed': 'The tool shed that this repository was installed from.', - # 'name': 'The name of the repository that failed to install.', - # 'owner': 'Owner of the failed repository.', - # 'changeset_revision': 'Changeset revision of the failed repository.', - # 'error_message': 'The error message that was returned when the repository failed to install.', - # }, - # ], - # { - # "name": "The name of the repository.", - # "owner": "The owner of the repository.", - # "changeset_revision": "The changeset revision of the repository.", - # "error_message": "The message stored in tool_dependency.error_message." - # }, - # } - # "missing_test_components": - # [ - # { - # "tool_id": "The tool ID that missing components.", - # "tool_version": "The version of the tool." - # "tool_guid": "The guid of the tool." - # "missing_components": "Which components are missing, e.g. the test data filename, or the test-data directory." - # }, - # ] - # } - # - # Optionally, "traceback" may be included in a test_errors dict, if it is relevant. No script should overwrite anything other - # than the list relevant to what it is testing. + test_errors = dict( tool_id=tool_id, tool_version=tool_version, tool_guid=tool_guid, missing_components=failure_reason ) # Only append this error dict if it hasn't already been added. if problem_found: if test_errors not in missing_test_components: @@ -443,8 +274,7 @@ if 'missing_components' in invalid_test: print '# %s' % invalid_test[ 'missing_components' ] if not info_only: - # If repository_status[ 'test_errors' ] is empty, no issues were found, and we can just update time_last_tested with the platform - # on which this script was run. + # The repository_metadata.time_last_tested column is not changed by this script since no testing is performed here. if missing_test_components: # If functional test definitions or test data are missing, set do_not_test = True if no tool with valid tests has been # found in this revision, and: @@ -456,14 +286,13 @@ # changeset revision will be created, either of which will be automatically checked and flagged as appropriate. # In the install and test script, this behavior is slightly different, since we do want to always run functional # tests on the most recent downloadable changeset revision. - if should_set_do_not_test_flag( app, metadata_record.repository, changeset_revision ) and not testable_revision_found: - metadata_record.do_not_test = True - metadata_record.tools_functionally_correct = False - metadata_record.missing_test_components = True + if should_set_do_not_test_flag( app, repository_metadata.repository, changeset_revision ) and not testable_revision_found: + repository_metadata.do_not_test = True + repository_metadata.tools_functionally_correct = False + repository_metadata.missing_test_components = True repository_status[ 'missing_test_components' ] = missing_test_components - metadata_record.tool_test_results = repository_status - metadata_record.time_last_tested = datetime.utcnow() - app.sa_session.add( metadata_record ) + repository_metadata.tool_test_results = repository_status + app.sa_session.add( repository_metadata ) app.sa_session.flush() stop = time.time() print '# -------------------------------------------------------------------------------------------' @@ -477,14 +306,6 @@ print "# Elapsed time: ", stop - start print "#############################################################################" -def get_repo_changelog_tuples( repo_path ): - repo = hg.repository( ui.ui(), repo_path ) - changelog_tuples = [] - for changeset in repo.changelog: - ctx = repo.changectx( changeset ) - changelog_tuples.append( ( ctx.rev(), str( ctx ) ) ) - return changelog_tuples - def check_for_missing_test_files( test_definition, test_data_path ): '''Process the tool's functional test definitions and check for each file specified as an input or output.''' missing_test_files = [] @@ -500,6 +321,14 @@ missing_test_files.append( required_file ) return missing_test_files +def get_repo_changelog_tuples( repo_path ): + repo = hg.repository( ui.ui(), repo_path ) + changelog_tuples = [] + for changeset in repo.changelog: + ctx = repo.changectx( changeset ) + changelog_tuples.append( ( ctx.rev(), str( ctx ) ) ) + return changelog_tuples + def is_most_recent_downloadable_revision( app, repository, changeset_revision, downloadable_revisions ): # Get a list of ( numeric revision, changeset hash ) tuples from the changelog. changelog = get_repo_changelog_tuples( repository.repo_path( app ) ) @@ -513,6 +342,41 @@ return True return False +def main(): + '''Script that checks repositories to see if the tools contained within them have functional tests defined.''' + parser = OptionParser() + parser.add_option( "-i", "--info_only", action="store_true", dest="info_only", help="info about the requested action", default=False ) + parser.add_option( "-s", "--section", action="store", dest="section", default='server:main', + help=".ini file section from which to to extract the host and port" ) + parser.add_option( "-v", "--verbose", action="count", dest="verbosity", default=1, help="Control the amount of detail in the log output.") + parser.add_option( "--verbosity", action="store", dest="verbosity", metavar='VERBOSITY', type="int", + help="Control the amount of detail in the log output. --verbosity=1 is the same as -v" ) + ( options, args ) = parser.parse_args() + try: + ini_file = args[ 0 ] + except IndexError: + print "Usage: python %s <tool shed .ini file> [options]" % sys.argv[ 0 ] + exit( 127 ) + config_parser = ConfigParser.ConfigParser( {'here':os.getcwd() } ) + config_parser.read( ini_file ) + config_dict = {} + for key, value in config_parser.items( "app:main" ): + config_dict[key] = value + config = tool_shed_config.Configuration( **config_dict ) + config_section = options.section + now = strftime( "%Y-%m-%d %H:%M:%S" ) + + print "#############################################################################" + print "# %s - Checking repositories for tools with functional tests." % now + print "# This tool shed is configured to listen on %s:%s." % ( config_parser.get( config_section, 'host' ), + config_parser.get( config_section, 'port' ) ) + app = FlagRepositoriesApplication( config ) + if options.info_only: + print "# Displaying info only ( --info_only )" + if options.verbosity: + print "# Displaying extra information ( --verbosity = %d )" % options.verbosity + check_and_flag_repositories( app, info_only=options.info_only, verbosity=options.verbosity ) + def should_set_do_not_test_flag( app, repository, changeset_revision ): ''' Returns True if: @@ -524,11 +388,11 @@ flagged as appropriate. In the install and test script, this behavior is slightly different, since we do want to always run functional tests on the most recent downloadable changeset revision. ''' - metadata_records = app.sa_session.query( app.model.RepositoryMetadata ) \ + repository_revisions = app.sa_session.query( app.model.RepositoryMetadata ) \ .filter( and_( app.model.RepositoryMetadata.table.c.downloadable == True, app.model.RepositoryMetadata.table.c.repository_id == repository.id ) ) \ .all() - downloadable_revisions = [ metadata_record.changeset_revision for metadata_record in metadata_records ] + downloadable_revisions = [ repository_metadata.changeset_revision for repository_metadata in repository_revisions ] is_latest_revision = is_most_recent_downloadable_revision( app, repository, changeset_revision, downloadable_revisions ) if len( downloadable_revisions ) == 1: return True @@ -539,26 +403,87 @@ else: return False - -class FlagRepositoriesApplication( object ): - """Encapsulates the state of a Universe application""" - def __init__( self, config ): - if config.database_connection is False: - config.database_connection = "sqlite:///%s?isolation_level=IMMEDIATE" % config.database - # Setup the database engine and ORM - self.model = galaxy.webapps.tool_shed.model.mapping.init( config.file_path, config.database_connection, engine_options={}, create_tables=False ) - self.hgweb_config_manager = self.model.hgweb_config_manager - self.hgweb_config_manager.hgweb_config_dir = config.hgweb_config_dir - print "# Using configured hgweb.config file: ", self.hgweb_config_manager.hgweb_config - @property - def sa_session( self ): - """ - Returns a SQLAlchemy session -- currently just gets the current - session from the threadlocal session context, but this is provided - to allow migration toward a more SQLAlchemy 0.4 style of use. - """ - return self.model.context.current - def shutdown( self ): - pass - -if __name__ == "__main__": main() +if __name__ == "__main__": + # The repository_metadata.tool_test_results json value should have the following structure: + # { + # "test_environment": + # { + # "galaxy_revision": "9001:abcd1234", + # "galaxy_database_version": "114", + # "tool_shed_revision": "9001:abcd1234", + # "tool_shed_mercurial_version": "2.3.1", + # "tool_shed_database_version": "17", + # "python_version": "2.7.2", + # "architecture": "x86_64", + # "system": "Darwin 12.2.0" + # }, + # "passed_tests": + # [ + # { + # "test_id": "The test ID, generated by twill", + # "tool_id": "The tool ID that was tested", + # "tool_version": "The tool version that was tested", + # }, + # ] + # "failed_tests": + # [ + # { + # "test_id": "The test ID, generated by twill", + # "tool_id": "The tool ID that was tested", + # "tool_version": "The tool version that was tested", + # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." + # "traceback": "The captured traceback." + # }, + # ] + # "installation_errors": + # { + # 'tool_dependencies': + # [ + # { + # 'type': 'Type of tool dependency, e.g. package, set_environment, etc.', + # 'name': 'Name of the tool dependency.', + # 'version': 'Version if this is a package, otherwise blank.', + # 'error_message': 'The error message returned when installation was attempted.', + # }, + # ], + # 'repository_dependencies': + # [ + # { + # 'tool_shed': 'The tool shed that this repository was installed from.', + # 'name': 'The name of the repository that failed to install.', + # 'owner': 'Owner of the failed repository.', + # 'changeset_revision': 'Changeset revision of the failed repository.', + # 'error_message': 'The error message that was returned when the repository failed to install.', + # }, + # ], + # 'current_repository': + # [ + # { + # 'tool_shed': 'The tool shed that this repository was installed from.', + # 'name': 'The name of the repository that failed to install.', + # 'owner': 'Owner of the failed repository.', + # 'changeset_revision': 'Changeset revision of the failed repository.', + # 'error_message': 'The error message that was returned when the repository failed to install.', + # }, + # ], + # { + # "name": "The name of the repository.", + # "owner": "The owner of the repository.", + # "changeset_revision": "The changeset revision of the repository.", + # "error_message": "The message stored in tool_dependency.error_message." + # }, + # } + # "missing_test_components": + # [ + # { + # "tool_id": "The tool ID that missing components.", + # "tool_version": "The version of the tool." + # "tool_guid": "The guid of the tool." + # "missing_components": "Which components are missing, e.g. the test data filename, or the test-data directory." + # }, + # ] + # } + # + # Optionally, "traceback" may be included in a test_errors dict, if it is relevant. No script should overwrite anything other + # than the list relevant to what it is testing. + main() diff -r e982fcaaaab558e6d4ecb3750d2ec721d5fd861d -r d448ed76962c8b36fde83e5cd8dec91e97c9d812 lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py --- a/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py +++ b/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py @@ -86,7 +86,7 @@ now = strftime( "%Y-%m-%d %H:%M:%S" ) print "#############################################################################" - print "# %s - Validating repositories of type %s on %s..." % ( TOOL_DEPENDENCY_DEFINITION, now, config_parser.get( config_section, 'host' ) ) + print "# %s - Validating repositories of type %s on %s..." % ( now, TOOL_DEPENDENCY_DEFINITION, config_parser.get( config_section, 'host' ) ) print "# This tool shed is configured to listen on %s:%s" % ( config_parser.get( config_section, 'host' ), config_parser.get( config_section, 'port' ) ) app = RepositoriesApplication( config ) @@ -175,14 +175,13 @@ invalid_metadata += 1 if not info_only: repository_metadata.tool_test_results = repository_status - repository_metadata.time_last_tested = datetime.utcnow() app.sa_session.add( repository_metadata ) app.sa_session.flush() stop = time.time() print '# -------------------------------------------------------------------------------------------' print '# Checked %d repository revisions.' % records_checked print '# %d revisions found with valid tool dependency definition metadata.' % valid_metadata - print '# %d revisions found with valid tool dependency definition metadata.' % invalid_metadata + print '# %d revisions found with invalid tool dependency definition metadata.' % invalid_metadata if info_only: print '# Database not updated with any information from this run.' print "# Elapsed time: ", stop - start diff -r e982fcaaaab558e6d4ecb3750d2ec721d5fd861d -r d448ed76962c8b36fde83e5cd8dec91e97c9d812 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -120,8 +120,8 @@ def get_webapp_global_conf(): """Get the global_conf dictionary sent as the first argument to app_factory. """ - # (was originally sent 'dict()') - nothing here for now except static settings - global_conf = dict() + # (was originally sent '{}') - nothing here for now except static settings + global_conf = {} if STATIC_ENABLED: global_conf.update( get_static_settings() ) return global_conf @@ -229,7 +229,7 @@ class ReportResults( Plugin ): '''Simple Nose plugin to record the IDs of all tests run, regardless of success.''' name = "reportresults" - passed = dict() + passed = {} def options( self, parser, env=os.environ ): super( ReportResults, self ).options( parser, env=env ) @@ -479,7 +479,7 @@ tool_test_results = repository_metadata.get( 'tool_test_results', {} ) # If, for some reason, the script that checks for functional tests has not run, tool_test_results will be None. if tool_test_results is None: - return dict() + return {} return tool_test_results def install_repository( repository_info_dict ): @@ -581,14 +581,16 @@ return exclude_list def register_test_result( url, metadata_id, test_results_dict, repository_info_dict, params ): - ''' - Update the repository metadata tool_test_results and appropriate flags using the API. - ''' + '''Update the repository metadata tool_test_results and appropriate flags using the API.''' params[ 'tool_test_results' ] = test_results_dict if '-info_only' in sys.argv or 'GALAXY_INSTALL_TEST_INFO_ONLY' in os.environ: return {} else: - return update( tool_shed_api_key, '%s' % ( suc.url_join( galaxy_tool_shed_url, 'api', 'repository_revisions', metadata_id ) ), params, return_formatted=False ) + url = '%s' % ( suc.url_join( galaxy_tool_shed_url,'api', 'repository_revisions', metadata_id ) ) + return update( tool_shed_api_key, + url, + params, + return_formatted=False ) def remove_generated_tests( app ): # Delete any configured tool functional tests from the test_toolbox.__dict__, otherwise nose will find them @@ -644,7 +646,7 @@ return result, test_config.plugins._plugins def show_summary_output( repository_info_dicts ): - repositories_by_owner = dict() + repositories_by_owner = {} for repository in repository_info_dicts: if repository[ 'owner' ] not in repositories_by_owner: repositories_by_owner[ repository[ 'owner' ] ] = [] @@ -706,6 +708,7 @@ def main(): # ---- Configuration ------------------------------------------------------ + # TODO: refactor this very large main method into smaller, more maintainable components. galaxy_test_host = os.environ.get( 'GALAXY_INSTALL_TEST_HOST', default_galaxy_test_host ) galaxy_test_port = os.environ.get( 'GALAXY_INSTALL_TEST_PORT', str( default_galaxy_test_port_max ) ) @@ -730,7 +733,7 @@ galaxy_migrated_tool_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_MIGRATED_TOOL_CONF', os.path.join( galaxy_test_tmp_dir, 'test_migrated_tool_conf.xml' ) ) galaxy_tool_sheds_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF', os.path.join( galaxy_test_tmp_dir, 'test_tool_sheds_conf.xml' ) ) galaxy_shed_tools_dict = os.environ.get( 'GALAXY_INSTALL_TEST_SHED_TOOL_DICT_FILE', os.path.join( galaxy_test_tmp_dir, 'shed_tool_dict' ) ) - file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( dict() ) ) + file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( {} ) ) if 'GALAXY_INSTALL_TEST_TOOL_DATA_PATH' in os.environ: tool_data_path = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_DATA_PATH' ) else: @@ -946,8 +949,8 @@ "user_id": "529fd61ab1c6cc36" } """ - repository_status = dict() - params = dict() + repository_status = {} + params = {} repository_id = str( repository_info_dict.get( 'repository_id', None ) ) changeset_revision = str( repository_info_dict.get( 'changeset_revision', None ) ) metadata_revision_id = repository_info_dict.get( 'id', None ) @@ -1027,7 +1030,7 @@ # Generate the shed_tools_dict that specifies the location of test data contained within this repository. If the repository # does not have a test-data directory, this will return has_test_data = False, and we will set the do_not_test flag to True, # and the tools_functionally_correct flag to False, as well as updating tool_test_results. - file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( dict() ) ) + file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( {} ) ) has_test_data, shed_tools_dict = parse_tool_panel_config( galaxy_shed_tool_conf_file, from_json_string( file( galaxy_shed_tools_dict, 'r' ).read() ) ) # The repository_status dict should always have the following structure: # { Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Apply the unicodify fix to repository dependencies' error messages as well.
by commits-noreply＠bitbucket.org 21 Nov '13

21 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e982fcaaaab5/ Changeset: e982fcaaaab5 User: Dave Bouvier Date: 2013-11-21 16:10:03 Summary: Apply the unicodify fix to repository dependencies' error messages as well. Affected #: 1 file diff -r 9a998cee3ea5acaf0f058b4743da37787c916594 -r e982fcaaaab558e6d4ecb3750d2ec721d5fd861d test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -1131,7 +1131,7 @@ repository_status[ 'installation_errors' ][ 'tool_dependencies' ].append( test_result ) for dependency in repository.missing_repository_dependencies: log.error( 'Missing repository dependency %s changeset revision %s owned by %s: %s' % \ - ( str( dependency.name ), str( dependency.changeset_revision ), str( dependency.owner ), str( dependency.error_message ) ) ) + ( str( dependency.name ), str( dependency.changeset_revision ), str( dependency.owner ), unicodify( dependency.error_message ) ) ) test_result = dict( tool_shed=dependency.tool_shed, name=dependency.name, owner=dependency.owner, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 21 Nov '13

21 Nov '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7117a2655ade/ Changeset: 7117a2655ade User: jmchilton Date: 2013-11-21 15:02:50 Summary: Fix model/check.py for app being None such as when using create_db.sh Thanks to Aaron Petkau for reporting this issue. Affected #: 1 file diff -r e6e9349e8da45a058fb5811eb89813694aeef729 -r 7117a2655ade212cfee20c9499d12b8ed5dc729b lib/galaxy/model/migrate/check.py --- a/lib/galaxy/model/migrate/check.py +++ b/lib/galaxy/model/migrate/check.py @@ -57,7 +57,7 @@ migrate_to_current_version( engine, db_schema ) meta = MetaData( bind=engine ) - if getattr( app.config, 'database_auto_migrate', False ): + if app and getattr( app.config, 'database_auto_migrate', False ): migrate() return https://bitbucket.org/galaxy/galaxy-central/commits/9a998cee3ea5/ Changeset: 9a998cee3ea5 User: jmchilton Date: 2013-11-21 15:05:36 Summary: Merge. Affected #: 1 file diff -r 7117a2655ade212cfee20c9499d12b8ed5dc729b -r 9a998cee3ea5acaf0f058b4743da37787c916594 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -43,6 +43,7 @@ from functional_tests import generate_config_file from galaxy import eggs +from galaxy.util import unicodify eggs.require( "nose" ) eggs.require( "NoseHTML" ) @@ -1122,7 +1123,7 @@ # In keeping with the standard display layout, add the error message to the dict for each tool individually. for dependency in repository.missing_tool_dependencies: log.error( 'Missing tool dependency %s of type %s version %s: %s' % \ - ( str( dependency.name ), str( dependency.type ), str( dependency.version ), str( dependency.error_message ) ) ) + ( str( dependency.name ), str( dependency.type ), str( dependency.version ), unicodify( dependency.error_message ) ) ) test_result = dict( type=dependency.type, name=dependency.name, version=dependency.version, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Handle unicode characters when logging tool dependency installation errors.
by commits-noreply＠bitbucket.org 21 Nov '13

21 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f6d687147f65/ Changeset: f6d687147f65 User: Dave Bouvier Date: 2013-11-21 14:59:22 Summary: Handle unicode characters when logging tool dependency installation errors. Affected #: 1 file diff -r e6e9349e8da45a058fb5811eb89813694aeef729 -r f6d687147f65098a82854f5c7d6aaaa2dc602c8a test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -43,6 +43,7 @@ from functional_tests import generate_config_file from galaxy import eggs +from galaxy.util import unicodify eggs.require( "nose" ) eggs.require( "NoseHTML" ) @@ -1122,7 +1123,7 @@ # In keeping with the standard display layout, add the error message to the dict for each tool individually. for dependency in repository.missing_tool_dependencies: log.error( 'Missing tool dependency %s of type %s version %s: %s' % \ - ( str( dependency.name ), str( dependency.type ), str( dependency.version ), str( dependency.error_message ) ) ) + ( str( dependency.name ), str( dependency.type ), str( dependency.version ), unicodify( dependency.error_message ) ) ) test_result = dict( type=dependency.type, name=dependency.name, version=dependency.version, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Add object_store_conf.xml to .hgignore
by commits-noreply＠bitbucket.org 21 Nov '13

21 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e6e9349e8da4/ Changeset: e6e9349e8da4 User: dannon Date: 2013-11-21 02:39:51 Summary: Add object_store_conf.xml to .hgignore Affected #: 1 file diff -r 672987cdb4bf67ed14915d7997618bf008ce844e -r e6e9349e8da45a058fb5811eb89813694aeef729 .hgignore --- a/.hgignore +++ b/.hgignore @@ -60,6 +60,7 @@ job_conf.xml data_manager_conf.xml shed_data_manager_conf.xml +object_store_conf.xml config/* static/welcome.html.* static/welcome.html Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Swap impersonate to select2 deferred load w/ filter. This adds a 'f_email' filter parameter to the Users api collection -- we should revisit this before the next stable release to decide how we want API filtering to actually work down the road.
by commits-noreply＠bitbucket.org 21 Nov '13

21 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/672987cdb4bf/ Changeset: 672987cdb4bf User: dannon Date: 2013-11-21 01:59:01 Summary: Swap impersonate to select2 deferred load w/ filter. This adds a 'f_email' filter parameter to the Users api collection -- we should revisit this before the next stable release to decide how we want API filtering to actually work down the road. Affected #: 2 files diff -r 7a882c5fdd34636ae5ca98f63709c13a6648c3cf -r 672987cdb4bf67ed14915d7997618bf008ce844e lib/galaxy/webapps/galaxy/api/users.py --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -14,7 +14,7 @@ class UserAPIController( BaseAPIController, UsesTagsMixin, CreatesUsersMixin, CreatesApiKeysMixin ): @web.expose_api - def index( self, trans, deleted='False', **kwd ): + def index( self, trans, deleted='False', f_email=None, **kwd ): """ GET /api/users GET /api/users/deleted @@ -23,6 +23,8 @@ rval = [] query = trans.sa_session.query( trans.app.model.User ) deleted = util.string_as_bool( deleted ) + if f_email: + query = query.filter(trans.app.model.User.email.like("%%%s%%" % f_email)) if deleted: query = query.filter( trans.app.model.User.table.c.deleted == True ) # only admins can see deleted users diff -r 7a882c5fdd34636ae5ca98f63709c13a6648c3cf -r 672987cdb4bf67ed14915d7997618bf008ce844e templates/admin/impersonate.mako --- a/templates/admin/impersonate.mako +++ b/templates/admin/impersonate.mako @@ -14,11 +14,8 @@ <label> User to impersonate: </label> - <select name="email" class='text-and-autocomplete-select'> - %for email in emails: - <option>${email}</option> - %endfor - </select> + <input type="hidden" id="email_select" name="email"> + </input></div><div class="form-row"><input type="submit" name="impersonate_button" value="Impersonate"/> @@ -26,4 +23,35 @@ </form></div></div> + <script type="text/javascript"> + /* This should be ripped out and made generic at some point for the + * various API bindings available, and once the API can filter list + * queries (term, below) */ + $("#email_select").select2({ + placeholder: "Select a user", + ajax: { + url: "/api/users/", + dataType: 'json', + quietMillis: 250, + matcher: function(term, text) { return text.toUpperCase().indexOf(term.toUpperCase())>=0; }, + data: function (term) { + return { + f_email: term + }; + }, + results: function (data) { + var results = []; + $.each(data, function(index, item){ + results.push({ + id: item.email, + text: item.email + }); + }); + return { + results: results + }; + } + } + }); + </script> %endif Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 20 Nov '13

20 Nov '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/23afd47845fe/ Changeset: 23afd47845fe User: jmchilton Date: 2013-11-20 23:44:09 Summary: Add stderr and stdout to provenance information. Affected #: 1 file diff -r ef45d85b1f579c5bb4b07284dd96ad8403003c03 -r 23afd47845fec9b8169712d0e78ee8d2baf0d96d lib/galaxy/webapps/galaxy/api/provenance.py --- a/lib/galaxy/webapps/galaxy/api/provenance.py +++ b/lib/galaxy/webapps/galaxy/api/provenance.py @@ -48,6 +48,8 @@ "uuid": ( lambda uuid: str( uuid ) if uuid else None )( item.dataset.uuid), "tool_id": job.tool_id, "parameters": self._get_job_record(trans, job, follow), + "stderr": job.stderr, + "stdout": job.stdout, } return None https://bitbucket.org/galaxy/galaxy-central/commits/bd8b3e2a9938/ Changeset: bd8b3e2a9938 User: jmchilton Date: 2013-11-20 23:44:09 Summary: Allow raw display of composite files via API. Affected #: 1 file diff -r 23afd47845fec9b8169712d0e78ee8d2baf0d96d -r bd8b3e2a99389f0564a1ff0dbfb9e3d56e56f990 lib/galaxy/webapps/galaxy/api/datasets.py --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -265,7 +265,12 @@ except KeyError: pass if raw: - rval = open( hda.file_name ) + if filename and filename != 'index': + file_path = trans.app.object_store.get_filename(hda.dataset, extra_dir='dataset_%s_files' % hda.dataset.id, alt_name=filename) + else: + file_path = hda.file_name + rval = open( file_path ) + else: rval = hda.datatype.display_data( trans, hda, preview, filename, to_ext, chunk, **display_kwd ) https://bitbucket.org/galaxy/galaxy-central/commits/7a882c5fdd34/ Changeset: 7a882c5fdd34 User: jmchilton Date: 2013-11-20 23:44:09 Summary: Allow simple use of multi-page tools via API. In tools module, when process_state=='populate' (i.e. coming from tool API without existing tool_state specified), populate and validate state of all pages of tools. Affected #: 1 file diff -r bd8b3e2a99389f0564a1ff0dbfb9e3d56e56f990 -r 7a882c5fdd34636ae5ca98f63709c13a6648c3cf lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1811,7 +1811,8 @@ from API). May want an incremental version of the API also at some point, that is why this is not just called for_api. """ - state, state_new = self.__fetch_state( trans, incoming, history ) + all_pages = ( process_state == "populate" ) # If process_state = update, handle all pages at once. + state, state_new = self.__fetch_state( trans, incoming, history, all_pages=all_pages ) if state_new: # This feels a bit like a hack. It allows forcing full processing # of inputs even when there is no state in the incoming dictionary @@ -1837,7 +1838,7 @@ error_message = "One or more errors were found in the input you provided. The specific errors are marked below." return "tool_form.mako", dict( errors=errors, tool_state=state, incoming=incoming, error_message=error_message ) # If we've completed the last page we can execute the tool - elif state.page == self.last_page: + elif all_pages or state.page == self.last_page: return self.__handle_tool_execute( trans, incoming, params, history ) # Otherwise move on to the next page else: @@ -1891,7 +1892,7 @@ return 'message.mako', dict( status='info', message="The interface for this tool cannot be displayed", refresh_frames=['everything'] ) return 'tool_form.mako', dict( errors=errors, tool_state=state ) - def __fetch_state( self, trans, incoming, history ): + def __fetch_state( self, trans, incoming, history, all_pages ): # Get the state or create if not found if "tool_state" in incoming: encoded_state = string_to_object( incoming["tool_state"] ) @@ -1899,7 +1900,7 @@ state.decode( encoded_state, self, trans.app ) new = False else: - state = self.new_state( trans, history=history ) + state = self.new_state( trans, history=history, all_pages=all_pages ) new = True return state, new @@ -1917,15 +1918,17 @@ # Update state for all inputs on the current page taking new # values from `incoming`. if process_state == "update": - errors = self.update_state( trans, self.inputs_by_page[state.page], state.inputs, incoming, old_errors=old_errors or {}, source=source ) + inputs = self.inputs_by_page[state.page] + errors = self.update_state( trans, inputs, state.inputs, incoming, old_errors=old_errors or {}, source=source ) elif process_state == "populate": - errors = self.populate_state( trans, self.inputs_by_page[state.page], state.inputs, incoming, history, source=source ) + inputs = self.inputs + errors = self.populate_state( trans, inputs, state.inputs, incoming, history, source=source ) else: raise Exception("Unknown process_state type %s" % process_state) # If the tool provides a `validate_input` hook, call it. validate_input = self.get_hook( 'validate_input' ) if validate_input: - validate_input( trans, errors, state.inputs, self.inputs_by_page[state.page] ) + validate_input( trans, errors, state.inputs, inputs ) params = state.inputs return errors, params Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 20 Nov '13

20 Nov '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/98962174b0b1/ Changeset: 98962174b0b1 User: jmchilton Date: 2013-11-19 20:07:50 Summary: Allow implicit use of tool default parameter values when using API. Affected #: 1 file diff -r f2186f4796ad4c6aa2c9191f9804d8f5d59eb15d -r 98962174b0b12e239eff913353d683e396d2b004 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2015,12 +2015,15 @@ test_param_key = prefix + input.test_param.name else: test_param_key = group_prefix + input.test_param.name - test_param_error = None - test_incoming = get_incoming_value( incoming, test_param_key, None ) + # Get value of test param and determine current case + value, test_param_error = check_param_from_incoming( trans, + group_state, + input.test_param, + incoming, + test_param_key, + context, + source ) - # Get value of test param and determine current case - value, test_param_error = \ - check_param( trans, input.test_param, test_incoming, context, source=source ) current_case = input.get_current_case( value, trans ) # Current case has changed, throw away old state group_state = state[input.name] = {} @@ -2083,8 +2086,7 @@ if any_group_errors: errors[input.name] = group_errors else: - incoming_value = get_incoming_value( incoming, key, None ) - value, error = check_param( trans, input, incoming_value, context, source=source ) + value, error = check_param_from_incoming( trans, state, input, incoming, key, context, source ) if error: errors[ input.name ] = error state[ input.name ] = value @@ -3481,7 +3483,24 @@ else: return val + +def check_param_from_incoming( trans, state, input, incoming, key, context, source ): + """ + Unlike "update" state, this preserves default if no incoming value found. + This lets API user specify just a subset of params and allow defaults to be + used when available. + """ + default_input_value = state.get( input.name, None ) + incoming_value = get_incoming_value( incoming, key, default_input_value ) + value, error = check_param( trans, input, incoming_value, context, source=source ) + return value, error + + def get_incoming_value( incoming, key, default ): + """ + Fetch value from incoming dict directly or check special nginx upload + created variants of this key. + """ if "__" + key + "__is_composite" in incoming: composite_keys = incoming["__" + key + "__keys"].split() value = dict() https://bitbucket.org/galaxy/galaxy-central/commits/ef45d85b1f57/ Changeset: ef45d85b1f57 User: jmchilton Date: 2013-11-20 23:37:49 Summary: Merged in jmchilton/galaxy-central-fork-1 (pull request #265) Allow implicit use of tool default parameter values when using API. Affected #: 1 file diff -r e507124fbe79903378c4b0fe150f2f3fd816ad32 -r ef45d85b1f579c5bb4b07284dd96ad8403003c03 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2015,12 +2015,15 @@ test_param_key = prefix + input.test_param.name else: test_param_key = group_prefix + input.test_param.name - test_param_error = None - test_incoming = get_incoming_value( incoming, test_param_key, None ) + # Get value of test param and determine current case + value, test_param_error = check_param_from_incoming( trans, + group_state, + input.test_param, + incoming, + test_param_key, + context, + source ) - # Get value of test param and determine current case - value, test_param_error = \ - check_param( trans, input.test_param, test_incoming, context, source=source ) current_case = input.get_current_case( value, trans ) # Current case has changed, throw away old state group_state = state[input.name] = {} @@ -2083,8 +2086,7 @@ if any_group_errors: errors[input.name] = group_errors else: - incoming_value = get_incoming_value( incoming, key, None ) - value, error = check_param( trans, input, incoming_value, context, source=source ) + value, error = check_param_from_incoming( trans, state, input, incoming, key, context, source ) if error: errors[ input.name ] = error state[ input.name ] = value @@ -3481,7 +3483,24 @@ else: return val + +def check_param_from_incoming( trans, state, input, incoming, key, context, source ): + """ + Unlike "update" state, this preserves default if no incoming value found. + This lets API user specify just a subset of params and allow defaults to be + used when available. + """ + default_input_value = state.get( input.name, None ) + incoming_value = get_incoming_value( incoming, key, default_input_value ) + value, error = check_param( trans, input, incoming_value, context, source=source ) + return value, error + + def get_incoming_value( incoming, key, default ): + """ + Fetch value from incoming dict directly or check special nginx upload + created variants of this key. + """ if "__" + key + "__is_composite" in incoming: composite_keys = incoming["__" + key + "__keys"].split() value = dict() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 4 new changesets
by commits-noreply＠bitbucket.org 20 Nov '13

20 Nov '13

4 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b7a7c9516429/ Changeset: b7a7c9516429 User: dannon Date: 2013-11-16 23:36:17 Summary: Update S3ObjectStore init to supply config, config_xml Affected #: 1 file diff -r e057f2263f4bf8d2f5c277ae09360e36aa96cbbb -r b7a7c951642952129a05536c3e46cdae1c3000d1 lib/galaxy/objectstore/s3.py --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -34,7 +34,7 @@ Galaxy and S3. """ def __init__(self, config, config_xml=None): - super(S3ObjectStore, self).__init__() + super(S3ObjectStore, self).__init__(config, config_xml) self.config = config self.staging_path = self.config.file_path self.s3_conn = get_OS_connection(self.config) https://bitbucket.org/galaxy/galaxy-central/commits/95db640a4071/ Changeset: 95db640a4071 User: dannon Date: 2013-11-18 17:01:42 Summary: Merge. Affected #: 33 files diff -r b7a7c951642952129a05536c3e46cdae1c3000d1 -r 95db640a4071ba613814bfafe1a0448b93955e5d lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1570,113 +1570,10 @@ store in `self.tests`. """ self.tests = [] - # Composite datasets need a unique name: each test occurs in a fresh - # history, but we'll keep it unique per set of tests - composite_data_names_counter = 0 for i, test_elem in enumerate( tests_elem.findall( 'test' ) ): - name = test_elem.get( 'name', 'Test-%d' % (i+1) ) - maxseconds = int( test_elem.get( 'maxseconds', '120' ) ) - test = ToolTestBuilder( self, name, maxseconds ) - try: - for param_elem in test_elem.findall( "param" ): - attrib = dict( param_elem.attrib ) - if 'values' in attrib: - value = attrib[ 'values' ].split( ',' ) - elif 'value' in attrib: - value = attrib['value'] - else: - value = None - attrib['children'] = list( param_elem.getchildren() ) - if attrib['children']: - # At this time, we can assume having children only - # occurs on DataToolParameter test items but this could - # change and would cause the below parsing to change - # based upon differences in children items - attrib['metadata'] = [] - attrib['composite_data'] = [] - attrib['edit_attributes'] = [] - # Composite datasets need to be renamed uniquely - composite_data_name = None - for child in attrib['children']: - if child.tag == 'composite_data': - attrib['composite_data'].append( child ) - if composite_data_name is None: - # Generate a unique name; each test uses a - # fresh history - composite_data_name = '_COMPOSITE_RENAMED_%i_' \ - % ( composite_data_names_counter ) - composite_data_names_counter += 1 - elif child.tag == 'metadata': - attrib['metadata'].append( child ) - elif child.tag == 'metadata': - attrib['metadata'].append( child ) - elif child.tag == 'edit_attributes': - attrib['edit_attributes'].append( child ) - if composite_data_name: - # Composite datasets need implicit renaming; - # inserted at front of list so explicit declarations - # take precedence - attrib['edit_attributes'].insert( 0, { 'type': 'name', 'value': composite_data_name } ) - test.add_param( attrib.pop( 'name' ), value, attrib ) - for output_elem in test_elem.findall( "output" ): - attrib = dict( output_elem.attrib ) - name = attrib.pop( 'name', None ) - if name is None: - raise Exception( "Test output does not have a 'name'" ) - assert_elem = output_elem.find("assert_contents") - assert_list = None - # Trying to keep testing patch as localized as - # possible, this function should be relocated - # somewhere more conventional. - def convert_elem(elem): - """ Converts and XML element to a dictionary format, used by assertion checking code. """ - tag = elem.tag - attributes = dict( elem.attrib ) - child_elems = list( elem.getchildren() ) - converted_children = [] - for child_elem in child_elems: - converted_children.append( convert_elem(child_elem) ) - return {"tag" : tag, "attributes" : attributes, "children" : converted_children} - if assert_elem is not None: - assert_list = [] - for assert_child in list(assert_elem): - assert_list.append(convert_elem(assert_child)) - file = attrib.pop( 'file', None ) - # File no longer required if an list of assertions was present. - if assert_list is None and file is None: - raise Exception( "Test output does not have a 'file'") - attributes = {} - # Method of comparison - attributes['compare'] = attrib.pop( 'compare', 'diff' ).lower() - # Number of lines to allow to vary in logs (for dates, etc) - attributes['lines_diff'] = int( attrib.pop( 'lines_diff', '0' ) ) - # Allow a file size to vary if sim_size compare - attributes['delta'] = int( attrib.pop( 'delta', '10000' ) ) - attributes['sort'] = string_as_bool( attrib.pop( 'sort', False ) ) - attributes['extra_files'] = [] - attributes['assert_list'] = assert_list - if 'ftype' in attrib: - attributes['ftype'] = attrib['ftype'] - for extra in output_elem.findall( 'extra_files' ): - # File or directory, when directory, compare basename - # by basename - extra_type = extra.get( 'type', 'file' ) - extra_name = extra.get( 'name', None ) - assert extra_type == 'directory' or extra_name is not None, \ - 'extra_files type (%s) requires a name attribute' % extra_type - extra_value = extra.get( 'value', None ) - assert extra_value is not None, 'extra_files requires a value attribute' - extra_attributes = {} - extra_attributes['compare'] = extra.get( 'compare', 'diff' ).lower() - extra_attributes['delta'] = extra.get( 'delta', '0' ) - extra_attributes['lines_diff'] = int( extra.get( 'lines_diff', '0' ) ) - extra_attributes['sort'] = string_as_bool( extra.get( 'sort', False ) ) - attributes['extra_files'].append( ( extra_type, extra_value, extra_name, extra_attributes ) ) - test.add_output( name, file, attributes ) - except Exception, e: - test.error = True - test.exception = e + test = ToolTestBuilder( self, test_elem, i ) self.tests.append( test ) + def parse_input_page( self, input_elem, enctypes ): """ Parse a page of inputs. This basically just calls 'parse_input_elem', @@ -2155,11 +2052,19 @@ #remove extra files while len( group_state ) > len( writable_files ): del group_state[-1] + + # Add new fileupload as needed + while len( writable_files ) > len( group_state ): + new_state = {} + new_state['__index__'] = len( group_state ) + self.fill_in_new_state( trans, input.inputs, new_state, context ) + group_state.append( new_state ) + if any_group_errors: + group_errors.append( {} ) + # Update state - max_index = -1 for i, rep_state in enumerate( group_state ): rep_index = rep_state['__index__'] - max_index = max( max_index, rep_index ) rep_prefix = "%s_%d|" % ( key, rep_index ) rep_errors = self.populate_state( trans, input.inputs, @@ -2174,16 +2079,6 @@ group_errors.append( rep_errors ) else: group_errors.append( {} ) - # Add new fileupload as needed - offset = 1 - while len( writable_files ) > len( group_state ): - new_state = {} - new_state['__index__'] = max_index + offset - offset += 1 - self.fill_in_new_state( trans, input.inputs, new_state, context ) - group_state.append( new_state ) - if any_group_errors: - group_errors.append( {} ) # Were there *any* errors for any repetition? if any_group_errors: errors[input.name] = group_errors diff -r b7a7c951642952129a05536c3e46cdae1c3000d1 -r 95db640a4071ba613814bfafe1a0448b93955e5d lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -1,21 +1,23 @@ -import new, sys import os.path -import galaxy.util -import parameters from parameters import basic from parameters import grouping -from elementtree.ElementTree import XML +from galaxy.util import string_as_bool import logging log = logging.getLogger( __name__ ) + class ToolTestBuilder( object ): """ Encapsulates information about a tool test, and allows creation of a dynamic TestCase class (the unittest framework is very class oriented, doing dynamic tests in this way allows better integration) """ - def __init__( self, tool, name, maxseconds ): + + def __init__( self, tool, test_elem, i ): + name = test_elem.get( 'name', 'Test-%d' % (i + 1) ) + maxseconds = int( test_elem.get( 'maxseconds', '120' ) ) + self.tool = tool self.name = name self.maxseconds = maxseconds @@ -24,7 +26,125 @@ self.outputs = [] self.error = False self.exception = None - def add_param( self, name, value, extra ): + + self.__parse_elem( test_elem, i ) + + def __parse_elem( self, test_elem, i ): + # Composite datasets need a unique name: each test occurs in a fresh + # history, but we'll keep it unique per set of tests - use i (test #) + # and composite_data_names_counter (instance per test #) + composite_data_names_counter = 0 + try: + for param_elem in test_elem.findall( "param" ): + attrib = dict( param_elem.attrib ) + if 'values' in attrib: + value = attrib[ 'values' ].split( ',' ) + elif 'value' in attrib: + value = attrib['value'] + else: + value = None + attrib['children'] = list( param_elem.getchildren() ) + if attrib['children']: + # At this time, we can assume having children only + # occurs on DataToolParameter test items but this could + # change and would cause the below parsing to change + # based upon differences in children items + attrib['metadata'] = [] + attrib['composite_data'] = [] + attrib['edit_attributes'] = [] + # Composite datasets need to be renamed uniquely + composite_data_name = None + for child in attrib['children']: + if child.tag == 'composite_data': + attrib['composite_data'].append( child ) + if composite_data_name is None: + # Generate a unique name; each test uses a + # fresh history + composite_data_name = '_COMPOSITE_RENAMED_t%i_d%i' \ + % ( i, composite_data_names_counter ) + composite_data_names_counter += 1 + elif child.tag == 'metadata': + attrib['metadata'].append( child ) + elif child.tag == 'metadata': + attrib['metadata'].append( child ) + elif child.tag == 'edit_attributes': + attrib['edit_attributes'].append( child ) + if composite_data_name: + # Composite datasets need implicit renaming; + # inserted at front of list so explicit declarations + # take precedence + attrib['edit_attributes'].insert( 0, { 'type': 'name', 'value': composite_data_name } ) + self.__add_param( attrib.pop( 'name' ), value, attrib ) + for output_elem in test_elem.findall( "output" ): + attrib = dict( output_elem.attrib ) + name = attrib.pop( 'name', None ) + if name is None: + raise Exception( "Test output does not have a 'name'" ) + + assert_list = self.__parse_assert_list( output_elem ) + file = attrib.pop( 'file', None ) + # File no longer required if an list of assertions was present. + if not assert_list and file is None: + raise Exception( "Test output does not have a 'file' to compare with or list of assertions to check") + attributes = {} + # Method of comparison + attributes['compare'] = attrib.pop( 'compare', 'diff' ).lower() + # Number of lines to allow to vary in logs (for dates, etc) + attributes['lines_diff'] = int( attrib.pop( 'lines_diff', '0' ) ) + # Allow a file size to vary if sim_size compare + attributes['delta'] = int( attrib.pop( 'delta', '10000' ) ) + attributes['sort'] = string_as_bool( attrib.pop( 'sort', False ) ) + attributes['extra_files'] = [] + attributes['assert_list'] = assert_list + if 'ftype' in attrib: + attributes['ftype'] = attrib['ftype'] + for extra in output_elem.findall( 'extra_files' ): + attributes['extra_files'].append( self.__parse_extra_files_elem( extra ) ) + self.__add_output( name, file, attributes ) + except Exception, e: + self.error = True + self.exception = e + + def __parse_assert_list( self, output_elem ): + assert_elem = output_elem.find("assert_contents") + assert_list = None + + # Trying to keep testing patch as localized as + # possible, this function should be relocated + # somewhere more conventional. + def convert_elem(elem): + """ Converts and XML element to a dictionary format, used by assertion checking code. """ + tag = elem.tag + attributes = dict( elem.attrib ) + child_elems = list( elem.getchildren() ) + converted_children = [] + for child_elem in child_elems: + converted_children.append( convert_elem(child_elem) ) + return {"tag": tag, "attributes": attributes, "children": converted_children} + if assert_elem is not None: + assert_list = [] + for assert_child in list(assert_elem): + assert_list.append(convert_elem(assert_child)) + + return assert_list + + def __parse_extra_files_elem( self, extra ): + # File or directory, when directory, compare basename + # by basename + extra_type = extra.get( 'type', 'file' ) + extra_name = extra.get( 'name', None ) + assert extra_type == 'directory' or extra_name is not None, \ + 'extra_files type (%s) requires a name attribute' % extra_type + extra_value = extra.get( 'value', None ) + assert extra_value is not None, 'extra_files requires a value attribute' + extra_attributes = {} + extra_attributes['compare'] = extra.get( 'compare', 'diff' ).lower() + extra_attributes['delta'] = extra.get( 'delta', '0' ) + extra_attributes['lines_diff'] = int( extra.get( 'lines_diff', '0' ) ) + extra_attributes['sort'] = string_as_bool( extra.get( 'sort', False ) ) + return extra_type, extra_value, extra_name, extra_attributes + + def __add_param( self, name, value, extra ): try: if name not in self.tool.inputs: found_parameter = False @@ -43,8 +163,10 @@ except Exception, e: log.debug( "Error for tool %s: could not add test parameter %s. %s" % ( self.tool.id, name, e ) ) self.inputs.append( ( name, value, extra ) ) - def add_output( self, name, file, extra ): + + def __add_output( self, name, file, extra ): self.outputs.append( ( name, file, extra ) ) + def __expand_grouping_for_data_input( self, name, value, extra, grouping_name, grouping_value ): # Currently handles grouping.Conditional and grouping.Repeat if isinstance( grouping_value, grouping.Conditional ): @@ -93,20 +215,21 @@ if found_parameter: return True, new_value return False, value + def __add_uploaded_dataset( self, name, value, extra, input_parameter ): if value is None: assert input_parameter.optional, '%s is not optional. You must provide a valid filename.' % name return value if ( value, extra ) not in self.required_files: - self.required_files.append( ( value, extra ) ) #these files will be uploaded + self.required_files.append( ( value, extra ) ) # these files will be uploaded name_change = [ att for att in extra.get( 'edit_attributes', [] ) if att.get( 'type' ) == 'name' ] if name_change: - name_change = name_change[-1].get( 'value' ) #only the last name change really matters - value = name_change #change value for select to renamed uploaded file for e.g. composite dataset + name_change = name_change[-1].get( 'value' ) # only the last name change really matters + value = name_change # change value for select to renamed uploaded file for e.g. composite dataset else: for end in [ '.zip', '.gz' ]: if value.endswith( end ): value = value[ :-len( end ) ] break - value = os.path.basename( value ) #if uploading a file in a path other than root of test-data + value = os.path.basename( value ) # if uploading a file in a path other than root of test-data return value diff -r b7a7c951642952129a05536c3e46cdae1c3000d1 -r 95db640a4071ba613814bfafe1a0448b93955e5d lib/galaxy/web/base/controller.py --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -568,6 +568,31 @@ error( "Please wait until this dataset finishes uploading before attempting to view it." ) return hda + def get_history_dataset_association_from_ids( self, trans, id, history_id ): + # Just to echo other TODOs, there seems to be some overlap here, still + # this block appears multiple places (dataset show, history_contents + # show, upcoming history job show) so I am consolodating it here. + # Someone smarter than me should determine if there is some redundancy here. + + # for anon users: + #TODO: check login_required? + #TODO: this isn't actually most_recently_used (as defined in histories) + if( ( trans.user == None ) + and ( history_id == trans.security.encode_id( trans.history.id ) ) ): + history = trans.history + #TODO: dataset/hda by id (from history) OR check_ownership for anon user + hda = self.get_history_dataset_association( trans, history, id, + check_ownership=False, check_accessible=True ) + else: + #TODO: do we really need the history? + history = self.get_history( trans, history_id, + check_ownership=True, check_accessible=True, deleted=False ) + hda = self.get_history_dataset_association( trans, history, id, + check_ownership=True, check_accessible=True ) + return hda + + + def get_hda_list( self, trans, hda_ids, check_ownership=True, check_accessible=False, check_state=True ): """ Returns one or more datasets in a list. diff -r b7a7c951642952129a05536c3e46cdae1c3000d1 -r 95db640a4071ba613814bfafe1a0448b93955e5d lib/galaxy/web/framework/helpers/grids.py --- a/lib/galaxy/web/framework/helpers/grids.py +++ b/lib/galaxy/web/framework/helpers/grids.py @@ -321,7 +321,7 @@ class GridColumn( object ): def __init__( self, label, key=None, model_class=None, method=None, format=None, \ - link=None, attach_popup=False, visible=True, ncells=1, nowrap=False, \ + link=None, attach_popup=False, visible=True, nowrap=False, \ # Valid values for filterable are ['standard', 'advanced', None] filterable=None, sortable=True, label_id_prefix=None ): """Create a grid column.""" @@ -334,7 +334,6 @@ self.nowrap = nowrap self.attach_popup = attach_popup self.visible = visible - self.ncells = ncells self.filterable = filterable # Column must have a key to be sortable. self.sortable = ( self.key is not None and sortable ) diff -r b7a7c951642952129a05536c3e46cdae1c3000d1 -r 95db640a4071ba613814bfafe1a0448b93955e5d lib/galaxy/webapps/galaxy/api/datasets.py --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -255,24 +255,9 @@ datatype prior to display (the defult if raw is unspecified or explicitly false. """ raw = string_as_bool_or_none( raw ) - # Huge amount of code overlap with lib/galaxy/webapps/galaxy/api/history_content:show here. rval = '' try: - # for anon users: - #TODO: check login_required? - #TODO: this isn't actually most_recently_used (as defined in histories) - if( ( trans.user == None ) - and ( history_id == trans.security.encode_id( trans.history.id ) ) ): - history = trans.history - #TODO: dataset/hda by id (from history) OR check_ownership for anon user - hda = self.get_history_dataset_association( trans, history, history_content_id, - check_ownership=False, check_accessible=True ) - - else: - history = self.get_history( trans, history_id, - check_ownership=True, check_accessible=True, deleted=False ) - hda = self.get_history_dataset_association( trans, history, history_content_id, - check_ownership=True, check_accessible=True ) + hda = self.get_history_dataset_association_from_ids( trans, history_content_id, history_id ) display_kwd = kwd.copy() try: diff -r b7a7c951642952129a05536c3e46cdae1c3000d1 -r 95db640a4071ba613814bfafe1a0448b93955e5d lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -135,32 +135,17 @@ :returns: dictionary containing detailed HDA information .. seealso:: :func:`galaxy.web.base.controller.UsesHistoryDatasetAssociationMixin.get_hda_dict` """ - hda_dict = {} try: - # for anon users: - #TODO: check login_required? - #TODO: this isn't actually most_recently_used (as defined in histories) - if( ( trans.user == None ) - and ( history_id == trans.security.encode_id( trans.history.id ) ) ): - history = trans.history - #TODO: dataset/hda by id (from history) OR check_ownership for anon user - hda = self.get_history_dataset_association( trans, history, id, - check_ownership=False, check_accessible=True ) - else: - #TODO: do we really need the history? - history = self.get_history( trans, history_id, - check_ownership=True, check_accessible=True, deleted=False ) - hda = self.get_history_dataset_association( trans, history, id, - check_ownership=True, check_accessible=True ) + hda = self.get_history_dataset_association_from_ids( trans, id, history_id ) hda_dict = self.get_hda_dict( trans, hda ) hda_dict[ 'display_types' ] = self.get_old_display_applications( trans, hda ) hda_dict[ 'display_apps' ] = self.get_display_apps( trans, hda ) + return hda_dict except Exception, e: msg = "Error in history API at listing dataset: %s" % ( str(e) ) log.error( msg, exc_info=True ) trans.response.status = 500 return msg - return hda_dict @web.expose_api def create( self, trans, history_id, payload, **kwd ): diff -r b7a7c951642952129a05536c3e46cdae1c3000d1 -r 95db640a4071ba613814bfafe1a0448b93955e5d lib/galaxy/webapps/galaxy/api/provenance.py --- a/lib/galaxy/webapps/galaxy/api/provenance.py +++ b/lib/galaxy/webapps/galaxy/api/provenance.py @@ -42,11 +42,12 @@ if item is not None: if item.copied_from_library_dataset_dataset_association: item = item.copied_from_library_dataset_dataset_association + job = item.creating_job return { - "id" : trans.security.encode_id(item.id), - "uuid" : ( lambda uuid: str( uuid ) if uuid else None )( item.dataset.uuid), - "tool_id" : item.creating_job.tool_id, - "parameters" : self._get_job_record(trans, item.creating_job, follow) + "id": trans.security.encode_id(item.id), + "uuid": ( lambda uuid: str( uuid ) if uuid else None )( item.dataset.uuid), + "tool_id": job.tool_id, + "parameters": self._get_job_record(trans, job, follow), } return None @@ -59,8 +60,8 @@ out[in_d.name] = self._get_record(trans, in_d.dataset, follow) else: out[in_d.name] = { - "id" : trans.security.encode_id(in_d.dataset.id), - "uuid" : ( lambda uuid: str( uuid ) if uuid else None )( in_d.dataset.dataset.uuid ) + "id": trans.security.encode_id(in_d.dataset.id), + "uuid": ( lambda uuid: str( uuid ) if uuid else None )( in_d.dataset.dataset.uuid ), } return out @@ -75,5 +76,3 @@ controller_name = "ldda_provenance" provenance_item_class = "LibraryDatasetDatasetAssociation" provenance_item_id = "library_content_id" - - diff -r b7a7c951642952129a05536c3e46cdae1c3000d1 -r 95db640a4071ba613814bfafe1a0448b93955e5d lib/galaxy/webapps/galaxy/controllers/dataset.py --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -100,7 +100,7 @@ grids.TextColumn( "Name", key="name", # Link name to dataset's history. link=( lambda item: iff( item.history.deleted, None, dict( operation="switch", id=item.id ) ) ), filterable="advanced", attach_popup=True ), - HistoryColumn( "History", key="history", + HistoryColumn( "History", key="history", sortable=False, link=( lambda item: iff( item.history.deleted, None, dict( operation="switch_history", id=item.id ) ) ) ), grids.IndividualTagsColumn( "Tags", key="tags", model_tag_association_class=model.HistoryDatasetAssociationTagAssociation, filterable="advanced", grid_name="HistoryDatasetAssocationListGrid" ), StatusColumn( "Status", key="deleted", attach_popup=False ), @@ -113,11 +113,12 @@ key="free-text-search", visible=False, filterable="standard" ) ) operations = [ - grids.GridOperation( "Copy to current history", condition=( lambda item: not item.deleted ), async_compatible=False ), + grids.GridOperation( "Copy to current history", condition=( lambda item: not item.deleted ), async_compatible=True ), ] standard_filters = [] default_filter = dict( name="All", deleted="False", tags="All" ) preserve_state = False + use_async = True use_paging = True num_rows_per_page = 50 def build_initial_query( self, trans, **kwargs ): diff -r b7a7c951642952129a05536c3e46cdae1c3000d1 -r 95db640a4071ba613814bfafe1a0448b93955e5d lib/galaxy/webapps/galaxy/controllers/history.py --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -30,16 +30,13 @@ def get_value( self, trans, grid, history ): state_count_dict = self.get_hda_state_counts( trans, history ) - rval = [] + rval = '' for state in ( 'ok', 'running', 'queued', 'error' ): count = state_count_dict.get( state, 0 ) if count: - rval.append( '<div class="count-box state-color-%s">%s</div>' % ( state, count ) ) - else: - rval.append( '' ) + rval += '<div class="count-box state-color-%s">%s</div>' % (state, count) return rval - class HistoryListNameColumn( NameColumn ): def get_link( self, trans, grid, history ): link = None @@ -72,7 +69,7 @@ default_sort_key = "-update_time" columns = [ HistoryListNameColumn( "Name", key="name", attach_popup=True, filterable="advanced" ), - DatasetsByStateColumn( "Datasets", key="datasets_by_state", ncells=4, sortable=False ), + DatasetsByStateColumn( "Datasets", key="datasets_by_state", sortable=False ), grids.IndividualTagsColumn( "Tags", key="tags", model_tag_association_class=model.HistoryTagAssociation, \ filterable="advanced", grid_name="HistoryListGrid" ), grids.SharingStatusColumn( "Sharing", key="sharing", filterable="advanced", sortable=False ), @@ -118,13 +115,11 @@ # Custom column types class DatasetsByStateColumn( grids.GridColumn ): def get_value( self, trans, grid, history ): - rval = [] + rval = '' for state in ( 'ok', 'running', 'queued', 'error' ): total = sum( 1 for d in history.active_datasets if d.state == state ) if total: - rval.append( '<div class="count-box state-color-%s">%s</div>' % ( state, total ) ) - else: - rval.append( '' ) + rval += '<div class="count-box state-color-%s">%s</div>' % ( state, total ) return rval class SharedByColumn( grids.GridColumn ): @@ -138,7 +133,7 @@ default_filter = {} columns = [ grids.GridColumn( "Name", key="name", attach_popup=True ), # link=( lambda item: dict( operation="View", id=item.id ) ), attach_popup=True ), - DatasetsByStateColumn( "Datasets", ncells=4, sortable=False ), + DatasetsByStateColumn( "Datasets", sortable=False ), grids.GridColumn( "Created", key="create_time", format=time_ago ), grids.GridColumn( "Last Updated", key="update_time", format=time_ago ), SharedByColumn( "Shared by", key="user_id" ) diff -r b7a7c951642952129a05536c3e46cdae1c3000d1 -r 95db640a4071ba613814bfafe1a0448b93955e5d lib/galaxy/webapps/galaxy/controllers/visualization.py --- a/lib/galaxy/webapps/galaxy/controllers/visualization.py +++ b/lib/galaxy/webapps/galaxy/controllers/visualization.py @@ -147,11 +147,11 @@ # Grid definition. title = "Insert into visualization" template = "/tracks/add_to_viz.mako" - async_template = "/page/select_items_grid_async.mako" model_class = model.Visualization default_sort_key = "-update_time" use_async = True use_paging = False + show_item_checkboxes = True columns = [ grids.TextColumn( "Title", key="title", model_class=model.Visualization, filterable="standard" ), grids.TextColumn( "Dbkey", key="dbkey", model_class=model.Visualization ), diff -r b7a7c951642952129a05536c3e46cdae1c3000d1 -r 95db640a4071ba613814bfafe1a0448b93955e5d scripts/functional_tests.py --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -217,9 +217,9 @@ database_auto_migrate = False + galaxy_test_proxy_port = None + psu_production = False if start_server: - psu_production = False - galaxy_test_proxy_port = None if 'GALAXY_TEST_PSU_PRODUCTION' in os.environ: if not galaxy_test_port: raise Exception( 'Please set GALAXY_TEST_PORT to the port to which the proxy server will proxy' ) diff -r b7a7c951642952129a05536c3e46cdae1c3000d1 -r 95db640a4071ba613814bfafe1a0448b93955e5d static/scripts/galaxy.grids.js --- a/static/scripts/galaxy.grids.js +++ b/static/scripts/galaxy.grids.js @@ -119,433 +119,456 @@ } // Add filter arguments to data, placing "f-" in front of all arguments. - // FIXME: when underscore updated, use pairs function(). var self = this; - _.each(_.keys(self.attributes.filters), function(k) { - url_data['f-' + k] = self.attributes.filters[k]; + _.each(_.pairs(self.attributes.filters), function(k) { + url_data['f-' + k[0]] = k[1]; }); return url_data; } }); -// -// Code to handle grid operations: filtering, sorting, paging, and operations. -// +// grid view +var GridView = Backbone.View.extend({ -// Initialize grid controls -function init_grid_controls() { - // Initialize submit image elements. - $('.submit-image').each( function() { - // On mousedown, add class to simulate click. - $(this).mousedown( function() { - $(this).addClass('gray-background'); + // model + grid: null, + + // Initialize + initialize: function(grid) + { + this.grid = grid; + this.init_grid_elements(); + this.init_grid_controls(); + + // Initialize text filters to select text on click and use normal font when user is typing. + $('input[type=text]').each(function() { + $(this).click(function() { $(this).select(); } ) + .keyup(function () { $(this).css("font-style", "normal"); }); + }); + }, + + // Initialize grid controls + init_grid_controls: function() { + // Initialize submit image elements. + $('.submit-image').each( function() { + // On mousedown, add class to simulate click. + $(this).mousedown( function() { + $(this).addClass('gray-background'); + }); + + // On mouseup, add class to simulate click. + $(this).mouseup( function() { + $(this).removeClass('gray-background'); + }); }); - // On mouseup, add class to simulate click. - $(this).mouseup( function() { - $(this).removeClass('gray-background'); + // link + var self = this; + + // Initialize sort links. + $('.sort-link').each( function() { + $(this).click( function() { + self.set_sort_condition( $(this).attr('sort_key') ); + return false; + }); }); - }); + + // Initialize page links. + $('.page-link > a').each( function() { + $(this).click( function() { + self.set_page( $(this).attr('page_num') ); + return false; + }); + }); + + // Initialize categorical filters. + $('.categorical-filter > a').each( function() { + $(this).click( function() { + self.set_categorical_filter( $(this).attr('filter_key'), $(this).attr('filter_val') ); + return false; + }); + }); + + // Initialize text filters. + $('.text-filter-form').each( function() { + $(this).submit( function() { + var column_key = $(this).attr('column_key'); + var text_input_obj = $('#input-' + column_key + '-filter'); + var text_input = text_input_obj.val(); + text_input_obj.val(''); + self.add_filter_condition(column_key, text_input, true); + return false; + }); + }); + + // Initialize autocomplete for text inputs in search UI. + var t = $("#input-tags-filter"); + if (t.length) { + t.autocomplete(this.grid.history_tag_autocomplete_url, + { selectFirst: false, autoFill: false, highlight: false, mustMatch: false }); + } + + var t2 = $("#input-name-filter"); + if (t2.length) { + t2.autocomplete(this.grid.history_name_autocomplete_url, + { selectFirst: false, autoFill: false, highlight: false, mustMatch: false }); + } + + // Initialize standard, advanced search toggles. + $('.advanced-search-toggle').each( function() { + $(this).click( function() { + $("#standard-search").slideToggle('fast'); + $('#advanced-search').slideToggle('fast'); + return false; + }); + }); + }, + + // Initialize grid elements. + init_grid_elements : function() { - // Initialize sort links. - $('.sort-link').each( function() { - $(this).click( function() { - set_sort_condition( $(this).attr('sort_key') ); - return false; + // Initialize grid selection checkboxes. + $(".grid").each( function() { + var checkboxes = $(this).find("input.grid-row-select-checkbox"); + var check_count = $(this).find("span.grid-selected-count"); + var update_checked = function() { + check_count.text( $(checkboxes).filter(":checked").length ); + }; + + $(checkboxes).each( function() { + $(this).change(update_checked); + }); + update_checked(); }); - }); - - // Initialize page links. - $('.page-link > a').each( function() { - $(this).click( function() { - set_page( $(this).attr('page_num') ); - return false; + + // Initialize item labels. + var self = this; + $(".label").each( function() { + // If href has an operation in it, do operation when clicked. Otherwise do nothing. + var href = $(this).attr('href'); + if ( href !== undefined && href.indexOf('operation=') != -1 ) { + $(this).click( function() { + self.do_operation_from_href( $(this).attr('href') ); + return false; + }); + } }); - }); + + // Initialize ratings. + if ($('.community_rating_star').length !== 0) + $('.community_rating_star').rating({}); + + // Initialize item menu operations. + make_popup_menus(); + }, - // Initialize categorical filters. - $('.categorical-filter > a').each( function() { - $(this).click( function() { - set_categorical_filter( $(this).attr('filter_key'), $(this).attr('filter_val') ); + // Go back to page one; this is useful when a filter is applied. + go_page_one: function () { + // Need to go back to page 1 if not showing all. + var cur_page = this.grid.get('cur_page'); + if (cur_page !== null && cur_page !== undefined && cur_page !== 'all') { + this.grid.set('cur_page', 1); + } + }, + + // Add a condition to the grid filter; this adds the condition and refreshes the grid. + add_filter_condition: function (name, value, append) { + // Do nothing is value is empty. + if (value === "") { return false; + } + + // Add condition to grid. + this.grid.add_filter(name, value, append); + + // Add button that displays filter and provides a button to delete it. + var t = $("<span>" + value + "<a href='javascript:void(0);'><span class='delete-search-icon' /></a></span>"); + t.addClass('text-filter-val'); + var self = this; + t.click(function() { + // Remove filter condition. + self.grid.remove_filter(name, value); + + // Remove visible element. + $(this).remove(); + + self.go_page_one(); + self.update_grid(); }); - }); - - // Initialize text filters. - $('.text-filter-form').each( function() { - $(this).submit( function() { - var column_key = $(this).attr('column_key'); - var text_input_obj = $('#input-' + column_key + '-filter'); - var text_input = text_input_obj.val(); - text_input_obj.val(''); - add_filter_condition(column_key, text_input, true); + + var container = $('#' + name + "-filtering-criteria"); + container.append(t); + + this.go_page_one(); + this.update_grid(); + }, + + // Set sort condition for grid. + set_sort_condition: function (col_key) { + // Set new sort condition. New sort is col_key if sorting new column; if reversing sort on + // currently sorted column, sort is reversed. + var cur_sort = this.grid.get('sort_key'); + var new_sort = col_key; + if (cur_sort.indexOf(col_key) !== -1) { + // Reverse sort. + if (cur_sort.substring(0,1) !== '-') { + new_sort = '-' + col_key; + } else { + // Sort reversed by using just col_key. + } + } + + // Remove sort arrows elements. + $('.sort-arrow').remove(); + + // Add sort arrow element to new sort column. + var sort_arrow = (new_sort.substring(0,1) == '-') ? "↑" : "↓"; + var t = $("<span>" + sort_arrow + "</span>").addClass('sort-arrow'); + var th = $("#" + col_key + '-header'); + th.append(t); + + // Update grid. + this.grid.set('sort_key', new_sort); + this.go_page_one(); + this.update_grid(); + }, + + // Set new value for categorical filter. + set_categorical_filter: function (name, new_value) { + // Update filter hyperlinks to reflect new filter value. + var category_filter = this.grid.get('categorical_filters')[name], + cur_value = this.grid.get('filters')[name]; + var self = this; + $("." + name + "-filter").each( function() { + var text = $.trim( $(this).text() ); + var filter = category_filter[text]; + var filter_value = filter[name]; + if (filter_value == new_value) { + // Remove filter link since grid will be using this filter. It is assumed that + // this element has a single child, a hyperlink/anchor with text. + $(this).empty(); + $(this).addClass("current-filter"); + $(this).append(text); + } else if (filter_value == cur_value) { + // Add hyperlink for this filter since grid will no longer be using this filter. It is assumed that + // this element has a single child, a hyperlink/anchor. + $(this).empty(); + var t = $("<a href='#'>" + text + "</a>"); + t.click(function() { + self.set_categorical_filter( name, filter_value ); + }); + $(this).removeClass("current-filter"); + $(this).append(t); + } + }); + + // Update grid. + this.grid.add_filter(name, new_value); + this.go_page_one(); + this.update_grid(); + }, + + // Set page to view. + set_page: function (new_page) { + // Update page hyperlink to reflect new page. + var self = this; + $(".page-link").each( function() { + var id = $(this).attr('id'), + page_num = parseInt( id.split("-")[2], 10 ), // Id has form 'page-link-<page_num> + cur_page = self.grid.get('cur_page'), + text; + if (page_num === new_page) { + // Remove link to page since grid will be on this page. It is assumed that + // this element has a single child, a hyperlink/anchor with text. + text = $(this).children().text(); + $(this).empty(); + $(this).addClass("inactive-link"); + $(this).text(text); + } + else if (page_num === cur_page) { + // Add hyperlink to this page since grid will no longer be on this page. It is assumed that + // this element has a single child, a hyperlink/anchor. + text = $(this).text(); + $(this).empty(); + $(this).removeClass("inactive-link"); + var t = $("<a href='#'>" + text + "</a>"); + t.click(function() { + set_page(page_num); + }); + $(this).append(t); + } + }); + + var maintain_page_links = true; + if (new_page === "all") { + this.grid.set('cur_page', new_page); + maintain_page_links = false; + } else { + this.grid.set('cur_page', parseInt(new_page, 10)); + } + this.update_grid(maintain_page_links); + }, + + // Perform a grid operation. + do_operation: function (operation, item_ids) { + operation = operation.toLowerCase(); + + // Update grid. + this.grid.set({ + operation: operation, + item_ids: item_ids + }); + + // Do operation. If operation cannot be performed asynchronously, redirect to location. + if (this.grid.can_async_op(operation)) { + this.update_grid(true); + } + else { + this.go_to_URL(); + } + }, + + // Perform a hyperlink click that initiates an operation. If there is no operation, ignore click. + do_operation_from_href: function (href) { + // Get operation, id in hyperlink's href. + var href_parts = href.split("?"); + if (href_parts.length > 1) { + var href_parms_str = href_parts[1]; + var href_parms = href_parms_str.split("&"); + var operation = null; + var id = -1; + for (var index = 0; index < href_parms.length; index++) { + if (href_parms[index].indexOf('operation') != -1) { + // Found operation parm; get operation value. + operation = href_parms[index].split('=')[1]; + } else if (href_parms[index].indexOf('id') != -1) { + // Found id parm; get id value. + id = href_parms[index].split('=')[1]; + } + } + // Do operation. + this.do_operation(operation, id); return false; + } + }, + + // Navigate window to the URL defined by url_args. This method can be used to short-circuit grid AJAXing. + go_to_URL: function () { + // Not async request. + this.grid.set('async', false); + + // Go. + window.location = this.grid.get('url_base') + "?" + $.param(this.grid.get_url_data()); + + // Clear grid of transient request attributes. + this.grid.set({ + operation: undefined, + item_ids: undefined }); - }); - - // Initialize autocomplete for text inputs in search UI. - var t = $("#input-tags-filter"); - if (t.length) { - t.autocomplete(history_tag_autocomplete_url, - { selectFirst: false, autoFill: false, highlight: false, mustMatch: false }); - } - - var t2 = $("#input-name-filter"); - if (t2.length) { - t2.autocomplete(history_name_autocomplete_url, - { selectFirst: false, autoFill: false, highlight: false, mustMatch: false }); - } - - // Initialize standard, advanced search toggles. - $('.advanced-search-toggle').each( function() { - $(this).click( function() { - $("#standard-search").slideToggle('fast'); - $('#advanced-search').slideToggle('fast'); - return false; - }); - }); -} - -// Initialize grid elements. -function init_grid_elements() { - // Initialize grid selection checkboxes. - $(".grid").each( function() { - var checkboxes = $(this).find("input.grid-row-select-checkbox"); - var check_count = $(this).find("span.grid-selected-count"); - var update_checked = function() { - check_count.text( $(checkboxes).filter(":checked").length ); - }; - - $(checkboxes).each( function() { - $(this).change(update_checked); - }); - update_checked(); - }); - - // Initialize item labels. - $(".label").each( function() { - // If href has an operation in it, do operation when clicked. Otherwise do nothing. - var href = $(this).attr('href'); - if ( href !== undefined && href.indexOf('operation=') != -1 ) { - $(this).click( function() { - do_operation_from_href( $(this).attr('href') ); - return false; - }); - } - }); - - // Initialize ratings. - if ($('.community_rating_star').length !== 0) - $('.community_rating_star').rating({}); - - // Initialize item menu operations. - make_popup_menus(); -} - -// Go back to page one; this is useful when a filter is applied. -function go_page_one() { - // Need to go back to page 1 if not showing all. - var cur_page = grid.get('cur_page'); - if (cur_page !== null && cur_page !== undefined && cur_page !== 'all') { - grid.set('cur_page', 1); - } -} - -// Add a condition to the grid filter; this adds the condition and refreshes the grid. -function add_filter_condition(name, value, append) { - // Do nothing is value is empty. - if (value === "") { - return false; - } - - // Add condition to grid. - grid.add_filter(name, value, append); - - // Add button that displays filter and provides a button to delete it. - var t = $("<span>" + value + "<a href='javascript:void(0);'><span class='delete-search-icon' /></a></span>"); - t.addClass('text-filter-val'); - t.click(function() { - // Remove filter condition. - grid.remove_filter(name, value); - - // Remove visible element. - $(this).remove(); - - go_page_one(); - update_grid(); - }); - - var container = $('#' + name + "-filtering-criteria"); - container.append(t); - - go_page_one(); - update_grid(); -} - -// Add tag to grid filter. -function add_tag_to_grid_filter(tag_name, tag_value) { - // Put tag name and value together. - var tag = tag_name + (tag_value !== undefined && tag_value !== "" ? ":" + tag_value : ""); - $('#advanced-search').show('fast'); - add_filter_condition("tags", tag, true); -} - -// Set sort condition for grid. -function set_sort_condition(col_key) { - // Set new sort condition. New sort is col_key if sorting new column; if reversing sort on - // currently sorted column, sort is reversed. - var cur_sort = grid.get('sort_key'); - var new_sort = col_key; - if (cur_sort.indexOf(col_key) !== -1) { - // Reverse sort. - if (cur_sort.substring(0,1) !== '-') { - new_sort = '-' + col_key; - } else { - // Sort reversed by using just col_key. - } - } - - // Remove sort arrows elements. - $('.sort-arrow').remove(); - - // Add sort arrow element to new sort column. - var sort_arrow = (new_sort.substring(0,1) == '-') ? "↑" : "↓"; - var t = $("<span>" + sort_arrow + "</span>").addClass('sort-arrow'); - var th = $("#" + col_key + '-header'); - th.append(t); - - // Update grid. - grid.set('sort_key', new_sort); - go_page_one(); - update_grid(); -} - -// Set new value for categorical filter. -function set_categorical_filter(name, new_value) { - // Update filter hyperlinks to reflect new filter value. - var category_filter = grid.get('categorical_filters')[name], - cur_value = grid.get('filters')[name]; - $("." + name + "-filter").each( function() { - var text = $.trim( $(this).text() ); - var filter = category_filter[text]; - var filter_value = filter[name]; - if (filter_value == new_value) { - // Remove filter link since grid will be using this filter. It is assumed that - // this element has a single child, a hyperlink/anchor with text. - $(this).empty(); - $(this).addClass("current-filter"); - $(this).append(text); - } else if (filter_value == cur_value) { - // Add hyperlink for this filter since grid will no longer be using this filter. It is assumed that - // this element has a single child, a hyperlink/anchor. - $(this).empty(); - var t = $("<a href='#'>" + text + "</a>"); - t.click(function() { - set_categorical_filter( name, filter_value ); - }); - $(this).removeClass("current-filter"); - $(this).append(t); - } - }); - - // Update grid. - grid.add_filter(name, new_value); - go_page_one(); - update_grid(); -} - -// Set page to view. -function set_page(new_page) { - // Update page hyperlink to reflect new page. - $(".page-link").each( function() { - var id = $(this).attr('id'), - page_num = parseInt( id.split("-")[2], 10 ), // Id has form 'page-link-<page_num> - cur_page = grid.get('cur_page'), - text; - if (page_num === new_page) { - // Remove link to page since grid will be on this page. It is assumed that - // this element has a single child, a hyperlink/anchor with text. - text = $(this).children().text(); - $(this).empty(); - $(this).addClass("inactive-link"); - $(this).text(text); - } - else if (page_num === cur_page) { - // Add hyperlink to this page since grid will no longer be on this page. It is assumed that - // this element has a single child, a hyperlink/anchor. - text = $(this).text(); - $(this).empty(); - $(this).removeClass("inactive-link"); - var t = $("<a href='#'>" + text + "</a>"); - t.click(function() { - set_page(page_num); - }); - $(this).append(t); - } - }); - - var maintain_page_links = true; - if (new_page === "all") { - grid.set('cur_page', new_page); - maintain_page_links = false; - } else { - grid.set('cur_page', parseInt(new_page, 10)); - } - update_grid(maintain_page_links); -} - -// Perform a grid operation. -function do_operation(operation, item_ids) { - operation = operation.toLowerCase(); + }, // Update grid. - grid.set({ - operation: operation, - item_ids: item_ids - }); - - // Do operation. If operation cannot be performed asynchronously, redirect to location. - if (grid.can_async_op(operation)) { - update_grid(true); - } - else { - go_to_URL(); - } -} + update_grid: function (maintain_page_links) { + // If grid is not using async, then go to URL. + if (!this.grid.get('async')) { + this.go_to_URL(); + return; + } -// Perform a hyperlink click that initiates an operation. If there is no operation, ignore click. -function do_operation_from_href(href) { - // Get operation, id in hyperlink's href. - var href_parts = href.split("?"); - if (href_parts.length > 1) { - var href_parms_str = href_parts[1]; - var href_parms = href_parms_str.split("&"); - var operation = null; - var id = -1; - for (var index = 0; index < href_parms.length; index++) { - if (href_parms[index].indexOf('operation') != -1) { - // Found operation parm; get operation value. - operation = href_parms[index].split('=')[1]; - } else if (href_parms[index].indexOf('id') != -1) { - // Found id parm; get id value. - id = href_parms[index].split('=')[1]; + // If there's an operation, do POST; otherwise, do GET. + var method = (this.grid.get('operation') ? "POST" : "GET" ); + $('.loading-elt-overlay').show(); // Show overlay to indicate loading and prevent user actions. + var self = this; + $.ajax({ + type: method, + url: self.grid.get('url_base'), + data: self.grid.get_url_data(), + error: function() { alert( "Grid refresh failed" ); }, + success: function(response_text) { + // HACK: use a simple string to separate the elements in the + // response: (1) table body; (2) number of pages in table; and (3) message. + var parsed_response_text = response_text.split("*****"); + + // Update grid body and footer. + $('#grid-table-body').html(parsed_response_text[0]); + // FIXME: this does not work at all; what's needed is a function + // that updates page links when number of pages changes. + $('#grid-table-footer').html(parsed_response_text[1]); + + // Trigger custom event to indicate grid body has changed. + $('#grid-table-body').trigger('update'); + + // Init grid. + self.init_grid_elements(); + make_popup_menus(); + + // Hide loading overlay. + $('.loading-elt-overlay').hide(); + + // Show message if there is one. + var message = $.trim( parsed_response_text[2] ); + if (message !== "") { + $('#grid-message').html( message ).show(); + setTimeout( function() { $('#grid-message').hide(); }, 5000); + } + }, + complete: function() { + // Clear grid of transient request attributes. + self.grid.set({ + operation: undefined, + item_ids: undefined + }); + } + }); + }, + + check_all_items: function () { + var chk_all = document.getElementById('check_all'), + checks = document.getElementsByTagName('input'), + total = 0, + i; + if ( chk_all.checked === true ) { + for ( i=0; i < checks.length; i++ ) { + if ( checks[i].name.indexOf( 'id' ) !== -1) { + checks[i].checked = true; + total++; + } } } - // Do operation. - do_operation(operation, id); - return false; - } -} + else { + for ( i=0; i < checks.length; i++ ) { + if ( checks[i].name.indexOf( 'id' ) !== -1) { + checks[i].checked = false; + } -// Navigate window to the URL defined by url_args. This method can be used to short-circuit grid AJAXing. -function go_to_URL() { - // Not async request. - grid.set('async', false); - - // Go. - window.location = grid.get('url_base') + "?" + $.param(grid.get_url_data()); -} - -// Update grid. -function update_grid(maintain_page_links) { - // If grid is not using async, then go to URL. - if (!grid.get('async')) { - go_to_URL(); - return; - } - - // If there's an operation, do POST; otherwise, do GET. - var method = (grid.get('operation') ? "POST" : "GET" ); - $('.loading-elt-overlay').show(); // Show overlay to indicate loading and prevent user actions. - $.ajax({ - type: method, - url: grid.get('url_base'), - data: grid.get_url_data(), - error: function() { alert( "Grid refresh failed" ); }, - success: function(response_text) { - // HACK: use a simple string to separate the elements in the - // response: (1) table body; (2) number of pages in table; and (3) message. - var parsed_response_text = response_text.split("*****"); - - // Update grid body and footer. - $('#grid-table-body').html(parsed_response_text[0]); - // FIXME: this does not work at all; what's needed is a function - // that updates page links when number of pages changes. - $('#grid-table-footer').html(parsed_response_text[1]); - - // Trigger custom event to indicate grid body has changed. - $('#grid-table-body').trigger('update'); - - // Init grid. - init_grid_elements(); - make_popup_menus(); - - // Hide loading overlay. - $('.loading-elt-overlay').hide(); - - // Show message if there is one. - var message = $.trim( parsed_response_text[2] ); - if (message !== "") { - $('#grid-message').html( message ).show(); - setTimeout( function() { $('#grid-message').hide(); }, 5000); - } - }, - complete: function() { - // Clear grid of transient request attributes. - grid.set({ - operation: undefined, - item_ids: undefined - }); - } - }); -} - -function check_all_items() { - var chk_all = document.getElementById('check_all'), - checks = document.getElementsByTagName('input'), - total = 0, - i; - if ( chk_all.checked === true ) { - for ( i=0; i < checks.length; i++ ) { - if ( checks[i].name.indexOf( 'id' ) !== -1) { - checks[i].checked = true; - total++; } } + this.init_grid_elements(); + }, + + // confirmation/submission of operation request + submit_operation: function (selected_button, confirmation_text) + { + // verify in any item is selected + var number_of_checked_ids = $('input[name="id"]:checked').length; + if (!number_of_checked_ids > 0) + return false; + + // show confirmation box + if (confirmation_text != 'None' && confirmation_text != '') + if(!confirm(confirmation_text)) + return false; + + // collect ids + var operation_name = $(selected_button).val(); + var item_ids = []; + $('input[name=id]:checked').each(function() { + item_ids.push( $(this).val() ); + }); + this.do_operation(operation_name, item_ids); + + // return + return true; } - else { - for ( i=0; i < checks.length; i++ ) { - if ( checks[i].name.indexOf( 'id' ) !== -1) { - checks[i].checked = false; - } - } - } - init_grid_elements(); -} - - -// confirmation/submission of operation request -function submit_operation(selected_button, confirmation_text) -{ - // verify in any item is selected - var number_of_checked_ids = $('input[name="id"]:checked').length; - if (!number_of_checked_ids > 0) - return false; - - // show confirmation box - if (confirmation_text != 'None' && confirmation_text != '') - if(!confirm(confirmation_text)) - return false; - - // add ids - var operation_name = selected_button.value; - var item_ids = []; - $('input[name=id]:checked').each(function() { - item_ids.push( $(this).val() ); - }); - do_operation(operation_name, item_ids); - - // return - return true; -} +}); This diff is so big that we needed to truncate the remainder. https://bitbucket.org/galaxy/galaxy-central/commits/7b889ba6fd44/ Changeset: 7b889ba6fd44 User: dannon Date: 2013-11-20 23:30:03 Summary: S3/Swift objectstore refactoring. Affected #: 3 files diff -r 95db640a4071ba613814bfafe1a0448b93955e5d -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b lib/galaxy/objectstore/__init__.py --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -11,9 +11,9 @@ import threading from galaxy import util +from galaxy.exceptions import ObjectInvalid, ObjectNotFound from galaxy.jobs import Sleeper from galaxy.model import directory_hash_id -from galaxy.exceptions import ObjectNotFound, ObjectInvalid from galaxy.util.odict import odict from sqlalchemy.orm import object_session @@ -609,9 +609,12 @@ if store == 'disk': return DiskObjectStore(config=config, config_xml=config_xml) - elif store == 's3' or store == 'swift': + elif store == 's3': from galaxy.objectstore.s3 import S3ObjectStore return S3ObjectStore(config=config, config_xml=config_xml) + elif store == 'swift': + from galaxy.objectstore.s3 import SwiftObjectStore + return SwiftObjectStore(config=config, config_xml=config_xml) elif store == 'distributed': return DistributedObjectStore(config=config, fsmon=fsmon, config_xml=config_xml) elif store == 'hierarchical': diff -r 95db640a4071ba613814bfafe1a0448b93955e5d -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b lib/galaxy/objectstore/s3.py --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -33,15 +33,13 @@ cache exists that is used as an intermediate location for files between Galaxy and S3. """ - def __init__(self, config, config_xml=None): + def __init__(self, config, config_xml): super(S3ObjectStore, self).__init__(config, config_xml) self.config = config self.staging_path = self.config.file_path - self.s3_conn = get_OS_connection(self.config) - self.bucket = self._get_bucket(self.config.os_bucket_name) - self.use_rr = self.config.os_use_reduced_redundancy - self.cache_size = self.config.object_store_cache_size self.transfer_progress = 0 + self._parse_config_xml(config_xml) + self._configure_connection() # Clean cache only if value is set in universe_wsgi.ini if self.cache_size != -1: # Convert GBs to bytes for comparison @@ -58,6 +56,31 @@ except OSError: self.use_axel = False + def _configure_connection(self): + log.debug("Configuring S3 Connection") + self.conn = S3Connection(self.access_key, self.secret_key) + + def _parse_config_xml(self, config_xml): + try: + a_xml = config_xml.findall('auth')[0] + self.access_key = a_xml.get('access_key') + self.secret_key = a_xml.get('secret_key') + b_xml = config_xml.findall('bucket')[0] + self.bucket = b_xml.get('name') + self.use_rr = b_xml.get('use_reduced_redundancy', False) + cn_xml = config_xml.findall('connection')[0] + self.host = cn_xml.get('host', None) + self.port = int(cn_xml.get('port', 6000)) + self.is_secure = cn_xml.get('is_secure', True) + self.conn_path = cn_xml.get('conn_path', '/') + c_xml = config_xml.findall('cache')[0] + self.cache_size = float(c_xml.get('size', -1)) + self.cache_path = c_xml.get('path') + except Exception: + # Toss it back up after logging, we can't continue loading at this point. + log.exception("Malformed ObjectStore Configuration XML -- unable to continue") + raise + def __cache_monitor(self): time.sleep(2) # Wait for things to load before starting the monitor while self.running: @@ -127,7 +150,7 @@ it a few times. Raise error is connection is not established. """ for i in range(5): try: - bucket = self.s3_conn.get_bucket(bucket_name) + bucket = self.conn.get_bucket(bucket_name) log.debug("Using cloud object store with bucket '%s'" % bucket.name) return bucket except S3ResponseError: @@ -290,7 +313,8 @@ # print "Pushing cache file '%s' of size %s bytes to key '%s'" % (source_file, os.path.getsize(source_file), rel_path) # print "+ Push started at '%s'" % start_time mb_size = os.path.getsize(source_file) / 1e6 - if mb_size < 60 or self.config.object_store == 'swift': + #DBTODO Hack, refactor this logic. + if mb_size < 60 or type(self) == SwiftObjectStore: self.transfer_progress = 0 # Reset transfer progress counter key.set_contents_from_filename(source_file, reduced_redundancy=self.use_rr, cb=self._transfer_cb, num_cb=10) @@ -512,24 +536,20 @@ return 0.0 -def get_OS_connection(config): +class SwiftObjectStore(S3ObjectStore): """ - Get a connection object for a cloud Object Store specified in the config. - Currently, this is a ``boto`` connection object. + Object store that stores objects as items in a Swift bucket. A local + cache exists that is used as an intermediate location for files between + Galaxy and Swift. """ - log.debug("Getting a connection object for '{0}' object store".format(config.object_store)) - a_key = config.os_access_key - s_key = config.os_secret_key - if config.object_store == 's3': - return S3Connection(a_key, s_key) - else: - # Establish the connection now - calling_format = boto.s3.connection.OrdinaryCallingFormat() - s3_conn = boto.connect_s3(aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=config.os_is_secure, - host=config.os_host, - port=int(config.os_port), - calling_format=calling_format, - path=config.os_conn_path) - return s3_conn + + def _configure_connection(self): + log.debug("Configuring Swift Connection") + self.conn = boto.connect_s3(aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + is_secure=self.is_secure, + host=self.host, + port=self.port, + calling_format=boto.s3.connection.OrdinaryCallingFormat(), + path=self.conn_path) + diff -r 95db640a4071ba613814bfafe1a0448b93955e5d -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b object_store_conf.xml.sample --- a/object_store_conf.xml.sample +++ b/object_store_conf.xml.sample @@ -20,5 +20,16 @@ <extra_dir type="temp" path="database/tmp3"/><extra_dir type="job_work" path="database/job_working_directory3"/></object_store> + </backends></object_store> + https://bitbucket.org/galaxy/galaxy-central/commits/e507124fbe79/ Changeset: e507124fbe79 User: dannon Date: 2013-11-20 23:30:28 Summary: Merge. Affected #: 55 files diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 config/plugins/visualizations/scatterplot/Gruntfile.js --- a/config/plugins/visualizations/scatterplot/Gruntfile.js +++ b/config/plugins/visualizations/scatterplot/Gruntfile.js @@ -9,7 +9,7 @@ // compile all hb templates into a single file in the build dir compile: { options: { - namespace: 'Templates', + namespace: 'scatterplot', processName : function( filepath ){ return filepath.match( /\w*\.handlebars/ )[0].replace( '.handlebars', '' ); } diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 config/plugins/visualizations/scatterplot/src/scatterplot-config-editor.js --- a/config/plugins/visualizations/scatterplot/src/scatterplot-config-editor.js +++ b/config/plugins/visualizations/scatterplot/src/scatterplot-config-editor.js @@ -37,14 +37,19 @@ /** initialize requires a configuration Object containing a dataset Object */ initialize : function( attributes ){ //console.log( this + '.initialize, attributes:', attributes ); - if( !attributes || !attributes.config || !attributes.config.dataset ){ + if( !attributes || !attributes.config || !attributes.dataset ){ throw new Error( "ScatterplotView requires a configuration and dataset" ); } - this.dataset = attributes.config.dataset; + //console.log( 'config:', attributes.config ); + + this.dataset = attributes.dataset; //console.log( 'dataset:', this.dataset ); +//TODO: ScatterplotView -> ScatterplotDisplay, this.plotView -> this.display this.plotView = new ScatterplotView({ + dataset : attributes.dataset, config : attributes.config +//TODO: if data }); }, @@ -197,8 +202,8 @@ // parse the column values for both indeces (for the data fetch) and names (for the chart) var $dataControls = this.$el.find( '#data-control' ); var settings = { - xColumn : $dataControls.find( '[name="xColumn"]' ).val(), - yColumn : $dataControls.find( '[name="yColumn"]' ).val() + xColumn : Number( $dataControls.find( '[name="xColumn"]' ).val() ), + yColumn : Number( $dataControls.find( '[name="yColumn"]' ).val() ) }; if( $dataControls.find( '#include-id-checkbox' ).prop( 'checked' ) ){ settings.idColumn = $dataControls.find( '[name="idColumn"]' ).val(); @@ -229,9 +234,9 @@ }); ScatterplotConfigEditor.templates = { - mainLayout : Templates.editor, - dataControl : Templates.datacontrol, - chartControl : Templates.chartcontrol + mainLayout : scatterplot.editor, + dataControl : scatterplot.datacontrol, + chartControl : scatterplot.chartcontrol }; //============================================================================== diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 config/plugins/visualizations/scatterplot/src/scatterplot-display.js --- a/config/plugins/visualizations/scatterplot/src/scatterplot-display.js +++ b/config/plugins/visualizations/scatterplot/src/scatterplot-display.js @@ -10,14 +10,10 @@ //TODO: should be a view on visualization(revision) model defaults : { - dataset : { - }, metadata : { dataLines : undefined }, - ajaxFn : null, - pagination : { currPage : 0, perPage : 3000 @@ -48,6 +44,7 @@ initialize : function( attributes ){ this.config = _.extend( _.clone( this.defaults ), attributes.config || {}); + this.dataset = attributes.dataset; //console.debug( this + '.config:', this.config ); }, @@ -65,7 +62,7 @@ //console.debug( 'currPage', this.config.pagination.currPage ); var view = this; //TODO: very tied to datasets - should be generalized eventually - xhr = jQuery.getJSON( '/api/datasets/' + this.config.dataset.id, { + xhr = jQuery.getJSON( '/api/datasets/' + this.dataset.id, { data_type : 'raw_data', provider : 'dataset-column', limit : this.config.pagination.perPage, @@ -151,7 +148,7 @@ }, renderLineInfo : function( data ){ - var totalLines = this.config.dataset.metadata_data_lines || 'an unknown number of', + var totalLines = this.dataset.metadata_data_lines || 'an unknown number of', lineStart = ( this.config.pagination.currPage * this.config.pagination.perPage ), lineEnd = lineStart + data.length; return $( '<p/>' ).addClass( 'scatterplot-data-info' ) @@ -168,9 +165,9 @@ } //TODO: cache numPages/numLines in config var view = this, - dataLines = this.config.dataset.metadata_data_lines, + dataLines = this.dataset.metadata_data_lines, numPages = ( dataLines )?( Math.ceil( dataLines / this.config.pagination.perPage ) ):( undefined ); - //console.debug( 'data:', this.config.dataset.metadata_data_lines, 'numPages:', numPages ); + //console.debug( 'data:', this.dataset.metadata_data_lines, 'numPages:', numPages ); // prev next buttons var $prev = makePage$Li( 'Prev' ).click( function(){ @@ -207,9 +204,9 @@ } //TODO: cache numPages/numLines in config var view = this, - dataLines = this.config.dataset.metadata_data_lines, + dataLines = this.dataset.metadata_data_lines, numPages = ( dataLines )?( Math.ceil( dataLines / this.config.pagination.perPage ) ):( undefined ); - //console.debug( 'data:', this.config.dataset.metadata_data_lines, 'numPages:', numPages ); + //console.debug( 'data:', this.dataset.metadata_data_lines, 'numPages:', numPages ); // page numbers (as separate control) //var $paginationContainer = $( '<div/>' ).addClass( 'pagination-container' ), diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 config/plugins/visualizations/scatterplot/src/visualization-templates.html --- a/config/plugins/visualizations/scatterplot/src/visualization-templates.html +++ /dev/null @@ -1,197 +0,0 @@ -<script type="text/template" class="template-visualization" id="template-visualization-scatterplotControlForm"> -{{! main layout }} - -<h1>WHAAAAA?</h1> -<div class="scatterplot-container chart-container tabbable tabs-left"> - {{! tab buttons/headers using Bootstrap }} - <ul class="nav nav-tabs"> - {{! start with the data controls as the displayed tab }} - <li class="active"> - <a title="Use this tab to change which data are used" - href="#data-control" data-toggle="tab">Data Controls</a> - </li> - <li> - <a title="Use this tab to change how the chart is drawn" - href="#chart-control" data-toggle="tab" >Chart Controls</a> - </li> - <li> - <a title="This tab will display overall statistics for your data" - href="#stats-display" data-toggle="tab">Statistics</a> - </li> - <li> - <a title="This tab will display the chart" - href="#chart-display" data-toggle="tab">Chart</a> - - <div id="loading-indicator" style="display: none;"> - <img class="loading-img" src="{{loadingIndicatorImagePath}}" /> - <span class="loading-message">{{message}}</span> - </div> - </li> - </ul> - - {{! data form, chart config form, stats, and chart all get their own tab }} - <div class="tab-content"> - {{! ---------------------------- tab for data settings form }} - <div id="data-control" class="tab-pane active"> - {{! rendered separately }} - </div> - - {{! ---------------------------- tab for chart graphics control form }} - <div id="chart-control" class="tab-pane"> - {{! rendered separately }} - </div> - - {{! ---------------------------- tab for data statistics }} - <div id="stats-display" class="tab-pane"> - <p class="help-text">By column:</p> - <table id="chart-stats-table"> - <thead><th></th><th>X</th><th>Y</th></thead> - {{#each stats}} - <tr><td>{{name}}</td><td>{{xval}}</td><td>{{yval}}</td></tr> - </tr> - {{/each}} - </table> - </div> - - {{! ---------------------------- tab for actual chart }} - <div id="chart-display" class="tab-pane"> - <svg width="{{width}}" height="{{height}}"></svg> - </div> - - </div>{{! end .tab-content }} -</div>{{! end .chart-control }} -</script> - -<script type="text/template" class="template-visualization" id="template-visualization-dataControl"> - - <p class="help-text"> - Use the following controls to change the data used by the chart. - Use the 'Draw' button to render (or re-render) the chart with the current settings. - </p> - - {{! column selector containers }} - <div class="column-select"> - <label for="X-select">Data column for X: </label> - <select name="X" id="X-select"> - {{#each numericColumns}} - <option value="{{index}}">{{name}}</option> - {{/each}} - </select> - </div> - <div class="column-select"> - <label for="Y-select">Data column for Y: </label> - <select name="Y" id="Y-select"> - {{#each numericColumns}} - <option value="{{index}}">{{name}}</option> - {{/each}} - </select> - </div> - - {{! optional id column }} - <div id="include-id"> - <label for="include-id-checkbox">Include a third column as data point IDs?</label> - <input type="checkbox" name="include-id" id="include-id-checkbox" /> - <p class="help-text-small"> - These will be displayed (along with the x and y values) when you hover over - a data point. - </p> - </div> - <div class="column-select" style="display: none"> - <label for="ID-select">Data column for IDs: </label> - <select name="ID" id="ID-select"> - {{#each allColumns}} - <option value="{{index}}">{{name}}</option> - {{/each}} - </select> - </div> - - {{! if we're using generic column selection names ('column 1') - allow the user to use the first line }} - <div id="first-line-header" style="display: none;"> - <p>Possible headers: {{ possibleHeaders }} - </p> - <label for="first-line-header-checkbox">Use the above as column headers?</label> - <input type="checkbox" name="include-id" id="first-line-header-checkbox" - {{#if usePossibleHeaders }}checked="true"{{/if}}/> - <p class="help-text-small"> - It looks like Galaxy couldn't get proper column headers for this data. - Would you like to use the column headers above as column names to select columns? - </p> - </div> - - <input id="render-button" type="button" value="Draw" /> - <div class="clear"></div> -</script> - -<script type="text/template" class="template-visualization" id="template-visualization-chartControl"> - <p class="help-text"> - Use the following controls to how the chart is displayed. - The slide controls can be moved by the mouse or, if the 'handle' is in focus, your keyboard's arrow keys. - Move the focus between controls by using the tab or shift+tab keys on your keyboard. - Use the 'Draw' button to render (or re-render) the chart with the current settings. - </p> - - <div id="datapointSize" class="form-input numeric-slider-input"> - <label for="datapointSize">Size of data point: </label> - <div class="slider-output">{{datapointSize}}</div> - <div class="slider"></div> - <p class="form-help help-text-small"> - Size of the graphic representation of each data point - </p> - </div> - - <div id="animDuration" class="form-input checkbox-input"> - <label for="animate-chart">Animate chart transitions?: </label> - <input type="checkbox" id="animate-chart" - class="checkbox control"{{#if animDuration}} checked="true"{{/if}} /> - <p class="form-help help-text-small"> - Uncheck this to disable the animations used on the chart - </p> - </div> - - <div id="width" class="form-input numeric-slider-input"> - <label for="width">Chart width: </label> - <div class="slider-output">{{width}}</div> - <div class="slider"></div> - <p class="form-help help-text-small"> - (not including chart margins and axes) - </p> - </div> - - <div id="height" class="form-input numeric-slider-input"> - <label for="height">Chart height: </label> - <div class="slider-output">{{height}}</div> - <div class="slider"></div> - <p class="form-help help-text-small"> - (not including chart margins and axes) - </p> - </div> - - <div id="X-axis-label"class="text-input form-input"> - <label for="X-axis-label">Re-label the X axis: </label> - <input type="text" name="X-axis-label" id="X-axis-label" value="{{xLabel}}" /> - <p class="form-help help-text-small"></p> - </div> - - <div id="Y-axis-label" class="text-input form-input"> - <label for="Y-axis-label">Re-label the Y axis: </label> - <input type="text" name="Y-axis-label" id="Y-axis-label" value="{{yLabel}}" /> - <p class="form-help help-text-small"></p> - </div> - - <input id="render-button" type="button" value="Draw" /> -</script> - -<script type="text/template" class="template-visualization" id="template-visualization-statsDisplay"> - <p class="help-text">By column:</p> - <table id="chart-stats-table"> - <thead><th></th><th>X</th><th>Y</th></thead> - {{#each stats}} - <tr><td>{{name}}</td><td>{{xval}}</td><td>{{yval}}</td></tr> - </tr> - {{/each}} - </table> -</script> - -<script type="text/template" class="template-visualization" id="template-visualization-chartDisplay"> - <svg width="{{width}}" height="{{height}}"></svg> -</script> diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 config/plugins/visualizations/scatterplot/static/scatterplot-edit.js --- a/config/plugins/visualizations/scatterplot/static/scatterplot-edit.js +++ b/config/plugins/visualizations/scatterplot/static/scatterplot-edit.js @@ -1,1 +1,1 @@ -function scatterplot(a,b,c){function d(){var a={v:{},h:{}};return a.v.lines=p.selectAll("line.v-grid-line").data(m.x.ticks(q.x.fn.ticks()[0])),a.v.lines.enter().append("svg:line").classed("grid-line v-grid-line",!0),a.v.lines.attr("x1",m.x).attr("x2",m.x).attr("y1",0).attr("y2",b.height),a.v.lines.exit().remove(),a.h.lines=p.selectAll("line.h-grid-line").data(m.y.ticks(q.y.fn.ticks()[0])),a.h.lines.enter().append("svg:line").classed("grid-line h-grid-line",!0),a.h.lines.attr("x1",0).attr("x2",b.width).attr("y1",m.y).attr("y2",m.y),a.h.lines.exit().remove(),a}function e(){return t.attr("cx",function(a,b){return m.x(j(a,b))}).attr("cy",function(a,b){return m.y(k(a,b))}).style("display","block").filter(function(){var a=d3.select(this).attr("cx"),c=d3.select(this).attr("cy");return 0>a||a>b.width?!0:0>c||c>b.height?!0:!1}).style("display","none")}function f(){q.redraw(),e(),s=d(),$(".chart-info-box").remove(),$(o.node()).trigger("zoom.scatterplot",[])}function g(a,c,d){return c+=8,$(['<div class="chart-info-box" style="position: absolute">',b.idColumn?"<div>"+d[b.idColumn]+"</div>":"","<div>",j(d),"</div>","<div>",k(d),"</div>","</div>"].join("")).css({top:a,left:c,"z-index":2})}var h=function(a,b){return"translate("+a+","+b+")"},i=function(a,b,c){return"rotate("+a+","+b+","+c+")"},j=function(a){return a[b.xColumn]},k=function(a){return a[b.yColumn]},l={x:{extent:d3.extent(c,j)},y:{extent:d3.extent(c,k)}},m={x:d3.scale.linear().domain(l.x.extent).range([0,b.width]),y:d3.scale.linear().domain(l.y.extent).range([b.height,0])},n=d3.behavior.zoom().x(m.x).y(m.y).scaleExtent([1,10]),o=d3.select(a).attr("class","scatterplot").attr("width","100%").attr("height",b.height+(b.margin.top+b.margin.bottom)),p=o.append("g").attr("class","content").attr("transform",h(b.margin.left,b.margin.top)).call(n);p.append("rect").attr("class","zoom-rect").attr("width",b.width).attr("height",b.height).style("fill","transparent");var q={x:{},y:{}};q.x.fn=d3.svg.axis().orient("bottom").scale(m.x).ticks(b.x.ticks).tickFormat(d3.format("s")),q.y.fn=d3.svg.axis().orient("left").scale(m.y).ticks(b.y.ticks).tickFormat(d3.format("s")),q.x.g=p.append("g").attr("class","x axis").attr("transform",h(0,b.height)).call(q.x.fn),q.y.g=p.append("g").attr("class","y axis").call(q.y.fn);var r=4;q.x.label=o.append("text").attr("class","axis-label").text(b.x.label).attr("text-anchor","middle").attr("dominant-baseline","text-after-edge").attr("x",b.width/2+b.margin.left).attr("y",b.height+b.margin.bottom+b.margin.top-r),q.y.label=o.append("text").attr("class","axis-label").text(b.y.label).attr("text-anchor","middle").attr("dominant-baseline","text-before-edge").attr("x",r).attr("y",b.height/2).attr("transform",i(-90,r,b.height/2)),q.redraw=function(){o.select(".x.axis").call(q.x.fn),o.select(".y.axis").call(q.y.fn)};var s=d(),t=p.selectAll(".glyph").data(c).enter().append("svg:circle").classed("glyph",!0).attr("cx",function(a,b){return m.x(j(a,b))}).attr("cy",b.height).attr("r",0);t.transition().duration(b.animDuration).attr("cy",function(a,b){return m.y(k(a,b))}).attr("r",b.datapointSize),n.on("zoom",f),t.on("mouseover",function(a,c){var d=d3.select(this);d.style("fill","red").style("fill-opacity",1),p.append("line").attr("stroke","red").attr("stroke-width",1).attr("x1",d.attr("cx")-b.datapointSize).attr("y1",d.attr("cy")).attr("x2",0).attr("y2",d.attr("cy")).classed("hoverline",!0),d.attr("cy")<b.height&&p.append("line").attr("stroke","red").attr("stroke-width",1).attr("x1",d.attr("cx")).attr("y1",+d.attr("cy")+b.datapointSize).attr("x2",d.attr("cx")).attr("y2",b.height).classed("hoverline",!0);var e=this.getBoundingClientRect();$("body").append(g(e.top,e.right,a)),$(o.node()).trigger("mouseover-datapoint.scatterplot",[this,a,c])}),t.on("mouseout",function(){d3.select(this).style("fill","black").style("fill-opacity",.2),p.selectAll(".hoverline").remove(),$(".chart-info-box").remove()})}this.Templates=this.Templates||{},this.Templates.chartcontrol=Handlebars.template(function(a,b,c,d,e){this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var f,g="",h="function",i=this.escapeExpression;return g+='<p class="help-text">\n Use the following controls to how the chart is displayed.\n The slide controls can be moved by the mouse or, if the \'handle\' is in focus, your keyboard\'s arrow keys.\n Move the focus between controls by using the tab or shift+tab keys on your keyboard.\n Use the \'Draw\' button to render (or re-render) the chart with the current settings.\n</p>\n\n<div data-config-key="datapointSize" class="form-input numeric-slider-input">\n <label for="datapointSize">Size of data point: </label>\n <div class="slider-output">',(f=c.datapointSize)?f=f.call(b,{hash:{},data:e}):(f=b.datapointSize,f=typeof f===h?f.apply(b):f),g+=i(f)+'</div>\n <div class="slider"></div>\n <p class="form-help help-text-small">\n Size of the graphic representation of each data point\n </p>\n</div>\n\n<div data-config-key="width" class="form-input numeric-slider-input">\n <label for="width">Chart width: </label>\n <div class="slider-output">',(f=c.width)?f=f.call(b,{hash:{},data:e}):(f=b.width,f=typeof f===h?f.apply(b):f),g+=i(f)+'</div>\n <div class="slider"></div>\n <p class="form-help help-text-small">\n (not including chart margins and axes)\n </p>\n</div>\n\n<div data-config-key="height" class="form-input numeric-slider-input">\n <label for="height">Chart height: </label>\n <div class="slider-output">',(f=c.height)?f=f.call(b,{hash:{},data:e}):(f=b.height,f=typeof f===h?f.apply(b):f),g+=i(f)+'</div>\n <div class="slider"></div>\n <p class="form-help help-text-small">\n (not including chart margins and axes)\n </p>\n</div>\n\n<div data-config-key="X-axis-label"class="text-input form-input">\n <label for="X-axis-label">Re-label the X axis: </label>\n <input type="text" name="X-axis-label" id="X-axis-label" value="'+i((f=b.x,f=null==f||f===!1?f:f.label,typeof f===h?f.apply(b):f))+'" />\n <p class="form-help help-text-small"></p>\n</div>\n\n<div data-config-key="Y-axis-label" class="text-input form-input">\n <label for="Y-axis-label">Re-label the Y axis: </label>\n <input type="text" name="Y-axis-label" id="Y-axis-label" value="'+i((f=b.y,f=null==f||f===!1?f:f.label,typeof f===h?f.apply(b):f))+'" />\n <p class="form-help help-text-small"></p>\n</div>\n\n<button class="render-button btn btn-primary active">Draw</button>\n'}),this.Templates.datacontrol=Handlebars.template(function(a,b,c,d,e){function f(a,b){var d,e="";return e+='\n <option value="',(d=c.index)?d=d.call(a,{hash:{},data:b}):(d=a.index,d=typeof d===j?d.apply(a):d),e+=k(d)+'">',(d=c.name)?d=d.call(a,{hash:{},data:b}):(d=a.name,d=typeof d===j?d.apply(a):d),e+=k(d)+"</option>\n "}function g(){return'checked="true"'}this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var h,i="",j="function",k=this.escapeExpression,l=this;return i+='<p class="help-text">\n Use the following controls to change the data used by the chart.\n Use the \'Draw\' button to render (or re-render) the chart with the current settings.\n</p>\n\n\n<div class="column-select">\n <label>Data column for X: </label>\n <select name="xColumn">\n ',h=c.each.call(b,b.numericColumns,{hash:{},inverse:l.noop,fn:l.program(1,f,e),data:e}),(h||0===h)&&(i+=h),i+='\n </select>\n</div>\n<div class="column-select">\n <label>Data column for Y: </label>\n <select name="yColumn">\n ',h=c.each.call(b,b.numericColumns,{hash:{},inverse:l.noop,fn:l.program(1,f,e),data:e}),(h||0===h)&&(i+=h),i+='\n </select>\n</div>\n\n\n<div id="include-id">\n <label for="include-id-checkbox">Include a third column as data point IDs?</label>\n <input type="checkbox" name="include-id" id="include-id-checkbox" />\n <p class="help-text-small">\n These will be displayed (along with the x and y values) when you hover over\n a data point.\n </p>\n</div>\n<div class="column-select" style="display: none">\n <label for="ID-select">Data column for IDs: </label>\n <select name="idColumn">\n ',h=c.each.call(b,b.allColumns,{hash:{},inverse:l.noop,fn:l.program(1,f,e),data:e}),(h||0===h)&&(i+=h),i+='\n </select>\n</div>\n\n\n<div id="first-line-header" style="display: none;">\n <p>Possible headers: ',(h=c.possibleHeaders)?h=h.call(b,{hash:{},data:e}):(h=b.possibleHeaders,h=typeof h===j?h.apply(b):h),i+=k(h)+'\n </p>\n <label for="first-line-header-checkbox">Use the above as column headers?</label>\n <input type="checkbox" name="include-id" id="first-line-header-checkbox"\n ',h=c["if"].call(b,b.usePossibleHeaders,{hash:{},inverse:l.noop,fn:l.program(3,g,e),data:e}),(h||0===h)&&(i+=h),i+='/>\n <p class="help-text-small">\n It looks like Galaxy couldn\'t get proper column headers for this data.\n Would you like to use the column headers above as column names to select columns?\n </p>\n</div>\n\n<button class="render-button btn btn-primary active">Draw</button>\n'}),this.Templates.editor=Handlebars.template(function(a,b,c,d,e){this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var f="";return f+='<div class="scatterplot-editor tabbable tabs-left">\n \n <ul class="nav nav-tabs">\n \n <li class="active">\n <a title="Use this tab to change which data are used"\n href="#data-control" data-toggle="tab">Data Controls</a>\n </li>\n <li>\n <a title="Use this tab to change how the chart is drawn"\n href="#chart-control" data-toggle="tab" >Chart Controls</a>\n </li>\n \n <li class="disabled">\n <a title="This tab will display the chart"\n href="#chart-display" data-toggle="tab">Chart</a>\n </li>\n </ul>\n\n \n <div class="tab-content">\n \n <div id="data-control" class="scatterplot-config-control tab-pane active">\n \n </div>\n \n \n <div id="chart-control" class="scatterplot-config-control tab-pane">\n \n </div>\n\n \n <div id="chart-display" class="scatterplot-display tab-pane"></div>\n\n </div>\n</div>\n'});var ScatterplotConfigEditor=BaseView.extend(LoggableMixin).extend({className:"scatterplot-control-form",initialize:function(a){if(!a||!a.config||!a.config.dataset)throw new Error("ScatterplotView requires a configuration and dataset");this.dataset=a.config.dataset,this.plotView=new ScatterplotView({config:a.config})},render:function(){return this.$el.append(ScatterplotConfigEditor.templates.mainLayout({})),this.$el.find("#data-control").append(this._render_dataControl()),this._render_chartControls(this.$el.find("#chart-control")),this._render_chartDisplay(),this.$el.find("[title]").tooltip(),this},_render_dataControl:function(){var a=this.dataset,b=_.map(a.metadata_column_types,function(b,c){var d={index:c,type:b,name:"column "+(c+1)};return a.metadata_column_names&&a.metadata_column_names[c]&&(d.name=a.metadata_column_names[c]),d}),c=_.filter(b,function(a){return"int"===a.type||"float"===a.type});2>c&&(c=b);var d=this.$el.find(".tab-pane#data-control");return d.html(ScatterplotConfigEditor.templates.dataControl({allColumns:b,numericColumns:c})),d.find('[name="xColumn"]').val(this.plotView.config.xColumn||c[0].index),d.find('[name="yColumn"]').val(this.plotView.config.yColumn||c[1].index),void 0!==this.plotView.config.idColumn&&(d.find("#include-id-checkbox").prop("checked",!0).trigger("change"),d.find('select[name="idColumn"]').val(this.plotView.config.idColumn)),d},_render_chartControls:function(a){function b(){var a=$(this);a.siblings(".slider-output").text(a.slider("value"))}a.html(ScatterplotConfigEditor.templates.chartControl(this.plotView.config));var c=this,d={datapointSize:{min:2,max:10,step:1},width:{min:200,max:800,step:20},height:{min:200,max:800,step:20}};return a.find(".numeric-slider-input").each(function(){var a=$(this),e=a.attr("data-config-key"),f=_.extend(d[e],{value:c.plotView.config[e],change:b,slide:b});a.find(".slider").slider(f)}),this.dataset.metadata_column_names,a},_render_chartDisplay:function(){var a=this.$el.find(".tab-pane#chart-display");return this.plotView.setElement(a),this.plotView.render(),a},events:{"change #include-id-checkbox":"toggleThirdColumnSelector","click #data-control .render-button":"renderChart","click #chart-control .render-button":"renderChart"},toggleThirdColumnSelector:function(){this.$el.find('select[name="idColumn"]').parent().toggle()},renderChart:function(){this.$el.find(".nav li.disabled").removeClass("disabled"),this.updateConfigWithDataSettings(),this.updateConfigWithChartSettings(),this.$el.find("ul.nav").find('a[href="#chart-display"]').tab("show"),this.plotView.fetchData()},updateConfigWithDataSettings:function(){var a=this.$el.find("#data-control"),b={xColumn:a.find('[name="xColumn"]').val(),yColumn:a.find('[name="yColumn"]').val()};return a.find("#include-id-checkbox").prop("checked")&&(b.idColumn=a.find('[name="idColumn"]').val()),_.extend(this.plotView.config,b)},updateConfigWithChartSettings:function(){var a=this.plotView,b=this.$el.find("#chart-control");return["datapointSize","width","height"].forEach(function(c){a.config[c]=b.find('.numeric-slider-input[data-config-key="'+c+'"]').find(".slider").slider("value")}),a.config.x.label=b.find('input[name="X-axis-label"]').val(),a.config.y.label=b.find('input[name="Y-axis-label"]').val(),a.config},toString:function(){return"ScatterplotConfigEditor("+(this.dataset?this.dataset.id:"")+")"}});ScatterplotConfigEditor.templates={mainLayout:Templates.editor,dataControl:Templates.datacontrol,chartControl:Templates.chartcontrol};var ScatterplotView=Backbone.View.extend({defaults:{dataset:{},metadata:{dataLines:void 0},ajaxFn:null,pagination:{currPage:0,perPage:3e3},width:400,height:400,margin:{top:16,right:16,bottom:40,left:54},x:{ticks:10,label:"X"},y:{ticks:10,label:"Y"},datapointSize:4,animDuration:500},initialize:function(a){this.config=_.extend(_.clone(this.defaults),a.config||{})},updateConfig:function(a){this.config=this.config||{},_.extend(this.config,a)},fetchData:function(){this.showLoadingIndicator("getting data");var a=this;return xhr=jQuery.getJSON("/api/datasets/"+this.config.dataset.id,{data_type:"raw_data",provider:"dataset-column",limit:this.config.pagination.perPage,offset:this.config.pagination.currPage*this.config.pagination.perPage}),xhr.done(function(b){a.renderData(b.data)}),xhr.fail(function(a,b,c){alert("Error loading data:\n"+a.responseText),console.error(a,b,c)}),xhr.always(function(){a.hideLoadingIndicator()}),xhr},render:function(a){return this.$el.addClass("scatterplot-display").html(['<div class="controls clear"></div>','<div class="loading-indicator">','<span class="fa fa-spinner fa-spin"></span>','<span class="loading-indicator-message"></span>',"</div>","<svg/>",'<div class="stats-display"></div>'].join("")),this.$el.children().hide(),a&&this.renderData(a),this},showLoadingIndicator:function(a,b){a=a||"",b=b||"fast";var c=this.$el.find(".loading-indicator");a&&c.find(".loading-indicator-message").text(a),c.is(":visible")||(this.toggleStats(!1),c.css({left:this.config.width/2,top:this.config.height/2}).show())},hideLoadingIndicator:function(a){a=a||"fast",this.$el.find(".loading-indicator").hide()},renderData:function(a){this.$el.find(".controls").empty().append(this.renderControls(a)).show(),this.renderPlot(a),this.getStats(a)},renderControls:function(a){var b=this,c=$('<div class="left"></div>'),d=$('<div class="right"></div>');return c.append([this.renderPrevNext(a),this.renderPagination(a)]),d.append([this.renderLineInfo(a),$("<button>Stats</button>").addClass("stats-toggle-btn").click(function(){b.toggleStats()}),$("<button>Redraw</button>").addClass("rerender-btn").click(function(){b.renderPlot(a)})]),[c,d]},renderLineInfo:function(a){var b=this.config.dataset.metadata_data_lines||"an unknown number of",c=this.config.pagination.currPage*this.config.pagination.perPage,d=c+a.length;return $("<p/>").addClass("scatterplot-data-info").text(["Displaying lines",c+1,"to",d,"of",b,"lines"].join(" "))},renderPrevNext:function(a){function b(a){return $(['<li><a href="javascript:void(0);">',a,"</a></li>"].join(""))}if(!a||0===this.config.pagination.currPage&&a.length<this.config.pagination.perPage)return null;var c=this,d=this.config.dataset.metadata_data_lines,e=d?Math.ceil(d/this.config.pagination.perPage):void 0,f=b("Prev").click(function(){c.config.pagination.currPage>0&&(c.config.pagination.currPage-=1,c.fetchData())}),g=b("Next").click(function(){(!e||c.config.pagination.currPage<e-1)&&(c.config.pagination.currPage+=1,c.fetchData())}),h=$("<ul/>").addClass("pagination data-prev-next").append([f,g]);return 0===c.config.pagination.currPage&&f.addClass("disabled"),e&&c.config.pagination.currPage===e-1&&g.addClass("disabled"),h},renderPagination:function(a){function b(a){return $(['<li><a href="javascript:void(0);">',a,"</a></li>"].join(""))}function c(){d.config.pagination.currPage=$(this).data("page"),d.fetchData()}if(!a||0===this.config.pagination.currPage&&a.length<this.config.pagination.perPage)return null;for(var d=this,e=this.config.dataset.metadata_data_lines,f=e?Math.ceil(e/this.config.pagination.perPage):void 0,g=$("<ul/>").addClass("pagination data-pages"),h=0;f>h;h+=1){var i=b(h+1).attr("data-page",h).click(c);h===this.config.pagination.currPage&&i.addClass("active"),g.append(i)}return g},renderPlot:function(a){this.toggleStats(!1);var b=this.$el.find("svg");b.off().empty().show(),scatterplot(b.get(0),this.config,a)},getStats:function(a){var b=this;meanWorker=new Worker("/plugins/visualizations/scatterplot/static/worker-stats.js"),meanWorker.postMessage({data:a,keys:[this.config.xColumn,this.config.yColumn]}),meanWorker.onerror=function(){meanWorker.terminate()},meanWorker.onmessage=function(a){b.renderStats(a.data)}},renderStats:function(a){var b=this.$el.find(".stats-display"),c=this.config.x.label,d=this.config.y.label,e=$("<table/>").addClass("table").append(["<thead><th></th><th>",c,"</th><th>",d,"</th></thead>"].join("")).append(_.map(a,function(a,b){return $(["<tr><td>",b,"</td><td>",a[0],"</td><td>",a[1],"</td></tr>"].join(""))}));b.empty().append(e)},toggleStats:function(a){var b=this.$el.find(".stats-display");a=void 0===a?b.is(":hidden"):a,a?(this.$el.find("svg").hide(),b.show(),this.$el.find(".controls .stats-toggle-btn").text("Plot")):(b.hide(),this.$el.find("svg").show(),this.$el.find(".controls .stats-toggle-btn").text("Stats"))},toString:function(){return"ScatterplotView()"}}); \ No newline at end of file +function scatterplot(a,b,c){function d(){var a={v:{},h:{}};return a.v.lines=p.selectAll("line.v-grid-line").data(m.x.ticks(q.x.fn.ticks()[0])),a.v.lines.enter().append("svg:line").classed("grid-line v-grid-line",!0),a.v.lines.attr("x1",m.x).attr("x2",m.x).attr("y1",0).attr("y2",b.height),a.v.lines.exit().remove(),a.h.lines=p.selectAll("line.h-grid-line").data(m.y.ticks(q.y.fn.ticks()[0])),a.h.lines.enter().append("svg:line").classed("grid-line h-grid-line",!0),a.h.lines.attr("x1",0).attr("x2",b.width).attr("y1",m.y).attr("y2",m.y),a.h.lines.exit().remove(),a}function e(){return t.attr("cx",function(a,b){return m.x(j(a,b))}).attr("cy",function(a,b){return m.y(k(a,b))}).style("display","block").filter(function(){var a=d3.select(this).attr("cx"),c=d3.select(this).attr("cy");return 0>a||a>b.width?!0:0>c||c>b.height?!0:!1}).style("display","none")}function f(){q.redraw(),e(),s=d(),$(".chart-info-box").remove(),$(o.node()).trigger("zoom.scatterplot",[])}function g(a,c,d){return c+=8,$(['<div class="chart-info-box" style="position: absolute">',b.idColumn?"<div>"+d[b.idColumn]+"</div>":"","<div>",j(d),"</div>","<div>",k(d),"</div>","</div>"].join("")).css({top:a,left:c,"z-index":2})}var h=function(a,b){return"translate("+a+","+b+")"},i=function(a,b,c){return"rotate("+a+","+b+","+c+")"},j=function(a){return a[b.xColumn]},k=function(a){return a[b.yColumn]},l={x:{extent:d3.extent(c,j)},y:{extent:d3.extent(c,k)}},m={x:d3.scale.linear().domain(l.x.extent).range([0,b.width]),y:d3.scale.linear().domain(l.y.extent).range([b.height,0])},n=d3.behavior.zoom().x(m.x).y(m.y).scaleExtent([1,10]),o=d3.select(a).attr("class","scatterplot").attr("width","100%").attr("height",b.height+(b.margin.top+b.margin.bottom)),p=o.append("g").attr("class","content").attr("transform",h(b.margin.left,b.margin.top)).call(n);p.append("rect").attr("class","zoom-rect").attr("width",b.width).attr("height",b.height).style("fill","transparent");var q={x:{},y:{}};q.x.fn=d3.svg.axis().orient("bottom").scale(m.x).ticks(b.x.ticks).tickFormat(d3.format("s")),q.y.fn=d3.svg.axis().orient("left").scale(m.y).ticks(b.y.ticks).tickFormat(d3.format("s")),q.x.g=p.append("g").attr("class","x axis").attr("transform",h(0,b.height)).call(q.x.fn),q.y.g=p.append("g").attr("class","y axis").call(q.y.fn);var r=4;q.x.label=o.append("text").attr("class","axis-label").text(b.x.label).attr("text-anchor","middle").attr("dominant-baseline","text-after-edge").attr("x",b.width/2+b.margin.left).attr("y",b.height+b.margin.bottom+b.margin.top-r),q.y.label=o.append("text").attr("class","axis-label").text(b.y.label).attr("text-anchor","middle").attr("dominant-baseline","text-before-edge").attr("x",r).attr("y",b.height/2).attr("transform",i(-90,r,b.height/2)),q.redraw=function(){o.select(".x.axis").call(q.x.fn),o.select(".y.axis").call(q.y.fn)};var s=d(),t=p.selectAll(".glyph").data(c).enter().append("svg:circle").classed("glyph",!0).attr("cx",function(a,b){return m.x(j(a,b))}).attr("cy",b.height).attr("r",0);t.transition().duration(b.animDuration).attr("cy",function(a,b){return m.y(k(a,b))}).attr("r",b.datapointSize),n.on("zoom",f),t.on("mouseover",function(a,c){var d=d3.select(this);d.style("fill","red").style("fill-opacity",1),p.append("line").attr("stroke","red").attr("stroke-width",1).attr("x1",d.attr("cx")-b.datapointSize).attr("y1",d.attr("cy")).attr("x2",0).attr("y2",d.attr("cy")).classed("hoverline",!0),d.attr("cy")<b.height&&p.append("line").attr("stroke","red").attr("stroke-width",1).attr("x1",d.attr("cx")).attr("y1",+d.attr("cy")+b.datapointSize).attr("x2",d.attr("cx")).attr("y2",b.height).classed("hoverline",!0);var e=this.getBoundingClientRect();$("body").append(g(e.top,e.right,a)),$(o.node()).trigger("mouseover-datapoint.scatterplot",[this,a,c])}),t.on("mouseout",function(){d3.select(this).style("fill","black").style("fill-opacity",.2),p.selectAll(".hoverline").remove(),$(".chart-info-box").remove()})}this.Templates=this.Templates||{},this.Templates.chartcontrol=Handlebars.template(function(a,b,c,d,e){this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var f,g="",h="function",i=this.escapeExpression;return g+='<p class="help-text">\n Use the following controls to how the chart is displayed.\n The slide controls can be moved by the mouse or, if the \'handle\' is in focus, your keyboard\'s arrow keys.\n Move the focus between controls by using the tab or shift+tab keys on your keyboard.\n Use the \'Draw\' button to render (or re-render) the chart with the current settings.\n</p>\n\n<div data-config-key="datapointSize" class="form-input numeric-slider-input">\n <label for="datapointSize">Size of data point: </label>\n <div class="slider-output">',(f=c.datapointSize)?f=f.call(b,{hash:{},data:e}):(f=b.datapointSize,f=typeof f===h?f.apply(b):f),g+=i(f)+'</div>\n <div class="slider"></div>\n <p class="form-help help-text-small">\n Size of the graphic representation of each data point\n </p>\n</div>\n\n<div data-config-key="width" class="form-input numeric-slider-input">\n <label for="width">Chart width: </label>\n <div class="slider-output">',(f=c.width)?f=f.call(b,{hash:{},data:e}):(f=b.width,f=typeof f===h?f.apply(b):f),g+=i(f)+'</div>\n <div class="slider"></div>\n <p class="form-help help-text-small">\n (not including chart margins and axes)\n </p>\n</div>\n\n<div data-config-key="height" class="form-input numeric-slider-input">\n <label for="height">Chart height: </label>\n <div class="slider-output">',(f=c.height)?f=f.call(b,{hash:{},data:e}):(f=b.height,f=typeof f===h?f.apply(b):f),g+=i(f)+'</div>\n <div class="slider"></div>\n <p class="form-help help-text-small">\n (not including chart margins and axes)\n </p>\n</div>\n\n<div data-config-key="X-axis-label"class="text-input form-input">\n <label for="X-axis-label">Re-label the X axis: </label>\n <input type="text" name="X-axis-label" id="X-axis-label" value="'+i((f=b.x,f=null==f||f===!1?f:f.label,typeof f===h?f.apply(b):f))+'" />\n <p class="form-help help-text-small"></p>\n</div>\n\n<div data-config-key="Y-axis-label" class="text-input form-input">\n <label for="Y-axis-label">Re-label the Y axis: </label>\n <input type="text" name="Y-axis-label" id="Y-axis-label" value="'+i((f=b.y,f=null==f||f===!1?f:f.label,typeof f===h?f.apply(b):f))+'" />\n <p class="form-help help-text-small"></p>\n</div>\n\n<button class="render-button btn btn-primary active">Draw</button>\n'}),this.Templates.datacontrol=Handlebars.template(function(a,b,c,d,e){function f(a,b){var d,e="";return e+='\n <option value="',(d=c.index)?d=d.call(a,{hash:{},data:b}):(d=a.index,d=typeof d===j?d.apply(a):d),e+=k(d)+'">',(d=c.name)?d=d.call(a,{hash:{},data:b}):(d=a.name,d=typeof d===j?d.apply(a):d),e+=k(d)+"</option>\n "}function g(){return'checked="true"'}this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var h,i="",j="function",k=this.escapeExpression,l=this;return i+='<p class="help-text">\n Use the following controls to change the data used by the chart.\n Use the \'Draw\' button to render (or re-render) the chart with the current settings.\n</p>\n\n\n<div class="column-select">\n <label>Data column for X: </label>\n <select name="xColumn">\n ',h=c.each.call(b,b.numericColumns,{hash:{},inverse:l.noop,fn:l.program(1,f,e),data:e}),(h||0===h)&&(i+=h),i+='\n </select>\n</div>\n<div class="column-select">\n <label>Data column for Y: </label>\n <select name="yColumn">\n ',h=c.each.call(b,b.numericColumns,{hash:{},inverse:l.noop,fn:l.program(1,f,e),data:e}),(h||0===h)&&(i+=h),i+='\n </select>\n</div>\n\n\n<div id="include-id">\n <label for="include-id-checkbox">Include a third column as data point IDs?</label>\n <input type="checkbox" name="include-id" id="include-id-checkbox" />\n <p class="help-text-small">\n These will be displayed (along with the x and y values) when you hover over\n a data point.\n </p>\n</div>\n<div class="column-select" style="display: none">\n <label for="ID-select">Data column for IDs: </label>\n <select name="idColumn">\n ',h=c.each.call(b,b.allColumns,{hash:{},inverse:l.noop,fn:l.program(1,f,e),data:e}),(h||0===h)&&(i+=h),i+='\n </select>\n</div>\n\n\n<div id="first-line-header" style="display: none;">\n <p>Possible headers: ',(h=c.possibleHeaders)?h=h.call(b,{hash:{},data:e}):(h=b.possibleHeaders,h=typeof h===j?h.apply(b):h),i+=k(h)+'\n </p>\n <label for="first-line-header-checkbox">Use the above as column headers?</label>\n <input type="checkbox" name="include-id" id="first-line-header-checkbox"\n ',h=c["if"].call(b,b.usePossibleHeaders,{hash:{},inverse:l.noop,fn:l.program(3,g,e),data:e}),(h||0===h)&&(i+=h),i+='/>\n <p class="help-text-small">\n It looks like Galaxy couldn\'t get proper column headers for this data.\n Would you like to use the column headers above as column names to select columns?\n </p>\n</div>\n\n<button class="render-button btn btn-primary active">Draw</button>\n'}),this.Templates.editor=Handlebars.template(function(a,b,c,d,e){this.compilerInfo=[4,">= 1.0.0"],c=this.merge(c,a.helpers),e=e||{};var f="";return f+='<div class="scatterplot-editor tabbable tabs-left">\n \n <ul class="nav nav-tabs">\n \n <li class="active">\n <a title="Use this tab to change which data are used"\n href="#data-control" data-toggle="tab">Data Controls</a>\n </li>\n <li>\n <a title="Use this tab to change how the chart is drawn"\n href="#chart-control" data-toggle="tab" >Chart Controls</a>\n </li>\n \n <li class="disabled">\n <a title="This tab will display the chart"\n href="#chart-display" data-toggle="tab">Chart</a>\n </li>\n </ul>\n\n \n <div class="tab-content">\n \n <div id="data-control" class="scatterplot-config-control tab-pane active">\n \n </div>\n \n \n <div id="chart-control" class="scatterplot-config-control tab-pane">\n \n </div>\n\n \n <div id="chart-display" class="scatterplot-display tab-pane"></div>\n\n </div>\n</div>\n'});var ScatterplotConfigEditor=BaseView.extend(LoggableMixin).extend({className:"scatterplot-control-form",initialize:function(a){if(!a||!a.config||!a.dataset)throw new Error("ScatterplotView requires a configuration and dataset");this.dataset=a.dataset,this.plotView=new ScatterplotView({dataset:a.dataset,config:a.config})},render:function(){return this.$el.append(ScatterplotConfigEditor.templates.mainLayout({})),this.$el.find("#data-control").append(this._render_dataControl()),this._render_chartControls(this.$el.find("#chart-control")),this._render_chartDisplay(),this.$el.find("[title]").tooltip(),this},_render_dataControl:function(){var a=this.dataset,b=_.map(a.metadata_column_types,function(b,c){var d={index:c,type:b,name:"column "+(c+1)};return a.metadata_column_names&&a.metadata_column_names[c]&&(d.name=a.metadata_column_names[c]),d}),c=_.filter(b,function(a){return"int"===a.type||"float"===a.type});2>c&&(c=b);var d=this.$el.find(".tab-pane#data-control");return d.html(ScatterplotConfigEditor.templates.dataControl({allColumns:b,numericColumns:c})),d.find('[name="xColumn"]').val(this.plotView.config.xColumn||c[0].index),d.find('[name="yColumn"]').val(this.plotView.config.yColumn||c[1].index),void 0!==this.plotView.config.idColumn&&(d.find("#include-id-checkbox").prop("checked",!0).trigger("change"),d.find('select[name="idColumn"]').val(this.plotView.config.idColumn)),d},_render_chartControls:function(a){function b(){var a=$(this);a.siblings(".slider-output").text(a.slider("value"))}a.html(ScatterplotConfigEditor.templates.chartControl(this.plotView.config));var c=this,d={datapointSize:{min:2,max:10,step:1},width:{min:200,max:800,step:20},height:{min:200,max:800,step:20}};return a.find(".numeric-slider-input").each(function(){var a=$(this),e=a.attr("data-config-key"),f=_.extend(d[e],{value:c.plotView.config[e],change:b,slide:b});a.find(".slider").slider(f)}),this.dataset.metadata_column_names,a},_render_chartDisplay:function(){var a=this.$el.find(".tab-pane#chart-display");return this.plotView.setElement(a),this.plotView.render(),a},events:{"change #include-id-checkbox":"toggleThirdColumnSelector","click #data-control .render-button":"renderChart","click #chart-control .render-button":"renderChart"},toggleThirdColumnSelector:function(){this.$el.find('select[name="idColumn"]').parent().toggle()},renderChart:function(){this.$el.find(".nav li.disabled").removeClass("disabled"),this.updateConfigWithDataSettings(),this.updateConfigWithChartSettings(),this.$el.find("ul.nav").find('a[href="#chart-display"]').tab("show"),this.plotView.fetchData()},updateConfigWithDataSettings:function(){var a=this.$el.find("#data-control"),b={xColumn:Number(a.find('[name="xColumn"]').val()),yColumn:Number(a.find('[name="yColumn"]').val())};return a.find("#include-id-checkbox").prop("checked")&&(b.idColumn=a.find('[name="idColumn"]').val()),_.extend(this.plotView.config,b)},updateConfigWithChartSettings:function(){var a=this.plotView,b=this.$el.find("#chart-control");return["datapointSize","width","height"].forEach(function(c){a.config[c]=b.find('.numeric-slider-input[data-config-key="'+c+'"]').find(".slider").slider("value")}),a.config.x.label=b.find('input[name="X-axis-label"]').val(),a.config.y.label=b.find('input[name="Y-axis-label"]').val(),a.config},toString:function(){return"ScatterplotConfigEditor("+(this.dataset?this.dataset.id:"")+")"}});ScatterplotConfigEditor.templates={mainLayout:Templates.editor,dataControl:Templates.datacontrol,chartControl:Templates.chartcontrol};var ScatterplotView=Backbone.View.extend({defaults:{metadata:{dataLines:void 0},pagination:{currPage:0,perPage:3e3},width:400,height:400,margin:{top:16,right:16,bottom:40,left:54},x:{ticks:10,label:"X"},y:{ticks:10,label:"Y"},datapointSize:4,animDuration:500},initialize:function(a){this.config=_.extend(_.clone(this.defaults),a.config||{}),this.dataset=a.dataset},updateConfig:function(a){this.config=this.config||{},_.extend(this.config,a)},fetchData:function(){this.showLoadingIndicator("getting data");var a=this;return xhr=jQuery.getJSON("/api/datasets/"+this.dataset.id,{data_type:"raw_data",provider:"dataset-column",limit:this.config.pagination.perPage,offset:this.config.pagination.currPage*this.config.pagination.perPage}),xhr.done(function(b){a.renderData(b.data)}),xhr.fail(function(a,b,c){alert("Error loading data:\n"+a.responseText),console.error(a,b,c)}),xhr.always(function(){a.hideLoadingIndicator()}),xhr},render:function(a){return this.$el.addClass("scatterplot-display").html(['<div class="controls clear"></div>','<div class="loading-indicator">','<span class="fa fa-spinner fa-spin"></span>','<span class="loading-indicator-message"></span>',"</div>","<svg/>",'<div class="stats-display"></div>'].join("")),this.$el.children().hide(),a&&this.renderData(a),this},showLoadingIndicator:function(a,b){a=a||"",b=b||"fast";var c=this.$el.find(".loading-indicator");a&&c.find(".loading-indicator-message").text(a),c.is(":visible")||(this.toggleStats(!1),c.css({left:this.config.width/2,top:this.config.height/2}).show())},hideLoadingIndicator:function(a){a=a||"fast",this.$el.find(".loading-indicator").hide()},renderData:function(a){this.$el.find(".controls").empty().append(this.renderControls(a)).show(),this.renderPlot(a),this.getStats(a)},renderControls:function(a){var b=this,c=$('<div class="left"></div>'),d=$('<div class="right"></div>');return c.append([this.renderPrevNext(a),this.renderPagination(a)]),d.append([this.renderLineInfo(a),$("<button>Stats</button>").addClass("stats-toggle-btn").click(function(){b.toggleStats()}),$("<button>Redraw</button>").addClass("rerender-btn").click(function(){b.renderPlot(a)})]),[c,d]},renderLineInfo:function(a){var b=this.dataset.metadata_data_lines||"an unknown number of",c=this.config.pagination.currPage*this.config.pagination.perPage,d=c+a.length;return $("<p/>").addClass("scatterplot-data-info").text(["Displaying lines",c+1,"to",d,"of",b,"lines"].join(" "))},renderPrevNext:function(a){function b(a){return $(['<li><a href="javascript:void(0);">',a,"</a></li>"].join(""))}if(!a||0===this.config.pagination.currPage&&a.length<this.config.pagination.perPage)return null;var c=this,d=this.dataset.metadata_data_lines,e=d?Math.ceil(d/this.config.pagination.perPage):void 0,f=b("Prev").click(function(){c.config.pagination.currPage>0&&(c.config.pagination.currPage-=1,c.fetchData())}),g=b("Next").click(function(){(!e||c.config.pagination.currPage<e-1)&&(c.config.pagination.currPage+=1,c.fetchData())}),h=$("<ul/>").addClass("pagination data-prev-next").append([f,g]);return 0===c.config.pagination.currPage&&f.addClass("disabled"),e&&c.config.pagination.currPage===e-1&&g.addClass("disabled"),h},renderPagination:function(a){function b(a){return $(['<li><a href="javascript:void(0);">',a,"</a></li>"].join(""))}function c(){d.config.pagination.currPage=$(this).data("page"),d.fetchData()}if(!a||0===this.config.pagination.currPage&&a.length<this.config.pagination.perPage)return null;for(var d=this,e=this.dataset.metadata_data_lines,f=e?Math.ceil(e/this.config.pagination.perPage):void 0,g=$("<ul/>").addClass("pagination data-pages"),h=0;f>h;h+=1){var i=b(h+1).attr("data-page",h).click(c);h===this.config.pagination.currPage&&i.addClass("active"),g.append(i)}return g},renderPlot:function(a){this.toggleStats(!1);var b=this.$el.find("svg");b.off().empty().show(),scatterplot(b.get(0),this.config,a)},getStats:function(a){var b=this;meanWorker=new Worker("/plugins/visualizations/scatterplot/static/worker-stats.js"),meanWorker.postMessage({data:a,keys:[this.config.xColumn,this.config.yColumn]}),meanWorker.onerror=function(){meanWorker.terminate()},meanWorker.onmessage=function(a){b.renderStats(a.data)}},renderStats:function(a){var b=this.$el.find(".stats-display"),c=this.config.x.label,d=this.config.y.label,e=$("<table/>").addClass("table").append(["<thead><th></th><th>",c,"</th><th>",d,"</th></thead>"].join("")).append(_.map(a,function(a,b){return $(["<tr><td>",b,"</td><td>",a[0],"</td><td>",a[1],"</td></tr>"].join(""))}));b.empty().append(e)},toggleStats:function(a){var b=this.$el.find(".stats-display");a=void 0===a?b.is(":hidden"):a,a?(this.$el.find("svg").hide(),b.show(),this.$el.find(".controls .stats-toggle-btn").text("Plot")):(b.hide(),this.$el.find("svg").show(),this.$el.find(".controls .stats-toggle-btn").text("Stats"))},toString:function(){return"ScatterplotView()"}}); \ No newline at end of file diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 config/plugins/visualizations/scatterplot/templates/scatterplot.mako --- a/config/plugins/visualizations/scatterplot/templates/scatterplot.mako +++ b/config/plugins/visualizations/scatterplot/templates/scatterplot.mako @@ -43,22 +43,18 @@ data = None ##data = list( hda.datatype.dataset_column_dataprovider( hda, limit=10000 ) ) %> - var hda = ${h.to_json_string( trans.security.encode_dict_ids( hda.to_dict() ) )}, - data = ${h.to_json_string( data )}, - querySettings = ${h.to_json_string( query_args )}, - config = _.extend( querySettings, { - containerSelector : '#chart', - dataset : hda, - }); - //console.debug( querySettings ); + var hda = ${h.to_json_string( trans.security.encode_dict_ids( hda.to_dict() ) )}; var editor = new ScatterplotConfigEditor({ el : $( '.scatterplot-editor' ).attr( 'id', 'scatterplot-editor-hda-' + hda.id ), - config : config + config : ${h.to_json_string( query_args )}, + dataset : ${h.to_json_string( trans.security.encode_dict_ids( hda.to_dict() ) )} }).render(); + window.editor = editor; // uncomment to auto render for development //$( '.render-button:visible' ).click(); }); + </script> %endif diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 lib/galaxy/datatypes/interval.py --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -389,7 +389,7 @@ MetadataElement( name="endCol", default=3, desc="End column", param=metadata.ColumnParameter ) MetadataElement( name="strandCol", desc="Strand column (click box & select)", param=metadata.ColumnParameter, optional=True, no_value=0 ) MetadataElement( name="columns", default=3, desc="Number of columns", readonly=True, visible=False ) - MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[4], param=metadata.ColumnParameter, multiple=True ) + MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[4], param=metadata.ColumnParameter, optional=True, multiple=True ) ###do we need to repeat these? they are the same as should be inherited from interval type def set_meta( self, dataset, overwrite = True, **kwd ): diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 lib/galaxy/datatypes/tabular.py --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -652,7 +652,7 @@ MetadataElement( name="columns", default=10, desc="Number of columns", readonly=True, visible=False ) MetadataElement( name="column_types", default=['str','int','str','str','str','int','str','list','str','str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False ) - MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[5], param=metadata.ColumnParameter, multiple=True, visible=False ) + MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[5], param=metadata.ColumnParameter, optional=True, multiple=True, visible=False ) MetadataElement( name="sample_names", default=[], desc="Sample names", readonly=True, visible=False, optional=True, no_value=[] ) def sniff( self, filename ): diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 lib/galaxy/model/search.py --- a/lib/galaxy/model/search.py +++ b/lib/galaxy/model/search.py @@ -297,10 +297,14 @@ FIELDS = { 'name' : ViewField('name', sqlalchemy_field=HistoryDatasetAssociation.name), 'id' : ViewField('id',sqlalchemy_field=HistoryDatasetAssociation.id, id_decode=True), + 'history_id' : ViewField('history_id',sqlalchemy_field=HistoryDatasetAssociation.history_id, id_decode=True), 'tag' : ViewField("tag", handler=history_dataset_handle_tag), 'copied_from_ldda_id' : ViewField("copied_from_ldda_id", sqlalchemy_field=HistoryDatasetAssociation.copied_from_library_dataset_dataset_association_id, id_decode=True), + 'copied_from_hda_id' : ViewField("copied_from_hda_id", + sqlalchemy_field=HistoryDatasetAssociation.copied_from_history_dataset_association_id, + id_decode=True), 'deleted' : ViewField('deleted', sqlalchemy_field=HistoryDatasetAssociation.deleted) } diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2083,19 +2083,11 @@ if any_group_errors: errors[input.name] = group_errors else: - if key not in incoming \ - and "__force_update__" + key not in incoming: - # No new value provided, and we are only updating, so keep - # the old value (which should already be in the state) and - # preserve the old error message. - pass - else: - incoming_value = get_incoming_value( incoming, key, None ) - value, error = check_param( trans, input, incoming_value, context, source=source ) - # If a callback was provided, allow it to process the value - if error: - errors[ input.name ] = error - state[ input.name ] = value + incoming_value = get_incoming_value( incoming, key, None ) + value, error = check_param( trans, input, incoming_value, context, source=source ) + if error: + errors[ input.name ] = error + state[ input.name ] = value return errors def update_state( self, trans, inputs, state, incoming, source='html', prefix="", context=None, diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 lib/galaxy/visualization/registry.py --- a/lib/galaxy/visualization/registry.py +++ b/lib/galaxy/visualization/registry.py @@ -477,7 +477,7 @@ return lambda o: getattr( o, next_attr_name ) # recursive case - return lambda o: getattr( self._build_getattr_lambda( attr_name_list[:-1] ), next_attr_name ) + return lambda o: getattr( self._build_getattr_lambda( attr_name_list[:-1] )( o ), next_attr_name ) def parse_tests( self, xml_tree_list ): """ @@ -493,7 +493,7 @@ return tests for test_elem in xml_tree_list: - test_type = test_elem.get( 'type' ) + test_type = test_elem.get( 'type', 'eq' ) test_result = test_elem.text if not test_type or not test_result: log.warn( 'Skipping test. Needs both type attribute and text node to be parsed: ' @@ -509,9 +509,10 @@ getter = self._build_getattr_lambda( test_attr ) # result type should tell the registry how to convert the result before the test - test_result_type = test_elem.get( 'result_type' ) or 'string' + test_result_type = test_elem.get( 'result_type', 'string' ) # test functions should be sent an object to test, and the parsed result expected from the test + # is test_attr attribute an instance of result if test_type == 'isinstance': #TODO: wish we could take this further but it would mean passing in the datatypes_registry diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 lib/galaxy/webapps/galaxy/api/visualizations.py --- a/lib/galaxy/webapps/galaxy/api/visualizations.py +++ b/lib/galaxy/webapps/galaxy/api/visualizations.py @@ -10,7 +10,7 @@ from sqlalchemy import or_ from galaxy import web, util -from galaxy.web.base.controller import BaseAPIController, UsesVisualizationMixin +from galaxy.web.base.controller import BaseAPIController, UsesVisualizationMixin, SharableMixin from galaxy.model.item_attrs import UsesAnnotations from galaxy.exceptions import ( ItemAccessibilityException, ItemDeletionException, ItemOwnershipException, MessageException ) @@ -20,7 +20,7 @@ import logging log = logging.getLogger( __name__ ) -class VisualizationsController( BaseAPIController, UsesVisualizationMixin, UsesAnnotations ): +class VisualizationsController( BaseAPIController, UsesVisualizationMixin, SharableMixin, UsesAnnotations ): """ RESTful controller for interactions with visualizations. """ @@ -122,9 +122,10 @@ else: payload = self._validate_and_parse_payload( payload ) + vis_type = payload.pop( 'type', False ) payload[ 'save' ] = True - # create needs defaults like wizard needs food - generate defaults - this will err if given a weird key? - visualization = self.create_visualization( trans, **payload ) + # generate defaults - this will err if given a weird key? + visualization = self.create_visualization( trans, vis_type, **payload ) rval = { 'id' : trans.security.encode_id( visualization.id ) } @@ -217,11 +218,20 @@ #TODO: deleted #TODO: importable + # must have a type (I've taken this to be the visualization name) + if 'type' not in payload: + raise ValueError( "key/value 'type' is required" ) + validated_payload = {} for key, val in payload.items(): - if key == 'config': + #TODO: validate types in VALID_TYPES/registry names at the mixin/model level? + if key == 'type': + if not ( isinstance( val, str ) or isinstance( val, unicode ) ): + raise ValueError( '%s must be a string or unicode: %s' %( key, str( type( val ) ) ) ) + val = util.sanitize_html.sanitize_html( val, 'utf-8' ) + elif key == 'config': if not isinstance( val, dict ): - raise ValueError( '%s must be a dictionary (JSON): %s' %( key, str( type( val ) ) ) ) + raise ValueError( '%s must be a dictionary: %s' %( key, str( type( val ) ) ) ) elif key == 'annotation': if not ( isinstance( val, str ) or isinstance( val, unicode ) ): @@ -235,21 +245,16 @@ raise ValueError( '%s must be a string or unicode: %s' %( key, str( type( val ) ) ) ) val = util.sanitize_html.sanitize_html( val, 'utf-8' ) elif key == 'slug': - if not isinstance( val, str ): + if not ( isinstance( val, str ) or isinstance( val, unicode ) ): raise ValueError( '%s must be a string: %s' %( key, str( type( val ) ) ) ) val = util.sanitize_html.sanitize_html( val, 'utf-8' ) - elif key == 'type': - if not isinstance( val, str ): - raise ValueError( '%s must be a string: %s' %( key, str( type( val ) ) ) ) - val = util.sanitize_html.sanitize_html( val, 'utf-8' ) - #TODO: validate types in VALID_TYPES/registry names at the mixin/model level? elif key == 'dbkey': if not ( isinstance( val, str ) or isinstance( val, unicode ) ): raise ValueError( '%s must be a string or unicode: %s' %( key, str( type( val ) ) ) ) val = util.sanitize_html.sanitize_html( val, 'utf-8' ) - elif key not in valid_but_uneditable_keys: - raise AttributeError( 'unknown key: %s' %( str( key ) ) ) + #elif key not in valid_but_uneditable_keys: + # raise AttributeError( 'unknown key: %s' %( str( key ) ) ) validated_payload[ key ] = val return validated_payload diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 lib/galaxy/webapps/galaxy/api/workflows.py --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -73,7 +73,10 @@ inputs = {} for step in latest_workflow.steps: if step.type == 'data_input': - inputs[step.id] = {'label':step.tool_inputs['name'], 'value':""} + if step.tool_inputs and "name" in step.tool_inputs: + inputs[step.id] = {'label':step.tool_inputs['name'], 'value':""} + else: + inputs[step.id] = {'label':"Input Dataset", 'value':""} else: pass # Eventually, allow regular tool parameters to be inserted and modified at runtime. diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -436,7 +436,9 @@ tool_dependencies=tool_dependencies ) for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status == trans.app.model.ToolDependency.installation_status.ERROR: - message += ' %s' % str( installed_tool_dependency.error_message ) + text = util.unicodify( installed_tool_dependency.error_message ) + if text is not None: + message += ' %s' % text tool_dependency_ids = [ trans.security.encode_id( td.id ) for td in tool_dependencies ] if message: status = 'error' @@ -681,8 +683,8 @@ action='uninstall_tool_dependencies', **kwd ) ) else: - kwd[ 'message' ] = 'All selected tool dependencies are already uninstalled.' - kwd[ 'status' ] = 'error' + message = 'No selected tool dependencies can be uninstalled, you may need to use the <b>Repair repository</b> feature.' + status = 'error' elif operation == "install": if trans.app.config.tool_dependency_dir: tool_dependencies_for_installation = [] @@ -694,13 +696,12 @@ if tool_dependencies_for_installation: self.initiate_tool_dependency_installation( trans, tool_dependencies_for_installation ) else: - kwd[ 'message' ] = 'All selected tool dependencies are already installed.' - kwd[ 'status' ] = 'error' + message = 'All selected tool dependencies are already installed.' + status = 'error' else: message = 'Set the value of your <b>tool_dependency_dir</b> setting in your Galaxy config file (universe_wsgi.ini) ' message += ' and restart your Galaxy server to install tool dependencies.' - kwd[ 'message' ] = message - kwd[ 'status' ] = 'error' + status = 'error' installed_tool_dependencies_select_field = suc.build_tool_dependencies_select_field( trans, tool_shed_repository=tool_shed_repository, name='inst_td_ids', diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 lib/galaxy/webapps/galaxy/controllers/workflow.py --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -1223,6 +1223,7 @@ for hid in dataset_ids: step = model.WorkflowStep() step.type = 'data_input' + step.tool_inputs = dict( name="Input Dataset" ) hid_to_output_pair[ hid ] = ( step, 'output' ) steps.append( step ) # Tool steps diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -795,12 +795,11 @@ for tool_dependency in repository.missing_tool_dependencies: if tool_dependency.status in [ trans.model.ToolDependency.installation_status.ERROR, trans.model.ToolDependency.installation_status.INSTALLING ]: - tool_dependency_util.set_tool_dependency_attributes( trans, - tool_dependency, - trans.model.ToolDependency.installation_status.UNINSTALLED, - None, - remove_from_disk=True ) - trans.sa_session.refresh( tool_dependency ) + tool_dependency = tool_dependency_util.set_tool_dependency_attributes( trans.app, + tool_dependency=tool_dependency, + status=trans.model.ToolDependency.installation_status.UNINSTALLED, + error_message=None, + remove_from_disk=True ) # Install tool dependencies. suc.update_tool_shed_repository_status( trans.app, repository, diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -28,6 +28,73 @@ INSTALLATION_LOG = 'INSTALLATION.log' VIRTUALENV_URL = 'https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.9.1.tar.gz' + +class EnvFileBuilder( object ): + + def __init__( self, install_dir ): + self.install_dir = install_dir + self.return_code = 0 + + def append_line( self, skip_if_contained=True, make_executable=True, **kwd ): + env_var_dict = dict( **kwd ) + env_entry, env_file = td_common_util.create_or_update_env_shell_file( self.install_dir, env_var_dict ) + return_code = file_append( env_entry, env_file, skip_if_contained=skip_if_contained, make_executable=make_executable ) + self.return_code = self.return_code or return_code + return self.return_code + + +class InstallEnvironment( object ): + """Object describing the environment built up as part of the process of building and installing a package.""" + + def add_env_shell_file_paths( self, paths ): + for path in paths: + self.env_shell_file_paths.append( str( path ) ) + + def build_command( self, command, action_type='shell_command' ): + """ + Build command line for execution from simple command, but + configuring environment described by this object. + """ + env_cmds = self.environment_commands( action_type ) + return '\n'.join( env_cmds + [ command ] ) + + def __call__( self, install_dir ): + with settings( warn_only=True, **td_common_util.get_env_var_values( install_dir ) ): + with prefix( self.__setup_environment() ): + yield + + def environment_commands( self, action_type ): + """Build a list of commands used to construct the environment described by this object.""" + cmds = [] + for env_shell_file_path in self.env_shell_file_paths: + if os.path.exists( env_shell_file_path ): + for env_setting in open( env_shell_file_path ): + cmds.append( env_setting.strip( '\n' ) ) + else: + log.debug( 'Invalid file %s specified, ignoring %s action.' % ( str( env_shell_file_path ), str( action_type ) ) ) + return cmds + + def environment_dict( self, action_type='template_command' ): + env_vars = dict() + for env_shell_file_path in self.env_shell_file_paths: + if os.path.exists( env_shell_file_path ): + for env_setting in open( env_shell_file_path ): + env_string = env_setting.split( ';' )[ 0 ] + env_name, env_path = env_string.split( '=' ) + env_vars[ env_name ] = env_path + else: + log.debug( 'Invalid file %s specified, ignoring template_command action.' % str( env_shell_file_path ) ) + return env_vars + + def __init__( self ): + self.env_shell_file_paths = [] + + def __setup_environment( self ): + return "&&".join( [ ". %s" % file for file in self.__valid_env_shell_file_paths() ] ) + + def __valid_env_shell_file_paths( self ): + return [ file for file in self.env_shell_file_paths if os.path.exists( file ) ] + def check_fabric_version(): version = env.version if int( version.split( "." )[ 0 ] ) < 1: @@ -76,6 +143,11 @@ filtered_actions.append( action ) return filtered_actions +def handle_action_shell_file_paths( env_file_builder, action_dict ): + shell_file_paths = action_dict.get( 'action_shell_file_paths', [] ) + for shell_file_path in shell_file_paths: + env_file_builder.append_line( action="source", value=shell_file_path ) + def handle_command( app, tool_dependency, install_dir, cmd, return_output=False ): sa_session = app.model.context.current with settings( warn_only=True ): @@ -200,79 +272,6 @@ shutil.move( full_path_to_dir, venv_dir ) return True - -class EnvFileBuilder( object ): - - def __init__( self, install_dir ): - self.install_dir = install_dir - self.return_code = 0 - - def append_line( self, skip_if_contained=True, make_executable=True, **kwds ): - env_var_dict = dict(**kwds) - env_entry, env_file = td_common_util.create_or_update_env_shell_file( self.install_dir, env_var_dict ) - return_code = file_append( env_entry, env_file, skip_if_contained=skip_if_contained, make_executable=make_executable ) - self.return_code = self.return_code or return_code - return self.return_code - - -class InstallEnvironment( object ): - """ - Object describing the environment built up as part of the process of building - and installing a package. - """ - - def __init__( self ): - self.env_shell_file_paths = [] - - def build_command( self, command, action_type='shell_command' ): - """ - Build command line for execution from simple command, but - configuring environment described by this object. - """ - env_cmds = self.environment_commands(action_type) - return '\n'.join(env_cmds + [command]) - - def environment_commands(self, action_type): - """ - Build a list of commands used to construct the environment described by - this object. - """ - cmds = [] - for env_shell_file_path in self.env_shell_file_paths: - if os.path.exists( env_shell_file_path ): - for env_setting in open( env_shell_file_path ): - cmds.append( env_setting.strip( '\n' ) ) - else: - log.debug( 'Invalid file %s specified, ignoring %s action.', env_shell_file_path, action_type ) - return cmds - - def __call__( self, install_dir ): - with settings( warn_only=True, **td_common_util.get_env_var_values( install_dir ) ): - with prefix( self.__setup_environment() ): - yield - - def __setup_environment(self): - return "&&".join( [". %s" % file for file in self.__valid_env_shell_file_paths() ] ) - - def __valid_env_shell_file_paths(self): - return [ file for file in self.env_shell_file_paths if os.path.exists( file ) ] - - def environment_dict(self, action_type='template_command'): - env_vars = dict() - for env_shell_file_path in self.env_shell_file_paths: - if os.path.exists( env_shell_file_path ): - for env_setting in open( env_shell_file_path ): - env_string = env_setting.split( ';' )[ 0 ] - env_name, env_path = env_string.split( '=' ) - env_vars[ env_name ] = env_path - else: - log.debug( 'Invalid file %s specified, ignoring template_command action.', env_shell_file_path ) - return env_vars - - def add_env_shell_file_paths(self, paths): - self.env_shell_file_paths.extend(paths) - - def install_and_build_package( app, tool_dependency, actions_dict ): """Install a Galaxy tool dependency package either via a url or a mercurial or git clone command.""" sa_session = app.model.context.current @@ -335,7 +334,6 @@ dir = td_common_util.url_download( work_dir, downloaded_filename, url, extract=True ) if is_binary: log_file = os.path.join( install_dir, INSTALLATION_LOG ) - log.debug( 'log_file: %s' % log_file ) if os.path.exists( log_file ): logfile = open( log_file, 'ab' ) else: @@ -349,7 +347,7 @@ filtered_actions = actions[ 1: ] return_code = handle_command( app, tool_dependency, install_dir, action_dict[ 'command' ] ) if return_code: - return + return tool_dependency dir = package_name elif action_type == 'download_file': # <action type="download_file">http://effectors.org/download/version/TTSS_GUI-1.0.1.jar</action> @@ -374,13 +372,15 @@ # <package>https://github.com/bgruening/download_store/raw/master/DESeq2-1_0_18/BiocGe…</package> # </action> filtered_actions = actions[ 1: ] - - if action_dict.get( 'env_shell_file_paths', False ): - install_environment.add_env_shell_file_paths( action_dict[ 'env_shell_file_paths' ] ) + env_shell_file_paths = action_dict.get( 'env_shell_file_paths', None ) + if env_shell_file_paths is None: + log.debug( 'Missing R environment. Please check your specified R installation exists.' ) + return tool_dependency else: - log.warning( 'Missing R environment. Please check your specified R installation exists.' ) - return - tarball_names = list() + install_environment.add_env_shell_file_paths( env_shell_file_paths ) + log.debug( 'Handling setup_r_environment for tool dependency %s with install_environment.env_shell_file_paths:\n%s"' % \ + ( str( tool_dependency.name ), str( install_environment.env_shell_file_paths ) ) ) + tarball_names = [] for url in action_dict[ 'r_packages' ]: filename = url.split( '/' )[ -1 ] tarball_names.append( filename ) @@ -391,20 +391,18 @@ with settings( warn_only=True ): for tarball_name in tarball_names: cmd = '''export PATH=$PATH:$R_HOME/bin && export R_LIBS=$INSTALL_DIR && - Rscript -e "install.packages(c('%s'),lib='$INSTALL_DIR', repos=NULL, dependencies=FALSE)"''' % (tarball_name) - + Rscript -e "install.packages(c('%s'),lib='$INSTALL_DIR', repos=NULL, dependencies=FALSE)"''' % ( str( tarball_name ) ) cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_dir ) ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: - return - + return tool_dependency # R libraries are installed to $INSTALL_DIR (install_dir), we now set the R_LIBS path to that directory env_file_builder = EnvFileBuilder( install_dir ) handle_action_shell_file_paths( env_file_builder, action_dict ) # Pull in R environment (runtime). env_file_builder.append_line( name="R_LIBS", action="prepend_to", value=install_dir ) return_code = env_file_builder.return_code if return_code: - return + return tool_dependency elif action_type == 'setup_ruby_environment': # setup an Ruby environment # <action type="setup_ruby_environment"> @@ -417,24 +415,27 @@ # <package>http://url-to-some-gem-file.de/protk.gem</package> # </action> filtered_actions = actions[ 1: ] - - if action_dict.get( 'env_shell_file_paths', False ): - install_environment.add_env_shell_file_paths( action_dict[ 'env_shell_file_paths' ] ) + env_shell_file_paths = action_dict.get( 'env_shell_file_paths', None ) + if env_shell_file_paths is None: + log.debug( 'Missing Ruby environment, make sure your specified Ruby installation exists.' ) + return tool_dependency else: - log.warning( 'Missing Ruby environment. Please check if your specified Ruby installation exists.' ) - return - + install_environment.add_env_shell_file_paths( env_shell_file_paths ) + log.debug( 'Handling setup_ruby_environment for tool dependency %s with install_environment.env_shell_file_paths:\n%s"' % \ + ( str( tool_dependency.name ), str( install_environment.env_shell_file_paths ) ) ) dir = os.path.curdir current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) with lcd( current_dir ): with settings( warn_only=True ): - for (gem, gem_version) in action_dict[ 'ruby_packages' ]: + ruby_package_tups = action_dict.get( 'ruby_package_tups', [] ) + for ruby_package_tup in ruby_package_tups: + gem, gem_version = ruby_package_tup if os.path.isfile( gem ): # we assume a local shipped gem file cmd = '''export PATH=$PATH:$RUBY_HOME/bin && export GEM_HOME=$INSTALL_DIR && gem install --local %s''' % ( gem ) - elif gem.find('://') != -1: - # we assume a URL to a gem file + elif gem.find( '://' ) != -1: + # We assume a URL to a gem file. url = gem gem_name = url.split( '/' )[ -1 ] td_common_util.url_download( work_dir, gem_name, url, extract=False ) @@ -453,15 +454,15 @@ cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_dir ) ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: - return - + return tool_dependency env_file_builder = EnvFileBuilder( install_dir ) - handle_action_shell_file_paths( env_file_builder, action_dict ) # Pull in ruby dependencies (runtime). + # Pull in ruby dependencies (runtime). + handle_action_shell_file_paths( env_file_builder, action_dict ) env_file_builder.append_line( name="GEM_PATH", action="prepend_to", value=install_dir ) env_file_builder.append_line( name="PATH", action="prepend_to", value=os.path.join(install_dir, 'bin') ) return_code = env_file_builder.return_code if return_code: - return + return tool_dependency elif action_type == 'setup_perl_environment': # setup an Perl environment # <action type="setup_perl_environment"> @@ -473,65 +474,59 @@ # <package>http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz</package> # </action> filtered_actions = actions[ 1: ] - - if action_dict.get( 'env_shell_file_paths', False ): - install_environment.add_env_shell_file_paths( action_dict[ 'env_shell_file_paths' ] ) + env_shell_file_paths = action_dict.get( 'env_shell_file_paths', None ) + if env_shell_file_paths is None: + log.debug( 'Missing Rerl environment, make sure your specified Rerl installation exists.' ) + return tool_dependency else: - log.warning( 'Missing Rerl environment. Please check if your specified Rerl installation exists.' ) - return - + install_environment.add_env_shell_file_paths( env_shell_file_paths ) + log.debug( 'Handling setup_perl_environment for tool dependency %s with install_environment.env_shell_file_paths:\n%s"' % \ + ( str( tool_dependency.name ), str( install_environment.env_shell_file_paths ) ) ) dir = os.path.curdir current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) with lcd( current_dir ): with settings( warn_only=True ): - - for package in action_dict[ 'perl_packages' ]: - """ - If set to a true value then MakeMaker's prompt function will always return the default without waiting for user input. - """ + perl_packages = action_dict.get( 'perl_packages', [] ) + for perl_package in perl_packages: + # If set to a true value then MakeMaker's prompt function will always + # return the default without waiting for user input. cmd = '''export PERL_MM_USE_DEFAULT=1 && ''' - - if package.find('://') != -1: - # we assume a URL to a gem file - url = package - package_name = url.split( '/' )[ -1 ] - dir = td_common_util.url_download( work_dir, package_name, url, extract=True ) - # search for Build.PL or Makefile.PL (ExtUtils::MakeMaker vs. Module::Build) - - tmp_work_dir = os.path.join( work_dir, dir) + if perl_package.find( '://' ) != -1: + # We assume a URL to a gem file. + url = perl_package + perl_package_name = url.split( '/' )[ -1 ] + dir = td_common_util.url_download( work_dir, perl_package_name, url, extract=True ) + # Search for Build.PL or Makefile.PL (ExtUtils::MakeMaker vs. Module::Build). + tmp_work_dir = os.path.join( work_dir, dir ) if os.path.exists( os.path.join( tmp_work_dir, 'Makefile.PL' ) ): - cmd += '''perl Makefile.PL INSTALL_BASE=$INSTALL_DIR && make && make install''' elif os.path.exists( os.path.join( tmp_work_dir, 'Build.PL' ) ): cmd += '''perl Build.PL --install_base $INSTALL_DIR && perl Build && perl Build install''' else: - log.warning( 'No Makefile.PL or Build.PL file found in %s. Skip installation of %s.' % ( url, package_name ) ) - return + log.debug( 'No Makefile.PL or Build.PL file found in %s. Skipping installation of %s.' % ( url, perl_package_name ) ) + return tool_dependency with lcd( tmp_work_dir ): cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_dir ) ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: - return + return tool_dependency else: - # perl package from CPAN without version number - # cpanm should be installed with the parent perl distribution, otherwise this will not work - cmd += '''cpanm --local-lib=$INSTALL_DIR %s''' % ( package ) - + # perl package from CPAN without version number. + # cpanm should be installed with the parent perl distribution, otherwise this will not work. + cmd += '''cpanm --local-lib=$INSTALL_DIR %s''' % ( perl_package ) cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_dir ) ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: - return - + return tool_dependency env_file_builder = EnvFileBuilder( install_dir ) + # Pull in perl dependencies (runtime). + handle_action_shell_file_paths( env_file_builder, action_dict ) # Recursively add dependent PERL5LIB and PATH to env.sh & anything else needed. - handle_action_shell_file_paths( env_file_builder, action_dict ) # Pull in ruby dependencies (runtime). - - env_file_builder.append_line( name="PERL5LIB", action="prepend_to", value=os.path.join(install_dir, 'lib', 'perl5') ) - env_file_builder.append_line( name="PATH", action="prepend_to", value=os.path.join(install_dir, 'bin') ) + env_file_builder.append_line( name="PERL5LIB", action="prepend_to", value=os.path.join( install_dir, 'lib', 'perl5' ) ) + env_file_builder.append_line( name="PATH", action="prepend_to", value=os.path.join( install_dir, 'bin' ) ) return_code = env_file_builder.return_code if return_code: - return - + return tool_dependency else: # We're handling a complex repository dependency where we only have a set_environment tag set. # <action type="set_environment"> @@ -577,7 +572,7 @@ env_file_builder.append_line( **env_var_dict ) return_code = env_file_builder.return_code if return_code: - return + return tool_dependency elif action_type == 'set_environment_for_install': # Currently the only action supported in this category is a list of paths to one or more tool dependency env.sh files, # the environment setting in each of which will be injected into the environment for all <action type="shell_command"> @@ -588,7 +583,7 @@ venv_src_directory = os.path.abspath( os.path.join( app.config.tool_dependency_dir, '__virtualenv_src' ) ) if not install_virtualenv( app, venv_src_directory ): log.error( 'Unable to install virtualenv' ) - return + return tool_dependency requirements = action_dict[ 'requirements' ] if os.path.exists( os.path.join( dir, requirements ) ): # requirements specified as path to a file @@ -608,26 +603,26 @@ full_setup_command = "%s; %s; %s" % ( setup_command, activate_command, install_command ) return_code = handle_command( app, tool_dependency, install_dir, full_setup_command ) if return_code: - return + return tool_dependency site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join( venv_directory, "bin", "python" ) output = handle_command( app, tool_dependency, install_dir, site_packages_command, return_output=True ) if output.return_code: - return + return tool_dependency if not os.path.exists( output.stdout ): - log.error( "virtualenv's site-packages directory '%s' does not exist", output.stdout ) - return + log.debug( "virtualenv's site-packages directory '%s' does not exist", output.stdout ) + return tool_dependency env_file_builder = EnvFileBuilder( install_dir ) env_file_builder.append_line( name="PYTHONPATH", action="prepend_to", value=output.stdout ) env_file_builder.append_line( name="PATH", action="prepend_to", value=os.path.join( venv_directory, "bin" ) ) return_code = env_file_builder.return_code if return_code: - return + return tool_dependency elif action_type == 'shell_command': with settings( warn_only=True ): cmd = install_environment.build_command( action_dict[ 'command' ] ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: - return + return tool_dependency elif action_type == 'template_command': env_vars = dict() env_vars = install_environment.environment_dict() @@ -639,7 +634,7 @@ cmd = fill_template( '#from fabric.api import env\n%s' % action_dict[ 'command' ], context=env_vars ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: - return + return tool_dependency elif action_type == 'make_install': # make; make install; allow providing make options with settings( warn_only=True ): @@ -647,7 +642,7 @@ cmd = install_environment.build_command( 'make %s && make install' % make_opts ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: - return + return tool_dependency elif action_type == 'autoconf': # Handle configure, make and make install allow providing configuration options with settings( warn_only=True ): @@ -659,7 +654,7 @@ cmd = install_environment.build_command( td_common_util.evaluate_template( pre_cmd, install_dir ) ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: - return + return tool_dependency elif action_type == 'download_file': # Download a single file to the current working directory. url = action_dict[ 'url' ] @@ -705,13 +700,7 @@ td_common_util.move_file( current_dir=work_dir, source=downloaded_filename, destination=full_path_to_dir ) - - -def handle_action_shell_file_paths( env_file_builder, action_dict ): - shell_file_paths = action_dict.get( 'action_shell_file_paths', []) - for shell_file_path in shell_file_paths: - env_file_builder.append_line( action="source", value=shell_file_path ) - + return tool_dependency def log_results( command, fabric_AttributeString, file_path ): """ diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -92,7 +92,7 @@ return text def handle_complex_repository_dependency_for_package( app, elem, package_name, package_version, tool_shed_repository ): - tool_dependency = None + handled_tool_dependencies = [] tool_shed = elem.attrib[ 'toolshed' ] required_repository_name = elem.attrib[ 'name' ] required_repository_owner = elem.attrib[ 'owner' ] @@ -156,26 +156,28 @@ required_repository_owner, required_repository_changeset_revision ) config_to_use = tmp_filename - tool_dependency, actions_dict = populate_actions_dict( app=app, - dependent_install_dir=dependent_install_dir, - required_install_dir=required_repository_package_install_dir, - tool_shed_repository=tool_shed_repository, - required_repository=required_repository, - package_name=package_name, - package_version=package_version, - tool_dependencies_config=config_to_use ) + tool_dependencies, actions_dict = populate_actions_dict( app=app, + dependent_install_dir=dependent_install_dir, + required_install_dir=required_repository_package_install_dir, + tool_shed_repository=tool_shed_repository, + required_repository=required_repository, + package_name=package_name, + package_version=package_version, + tool_dependencies_config=config_to_use ) if tmp_filename: try: os.remove( tmp_filename ) except: pass - # Install and build the package via fabric. - install_and_build_package_via_fabric( app, tool_dependency, actions_dict ) + for tool_dependency in tool_dependencies: + # Install and build the package via fabric and update the tool_dependency record accordingly.. + tool_dependency = install_and_build_package_via_fabric( app, tool_dependency, actions_dict ) + handled_tool_dependencies.append( tool_dependency ) else: message = "Unable to locate required tool shed repository named %s owned by %s with revision %s." % \ ( str( required_repository_name ), str( required_repository_owner ), str( default_required_repository_changeset_revision ) ) raise Exception( message ) - return tool_dependency + return handled_tool_dependencies def handle_set_environment_entry_for_package( app, install_dir, tool_shed_repository, package_name, package_version, elem, required_repository ): """ @@ -184,19 +186,24 @@ """ action_dict = {} actions = [] + tool_dependencies = [] for package_elem in elem: if package_elem.tag == 'install': - # Create the tool_dependency record in the database. + # Create the new tool_dependency record in the database. tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app, tool_shed_repository=tool_shed_repository, name=package_name, version=package_version, type='package', - status=app.model.ToolDependency.installation_status.INSTALLING, + status=app.model.ToolDependency.installation_status.NEVER_INSTALLED, set_status=True ) # Get the installation method version from a tag like: <install version="1.0"> package_install_version = package_elem.get( 'version', '1.0' ) if package_install_version == '1.0': + # Update the tool dependency's status. + tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, + tool_dependency=tool_dependency, + status=app.model.ToolDependency.installation_status.INSTALLING ) # Since the required tool dependency is installed for a repository dependency, we first need to inspect the <actions> tag set to find # the <action type="set_environment"> tag. env_var_dicts = [] @@ -219,9 +226,10 @@ site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join( install_dir, "venv", "bin", "python" ) output = fabric_util.handle_command( app, tool_dependency, install_dir, site_packages_command, return_output=True ) if output.return_code: - log.error( 'Dependency includes a setup_virtualenv action but venv python is broken:', output.stderr ) + log.error( 'Tool dependency %s includes a setup_virtualenv action but venv python is broken: ' % \ + ( str( tool_dependency.name ), str( output.stderr ) ) ) elif not os.path.exists( output.stdout ): - log.error( "virtualenv's site-packages directory '%s' does not exist", output.stdout ) + log.error( "virtualenv's site-packages directory '%s' does not exist", str( output.stdout ) ) else: env_var_dicts.append( dict( name="PYTHONPATH", action="prepend_to", value=output.stdout ) ) env_var_dicts.append( dict( name="PATH", action="prepend_to", value=os.path.join( install_dir, 'venv', 'bin' ) ) ) @@ -234,8 +242,8 @@ # we will replace the current "value" entries in each env_var_dict with the actual path taken from the env.sh # file generated for the installed required repository. Each env_var_dict currently looks something like this: # {'action': 'append_to', 'name': 'LD_LIBRARY_PATH', 'value': '$BOOST_ROOT_DIR/lib/'} - # We'll read the contents of the received required_repository's env.sh file and replace the 'value' entry of each env_var_dict - # with the associated value in the env.sh file. + # We'll read the contents of the received required_repository's env.sh file and replace the 'value' entry of + # each env_var_dict with the associated value in the env.sh file. new_env_var_dicts = [] env_sh_file_dir = tool_dependency_util.get_tool_dependency_install_dir( app=app, repository_name=required_repository.name, @@ -265,16 +273,25 @@ else: action_dict[ 'environment_variable' ] = env_var_dicts actions.append( ( 'set_environment', action_dict ) ) + if tool_dependency.status not in [ app.model.ToolDependency.installation_status.ERROR, + app.model.ToolDependency.installation_status.INSTALLED ]: + # Update the tool dependency's status. + tool_dependency = \ + tool_dependency_util.set_tool_dependency_attributes( app, + tool_dependency=tool_dependency, + status=app.model.ToolDependency.installation_status.INSTALLED ) + # Accumulate processed tool dependencies to return to the caller. + tool_dependencies.append( tool_dependency ) else: raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).' ) - return tool_dependency, actions - return None, actions + return tool_dependencies, actions + return tool_dependencies, actions def install_and_build_package_via_fabric( app, tool_dependency, actions_dict ): sa_session = app.model.context.current try: # There is currently only one fabric method. - fabric_util.install_and_build_package( app, tool_dependency, actions_dict ) + tool_dependency = fabric_util.install_and_build_package( app, tool_dependency, actions_dict ) except Exception, e: log.exception( 'Error installing tool dependency %s version %s.', str( tool_dependency.name ), str( tool_dependency.version ) ) # Since there was an installation error, update the tool dependency status to Error. The remove_installation_path option must @@ -284,10 +301,16 @@ tool_dependency, error_message, remove_installation_path=False ) - if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: - tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED - sa_session.add( tool_dependency ) - sa_session.flush() + if tool_dependency.status not in [ app.model.ToolDependency.installation_status.ERROR, + app.model.ToolDependency.installation_status.INSTALLED ]: + log.debug( 'Changing status for tool dependency %s from %s to %s.' % \ + ( str( tool_dependency.name ), str( tool_dependency.status ), str( app.model.ToolDependency.installation_status.INSTALLED ) ) ) + tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, + tool_dependency=tool_dependency, + status=app.model.ToolDependency.installation_status.INSTALLED, + error_message=None, + remove_from_disk=False ) + return tool_dependency def install_package( app, elem, tool_shed_repository, tool_dependencies=None ): # The value of tool_dependencies is a partial or full list of ToolDependency records associated with the tool_shed_repository. @@ -300,9 +323,15 @@ for package_elem in elem: if package_elem.tag == 'repository': # We have a complex repository dependency definition. - rd_tool_dependency = handle_complex_repository_dependency_for_package( app, package_elem, package_name, package_version, tool_shed_repository ) - if rd_tool_dependency and rd_tool_dependency.status == app.model.ToolDependency.installation_status.ERROR: - print "Error installing tool dependency for required repository: %s" % str( rd_tool_dependency.error_message ) + rd_tool_dependencies = handle_complex_repository_dependency_for_package( app, + package_elem, + package_name, + package_version, + tool_shed_repository ) + for rd_tool_dependency in rd_tool_dependencies: + if rd_tool_dependency.status == app.model.ToolDependency.installation_status.ERROR: + # We'll log the error here, but continue installing packages since some may not require this dependency. + print "Error installing tool dependency for required repository: %s" % str( rd_tool_dependency.error_message ) elif package_elem.tag == 'install': # <install version="1.0"> # Get the installation directory for tool dependencies that will be installed for the received tool_shed_repository. @@ -324,24 +353,27 @@ package_version, 'package' ) if tool_dependency.status == app.model.ToolDependency.installation_status.INSTALLING: - # The tool dependency is in an Installing state, so we don't want to do anything to it. + # The tool dependency is in an Installing state, so we don't want to do anything to it. If the tool + # dependency is being installed by someone else, we don't want to interfere with that. This assumes + # the installation by "someone else" is not hung in an Installing state, which is a weakness if that + # "someone else" never repaired it. log.debug( 'Skipping installation of tool dependency %s version %s because it has a status of %s' % \ - ( tool_dependency.name, tool_dependency.version, tool_dependency.status ) ) + ( str( tool_dependency.name ), str( tool_dependency.version ), str( tool_dependency.status ) ) ) can_install_tool_dependency = False else: - # If the tool dependency is being installed by someone else, we don't want to interfere with that. - # This assumes the installation by "someone else" is not hung in an Installing state, which is a - # weakness if "someone else" never repaired it. tool_dependency_installation_directory_contents = os.listdir( install_dir ) if fabric_util.INSTALLATION_LOG in tool_dependency_installation_directory_contents: - print '\nSkipping installation of tool dependency', package_name, 'version', package_version, \ - 'since it is installed in', install_dir, '\n' + # Since this tool dependency's installation directory contains an installation log, we consider it to be + # installed. In some cases the record may be missing from the database due to some activity outside of + # the control of the Tool Shed. Since a new record was created for it and we don't know the state of the + # files on disk, we will set it to an error state (unless we are running Tool Shed functional tests - see + # below). + log.debug( 'Skipping installation of tool dependency %s version %s because it is installed in %s' % \ + ( str( tool_dependency.name ), str( tool_dependency.version ), str( install_dir ) ) ) can_install_tool_dependency = False - # This tool dependency was previously installed, but the record was missing from the database due to some - # activity outside of the control of the tool shed. Since a new record was created for it and we don't know - # the state of the files on disk, we will set it to an error state. If we are running functional tests, the - # state will be set to Installed, because previously compiled tool dependencies are not deleted by default. if app.config.running_functional_tests: + # If we are running functional tests, the state will be set to Installed because previously compiled + # tool dependencies are not deleted by default, from the "install and test" framework.. tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED else: error_message = 'The installation directory for this tool dependency had contents, but the database had no record. ' @@ -351,9 +383,9 @@ tool_dependency.error_message = error_message else: error_message = '\nInstallation path %s for tool dependency %s version %s exists, but the expected file %s' % \ - ( install_dir, package_name, package_version, fabric_util.INSTALLATION_LOG ) - error_message += ' is missing. This indicates an installation error, so the tool dependency is being' - error_message += ' prepared for reinstallation.' + ( str( install_dir ), str( package_name ), str( package_version ), str( fabric_util.INSTALLATION_LOG ) ) + error_message += ' is missing. This indicates an installation error so the tool dependency is being' + error_message += ' prepared for re-installation.' print error_message tool_dependency.status = app.model.ToolDependency.installation_status.NEVER_INSTALLED try: @@ -373,7 +405,7 @@ version=package_version, type='package', status=app.model.ToolDependency.installation_status.INSTALLING, - set_status=False ) + set_status=True ) # Get the information about the current platform in case the tool dependency definition includes tag sets for installing # compiled binaries. platform_info_dict = tool_dependency_util.get_platform_info_dict() @@ -404,13 +436,12 @@ if binary_installed: continue # No platform-specific <actions> recipe has yet resulted in a successful installation. - install_via_fabric( app, - tool_dependency, - install_dir, - package_name=package_name, - actions_elem=actions_elem, - action_elem=None ) - sa_session.refresh( tool_dependency ) + tool_dependency = install_via_fabric( app, + tool_dependency, + install_dir, + package_name=package_name, + actions_elem=actions_elem, + action_elem=None ) if tool_dependency.status == app.model.ToolDependency.installation_status.INSTALLED: # If an <actions> tag was found that matches the current platform, and the install_via_fabric method # did not result in an error state, set binary_installed to True in order to skip any remaining @@ -419,47 +450,53 @@ else: # Process the next matching <actions> tag, or any defined <actions> tags that do not contain platform # dependent recipes. - print 'Error downloading binary for %s version %s: %s' % \ - ( package_name, package_version, tool_dependency.error_message ) + log.debug( 'Error downloading binary for tool dependency %s version %s: %s' % \ + ( str( package_name ), str( package_version ), str( tool_dependency.error_message ) ) ) else: # If no <actions> tags have been defined that match our current platform, or none of the matching # <actions> tags resulted in a successful tool dependency status, proceed with one and only one # <actions> tag that is not defined to be platform-specific. if not binary_installed: - print 'Binary installation did not occur, so proceeding with install and compile recipe.' + log.debug( 'Proceeding with install and compile recipe for tool dependency %s.' % str( tool_dependency.name ) ) # Make sure to reset for installation if attempt at binary installation resulted in an error. + can_install = True if tool_dependency.status != app.model.ToolDependency.installation_status.NEVER_INSTALLED: removed, error_message = tool_dependency_util.remove_tool_dependency( app, tool_dependency ) - install_via_fabric( app, - tool_dependency, - install_dir, - package_name=package_name, - actions_elem=actions_elem, - action_elem=None ) + if not removed: + log.debug( 'Error removing old files from installation directory %s: %s' % \ + ( str( tool_dependency.installation_directory( app ), str( error_message ) ) ) ) + can_install = False + if can_install: + tool_dependency = install_via_fabric( app, + tool_dependency, + install_dir, + package_name=package_name, + actions_elem=actions_elem, + action_elem=None ) # Perform any final actions that have been defined within the actions_group tag set, but outside of # an <actions> tag, such as a set_environment entry, or a download_file or download_by_url command to # retrieve extra data for this tool dependency. Only do this if the tool dependency is not in an error # state, otherwise skip this action. if actions_elem.tag == 'action' and tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: - install_via_fabric( app, - tool_dependency, - install_dir, - package_name=package_name, - actions_elem=None, - action_elem=actions_elem ) + tool_dependency = install_via_fabric( app, + tool_dependency, + install_dir, + package_name=package_name, + actions_elem=None, + action_elem=actions_elem ) else: # <actions> tags outside of an <actions_group> tag shall not check os or architecture, and if the attributes are # defined, they will be ignored. All <actions> tags outside of an <actions_group> tag set shall always be processed. # This is the default and original behavior of the install_package method. - install_via_fabric( app, - tool_dependency, - install_dir, - package_name=package_name, - actions_elem=actions_elems, - action_elem=None ) - sa_session.refresh( tool_dependency ) + tool_dependency = install_via_fabric( app, + tool_dependency, + install_dir, + package_name=package_name, + actions_elem=actions_elems, + action_elem=None ) if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: - print package_name, 'version', package_version, 'installed in', install_dir + log.debug( 'Tool dependency %s version %s has been installed in %s.' % \ + ( str( package_name ), str( package_version ), str( install_dir ) ) ) else: error_message = 'Version %s of the %s package cannot be installed because ' % ( str( package_version ), str( package_name ) ) error_message += 'the recipe for installing the package is missing either an <actions> tag set or an <actions_group> ' @@ -489,7 +526,6 @@ def install_via_fabric( app, tool_dependency, install_dir, package_name=None, proprietary_fabfile_path=None, actions_elem=None, action_elem=None, **kwd ): """Parse a tool_dependency.xml file's <actions> tag set to gather information for the installation via fabric.""" - sa_session = app.model.context.current if not os.path.exists( install_dir ): os.makedirs( install_dir ) @@ -657,7 +693,7 @@ configure_opts = td_common_util.evaluate_template( action_elem.text, install_dir ) action_dict[ 'configure_opts' ] = configure_opts elif action_type == 'setup_r_environment': - # setup an R environment + # setup an R environment. # <action type="setup_r_environment"> # <repository name="package_r_3_0_1" owner="bgruening"> # <package name="R" version="3.0.1" /> @@ -665,18 +701,19 @@ #  # <package>https://github.com/bgruening/download_store/raw/master/DESeq2-1_0_18/BiocGe…</package> # </action> - td_common_util.parse_setup_environment_repositories( app, all_env_shell_file_paths, action_elem, action_dict ) + # Discover all child repository dependency tags and define the path to an env.sh file associated with each repository. + # This will potentially update the value of the 'env_shell_file_paths' entry in action_dict. + action_dict = td_common_util.get_env_shell_file_paths_from_setup_environment_elem( app, all_env_shell_file_paths, action_elem, action_dict ) r_packages = list() for env_elem in action_elem: if env_elem.tag == 'package': r_packages.append( env_elem.text.strip() ) - if r_packages: action_dict[ 'r_packages' ] = r_packages else: continue elif action_type == 'setup_ruby_environment': - # setup an Ruby environment + # setup a Ruby environment. # <action type="setup_ruby_environment"> # <repository name="package_ruby_2_0" owner="bgruening"> # <package name="ruby" version="2.0" /> @@ -686,33 +723,32 @@ # <package>protk=1.2.4</package> # <package>http://url-to-some-gem-file.de/protk.gem</package> # </action> - td_common_util.parse_setup_environment_repositories( app, all_env_shell_file_paths, action_elem, action_dict ) - ruby_packages = list() + # Discover all child repository dependency tags and define the path to an env.sh file associated with each repository. + # This will potentially update the value of the 'env_shell_file_paths' entry in action_dict. + action_dict = td_common_util.get_env_shell_file_paths_from_setup_environment_elem( app, all_env_shell_file_paths, action_elem, action_dict ) + ruby_package_tups = [] for env_elem in action_elem: if env_elem.tag == 'package': - """ - A valid gem definition can be: - protk=1.2.4 - protk - ftp://ftp.gruening.de/protk.gem - """ - gem_token = env_elem.text.strip().split('=') - if len(gem_token) == 2: + #A valid gem definition can be: + # protk=1.2.4 + # protk + # ftp://ftp.gruening.de/protk.gem + gem_token = env_elem.text.strip().split( '=' ) + if len( gem_token ) == 2: # version string - gem_name = gem_token[0] - gem_version = gem_token[1] - ruby_packages.append( [gem_name, gem_version] ) + gem_name = gem_token[ 0 ] + gem_version = gem_token[ 1 ] + ruby_package_tups.append( ( gem_name, gem_version ) ) else: # gem name for rubygems.org without version number gem = env_elem.text.strip() - ruby_packages.append( [gem, None] ) - - if ruby_packages: - action_dict[ 'ruby_packages' ] = ruby_packages + ruby_package_tups.append( ( gem, None ) ) + if ruby_package_tups: + action_dict[ 'ruby_package_tups' ] = ruby_package_tups else: continue elif action_type == 'setup_perl_environment': - # setup an Perl environment + # setup a Perl environment. # <action type="setup_perl_environment"> # <repository name="package_perl_5_18" owner="bgruening"> # <package name="perl" version="5.18.1" /> @@ -721,17 +757,17 @@ # <package>XML::Parser</package> # <package>http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz</package> # </action> - td_common_util.parse_setup_environment_repositories( app, all_env_shell_file_paths, action_elem, action_dict ) - perl_packages = list() + # Discover all child repository dependency tags and define the path to an env.sh file associated with each repository. + # This will potentially update the value of the 'env_shell_file_paths' entry in action_dict. + action_dict = td_common_util.get_env_shell_file_paths_from_setup_environment_elem( app, all_env_shell_file_paths, action_elem, action_dict ) + perl_packages = [] for env_elem in action_elem: if env_elem.tag == 'package': - """ - A valid package definition can be: - XML::Parser - http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz - Unfortunately, CPAN does not support versioning. If you want real Reproducibility, - you need to specify the tarball path and the right order of different tarballs manually. - """ + # A valid package definition can be: + # XML::Parser + # http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz + # Unfortunately CPAN does not support versioning, so if you want real reproducibility you need to specify + # the tarball path and the right order of different tarballs manually. perl_packages.append( env_elem.text.strip() ) if perl_packages: action_dict[ 'perl_packages' ] = perl_packages @@ -782,7 +818,8 @@ # run_proprietary_fabric_method( app, elem, proprietary_fabfile_path, install_dir, package_name=package_name ) raise Exception( 'Tool dependency installation using proprietary fabric scripts is not yet supported.' ) else: - install_and_build_package_via_fabric( app, tool_dependency, actions_dict ) + tool_dependency = install_and_build_package_via_fabric( app, tool_dependency, actions_dict ) + return tool_dependency def parse_env_shell_entry( action, name, value, line ): new_value = value @@ -814,7 +851,7 @@ actions_dict = dict( install_dir=dependent_install_dir ) if package_name: actions_dict[ 'package_name' ] = package_name - tool_dependency = None + tool_dependencies = [] action_dict = {} if tool_dependencies_config: required_td_tree, error_message = xml_util.parse_xml( tool_dependencies_config ) @@ -827,17 +864,17 @@ required_td_package_name = required_td_elem.get( 'name', None ) required_td_package_version = required_td_elem.get( 'version', None ) if required_td_package_name==package_name and required_td_package_version==package_version: - tool_dependency, actions = handle_set_environment_entry_for_package( app=app, - install_dir=required_install_dir, - tool_shed_repository=tool_shed_repository, - package_name=package_name, - package_version=package_version, - elem=required_td_elem, - required_repository=required_repository ) + tool_dependencies, actions = handle_set_environment_entry_for_package( app=app, + install_dir=required_install_dir, + tool_shed_repository=tool_shed_repository, + package_name=package_name, + package_version=package_version, + elem=required_td_elem, + required_repository=required_repository ) if actions: actions_dict[ 'actions' ] = actions break - return tool_dependency, actions_dict + return tool_dependencies, actions_dict def run_proprietary_fabric_method( app, elem, proprietary_fabfile_path, install_dir, package_name=None, **kwd ): """ @@ -950,13 +987,22 @@ # Handle setting environment variables using a fabric method. fabric_util.file_append( env_entry, env_file, skip_if_contained=True, make_executable=True ) sa_session.refresh( tool_dependency ) - if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: - tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED - sa_session.add( tool_dependency ) - sa_session.flush() - print 'Environment variable ', env_var_name, 'set in', install_dir + if tool_dependency.status not in [ app.model.ToolDependency.installation_status.ERROR, + app.model.ToolDependency.installation_status.INSTALLED ]: + tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, + tool_dependency=tool_dependency, + status=app.model.ToolDependency.installation_status.INSTALLED, + error_message=None, + remove_from_disk=False ) + log.debug( 'Environment variable %s set in %s for tool dependency %s.' % \ + ( str( env_var_name ), str( install_dir ), str( tool_dependency.name ) ) ) else: - raise NotImplementedError( 'Only set_environment version 1.0 is currently supported (i.e., change your tag to be <set_environment version="1.0">).' ) + error_message = 'Only set_environment version 1.0 is currently supported (i.e., change your tag to be <set_environment version="1.0">).' + tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, + tool_dependency=tool_dependency, + status=app.model.ToolDependency.installation_status.ERROR, + error_message=error_message, + remove_from_disk=False ) return tool_dependency def strip_path( fpath ): diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py @@ -257,6 +257,33 @@ log.debug( error_message ) return env_shell_file_paths +def get_env_shell_file_paths_from_setup_environment_elem( app, all_env_shell_file_paths, elem, action_dict ): + """ + Parse an XML tag set to discover all child repository dependency tags and define the path to an env.sh file associated + with the repository (this requires the repository dependency to be in an installed state). The received action_dict + will be updated with these discovered paths and returned to the caller. This method handles tool dependency definition + tag sets <setup_r_environment>, <setup_ruby_environment> and <setup_perl_environment>. + """ + # An example elem is: + # <action type="setup_perl_environment"> + # <repository name="package_perl_5_18" owner="iuc"> + # <package name="perl" version="5.18.1" /> + # </repository> + # <repository name="package_expat_2_1" owner="iuc" prior_installation_required="True"> + # <package name="expat" version="2.1.0" /> + # </repository> + # <package>http://search.cpan.org/CPAN/authors/id/T/TO/TODDR/XML-Parser-2.41.tar.gz</package> + # <package>http://search.cpan.org/CPAN/authors/id/L/LD/LDS/CGI.pm-3.43.tar.gz</package> + # </action> + for action_elem in elem: + if action_elem.tag == 'repository': + env_shell_file_paths = get_env_shell_file_paths( app, action_elem ) + all_env_shell_file_paths.extend( env_shell_file_paths ) + if all_env_shell_file_paths: + action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths + action_dict[ 'action_shell_file_paths' ] = env_shell_file_paths + return action_dict + def get_env_var_values( install_dir ): env_var_dict = {} env_var_dict[ 'INSTALL_DIR' ] = install_dir @@ -415,16 +442,6 @@ continue return actions_elem_tuples - -def parse_setup_environment_repositories( app, all_env_shell_file_paths, action_elem, action_dict ): - env_shell_file_paths = get_env_shell_file_paths( app, action_elem.find('repository') ) - - all_env_shell_file_paths.extend( env_shell_file_paths ) - if all_env_shell_file_paths: - action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths - action_dict[ 'action_shell_file_paths' ] = env_shell_file_paths - - def url_download( install_dir, downloaded_file_name, download_url, extract=True ): file_path = os.path.join( install_dir, downloaded_file_name ) src = None diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -284,18 +284,38 @@ # If there are no tests, this tool should not be tested, since the tool functional tests only report failure if the test itself fails, # not if it's missing or undefined. Filtering out those repositories at this step will reduce the number of "false negatives" the # automated functional test framework produces. - tool_has_tests = True - if 'tests' not in tool_metadata or not tool_metadata[ 'tests' ]: - tool_has_tests = False - if verbosity >= 2: - print '# No functional tests defined for %s.' % tool_id - no_tests += 1 - else: - tool_has_tests = True + tool_has_tests = False + defined_test_dicts = tool_metadata.get( 'tests', None ) + if defined_test_dicts is not None: + # We need to inspect the <test> tags because the following tags... + # <tests> + # </tests> + # ...will produce the following metadata: + # "tests": [] + # And the following tags... + # <tests> + # <test> + # </test> + # </tests> + # ...will produce the following metadata: + # "tests": + # [{"inputs": [], "name": "Test-1", "outputs": [], "required_files": []}] + for defined_test_dict in defined_test_dicts: + inputs = defined_test_dict.get( 'inputs', [] ) + outputs = defined_test_dict.get( 'outputs', [] ) + if inputs and outputs: + # At least one tool within the repository has a valid <test> tag. + tool_has_tests = True + break + if tool_has_tests: if verbosity >= 2: print "# Tool ID '%s' in changeset revision %s of %s has one or more valid functional tests defined." % \ ( tool_id, changeset_revision, name ) has_tests += 1 + else: + if verbosity >= 2: + print '# No functional tests defined for %s.' % tool_id + no_tests += 1 failure_reason = '' problem_found = False missing_test_files = [] diff -r 7b889ba6fd441c4ef41172e90a7342e77637fb4b -r e507124fbe79903378c4b0fe150f2f3fd816ad32 lib/tool_shed/scripts/check_s3_for_empty_tool_dependency_installation_paths.py --- /dev/null +++ b/lib/tool_shed/scripts/check_s3_for_empty_tool_dependency_installation_paths.py @@ -0,0 +1,155 @@ +import argparse +import os +import sys + +new_path = [ os.path.join( os.getcwd(), "lib" ) ] +new_path.extend( sys.path[1:] ) +sys.path = new_path + +from galaxy.util import asbool +from galaxy import eggs +eggs.require( 'boto' ) + +import boto + +from tool_shed.galaxy_install.tool_dependencies.fabric_util import INSTALLATION_LOG + + +class BucketList( object ): + + def __init__( self, amazon_id, amazon_secret, bucket ): + # Connect to S3 using the provided Amazon access key and secret identifier. + self.s3 = boto.connect_s3( amazon_id, amazon_secret ) + self.bucket_name = bucket + # Connect to S3 using the received bucket name. + self.bucket = boto.s3.bucket.Bucket( self.s3, bucket ) + self.install_dirs = self.get_tool_dependency_install_paths() + self.empty_installation_paths = self.check_for_empty_tool_dependency_installation_paths() + + def display_empty_installation_paths( self ): + for empty_installation_path in self.empty_installation_paths: + print empty_installation_path + + def delete_empty_installation_paths( self ): + print 'Deleting empty installation paths.' + for empty_installation_path in self.empty_installation_paths: + # Get all keys in the S3 bucket that start with the installation path, and delete each one. + for path_to_delete in self.bucket.list( prefix=empty_installation_path ): + self.bucket.delete_key( path_to_delete.key ) + print 'Deleted empty path %s' % str( empty_installation_path ) + + def get_tool_dependency_install_paths( self ): + found_paths = [] + for item in self.bucket.list(): + name = str( item.name ) + # Skip environment_settings and __virtualenv_src, since these directories do not contain package tool dependencies. + if name.startswith( 'environment_settings' ) or name.startswith( '__virtualenv_src' ): + continue + paths = name.rstrip('/').split( '/' ) + # Paths are in the format name/version/owner/repository/changeset_revision. If the changeset revision is + # present, we need to check the contents of that path. If not, then the tool dependency was completely + # uninstalled. + if len( paths ) >= 5: + td_install_dir = '/'.join( paths[ :5 ] ) + '/' + if td_install_dir not in found_paths: + found_paths.append( name ) + return found_paths + + def check_for_empty_tool_dependency_installation_paths( self ): + empty_directories = [] + for item in self.install_dirs: + # Get all entries under the path for this tool dependency. + contents = self.bucket.list( prefix=item ) + tool_dependency_path_contents = [] + # Find out if there are two or less items in the path. The first entry will be the installation path itself. + # If only one other item exists, and the full path ends with the installation log, this is an incorrectly installed + # tool dependency. + for item in contents: + tool_dependency_path_contents.append( item ) + # If there are more than two items in the path, we cannot safely assume that the dependency failed to + # install correctly. + if len( tool_dependency_path_contents ) > 2: + break + # If the root directory is the only entry in the path, we have an empty tool dependency installation path. + if len( tool_dependency_path_contents ) == 1: + empty_directories.append( tool_dependency_path_contents[ 0 ] ) + # Otherwise, if the only other entry is the installation log, we have an installation path that should be deleted. + # This would not be the case in a Galaxy instance, since the Galaxy admin will need to verify the contents of + # the installation path in order to determine which action should be taken. + elif len( tool_dependency_path_contents ) == 2 and tool_dependency_path_contents[1].name.endswith( INSTALLATION_LOG ): + empty_directories.append( tool_dependency_path_contents[ 0 ] ) + return [ item.name for item in empty_directories ] + +def main( args ): + ''' + Amazon credentials can be provided in one of three ways: + 1. By specifying them on the command line with the --id and --secret arguments. + 2. By specifying a path to a file that contains the credentials in the form ACCESS_KEY:SECRET_KEY + using the --s3passwd argument. + 3. By specifying the above path in the 's3passwd' environment variable. + Each listed option will override the ones below it, if present. + ''' + if None in [ args.id, args.secret ]: + if args.s3passwd is None: + args.s3passwd = os.environ.get( 's3passwd', None ) + if args.s3passwd is not None and os.path.exists( args.s3passwd ): + awsid, secret = file( args.s3passwd, 'r' ).read().rstrip( '\n' ).split( ':' ) + else: + print 'Amazon ID and secret not provided, and no s3passwd file found.' + return 1 + else: + awsid = args.id + secret = args.secret + dependency_cleaner = BucketList( awsid, secret, args.bucket ) + if len( dependency_cleaner.empty_installation_paths ) == 0: + print 'No empty installation paths found, exiting.' + return 0 + print 'The following %d tool dependency installation paths were found to be empty or contain only the file %s.' % \ + ( len( dependency_cleaner.empty_installation_paths ), INSTALLATION_LOG ) + if asbool( args.delete ): + dependency_cleaner.delete_empty_installation_paths() + else: + for empty_installation_path in dependency_cleaner.empty_installation_paths: + print empty_installation_path + return 0 + +if __name__ == '__main__': + description = 'Determine if there are any tool dependency installation paths that should be removed. Remove them if ' + description += 'the --delete command line argument is provided with a true value.' + parser = argparse.ArgumentParser( description=description ) + parser.add_argument( '--delete', + dest='delete', + required=True, + action='store', + default=False, + type=asbool, + help='Whether to delete empty folders or list them on exit.' ) + parser.add_argument( '--bucket', + dest='bucket', + required=True, + action='store', + metavar='name', + help='The S3 bucket where tool dependencies are installed.' ) + parser.add_argument( '--id', + dest='id', + required=False, + action='store', + default=None, + metavar='ACCESS_KEY', + help='The identifier for an amazon account that has read access to the bucket.' ) + parser.add_argument( '--secret', + dest='secret', + required=False, + action='store', + default=None, + metavar='SECRET_KEY', + help='The secret key for an amazon account that has upload/delete access to the bucket.' ) + parser.add_argument( '--s3passwd', + dest='s3passwd', + required=False, + action='store', + default=None, + metavar='path/file', + help='The path to a file containing Amazon access credentials, in the format KEY:SECRET.' ) + args = parser.parse_args() + sys.exit( main( args ) ) This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Fix string interpolation in a log message.
by commits-noreply＠bitbucket.org 20 Nov '13

20 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/30d30843b57f/ Changeset: 30d30843b57f User: Dave Bouvier Date: 2013-11-20 22:51:02 Summary: Fix string interpolation in a log message. Affected #: 1 file diff -r d0fe08c597a40b32a5d6c9a57b4be07f610fdfc7 -r 30d30843b57f5b8fe809ed2c226cf41e82b8465e lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py --- a/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py +++ b/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py @@ -86,7 +86,7 @@ now = strftime( "%Y-%m-%d %H:%M:%S" ) print "#############################################################################" - print "# %s - Validating repositories of type %s on %s..." % ( TOOL_DEPENDENCY_DEFINITION, now ) + print "# %s - Validating repositories of type %s on %s..." % ( TOOL_DEPENDENCY_DEFINITION, now, config_parser.get( config_section, 'host' ) ) print "# This tool shed is configured to listen on %s:%s" % ( config_parser.get( config_section, 'host' ), config_parser.get( config_section, 'port' ) ) app = RepositoriesApplication( config ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add a script to check repositories of type tool dependency definition in the tool shed to prep for installation testing.
by commits-noreply＠bitbucket.org 20 Nov '13

20 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d0fe08c597a4/ Changeset: d0fe08c597a4 User: greg Date: 2013-11-20 22:41:33 Summary: Add a script to check repositories of type tool dependency definition in the tool shed to prep for installation testing. Affected #: 3 files diff -r f8bebd9e12f53fca874739df55a34b73be224c0b -r d0fe08c597a40b32a5d6c9a57b4be07f610fdfc7 lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py --- /dev/null +++ b/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python + +from datetime import datetime +from optparse import OptionParser +from time import strftime + +import ConfigParser +import logging +import os +import shutil +import sys +import time +import tempfile + +new_path = [ os.path.join( os.getcwd(), "lib" ), os.path.join( os.getcwd(), "test" ) ] +new_path.extend( sys.path[1:] ) +sys.path = new_path + +from galaxy import eggs +eggs.require( "SQLAlchemy >= 0.4" ) +eggs.require( 'mercurial' ) + +from mercurial import __version__ + +import galaxy.webapps.tool_shed.config as tool_shed_config +import galaxy.webapps.tool_shed.model.mapping + +from base.util import get_database_version +from base.util import get_repository_current_revision +from base.util import get_test_environment +from galaxy.model.orm import and_, not_, select +from galaxy.web import url_for +from tool_shed.repository_types.util import TOOL_DEPENDENCY_DEFINITION + +log = logging.getLogger() +log.setLevel( 10 ) +log.addHandler( logging.StreamHandler( sys.stdout ) ) + +assert sys.version_info[ :2 ] >= ( 2, 6 ) + +class RepositoriesApplication( object ): + """Encapsulates the state of a Universe application""" + def __init__( self, config ): + if config.database_connection is False: + config.database_connection = "sqlite:///%s?isolation_level=IMMEDIATE" % config.database + # Setup the database engine and ORM + self.model = galaxy.webapps.tool_shed.model.mapping.init( config.file_path, config.database_connection, engine_options={}, create_tables=False ) + self.hgweb_config_manager = self.model.hgweb_config_manager + self.hgweb_config_manager.hgweb_config_dir = config.hgweb_config_dir + print "# Using configured hgweb.config file: ", self.hgweb_config_manager.hgweb_config + + @property + def sa_session( self ): + """ + Returns a SQLAlchemy session -- currently just gets the current + session from the threadlocal session context, but this is provided + to allow migration toward a more SQLAlchemy 0.4 style of use. + """ + return self.model.context.current + + def shutdown( self ): + pass + +def main(): + '''Script that validates all repositories of type tool_dependency_definition.''' + parser = OptionParser() + parser.add_option( "-i", "--info_only", action="store_true", dest="info_only", help="info about the requested action", default=False ) + parser.add_option( "-s", "--section", action="store", dest="section", default='server:main', + help=".ini file section from which to extract the host and port" ) + parser.add_option( "-v", "--verbose", action="count", dest="verbosity", default=1, help="Control the amount of detail in the log output." ) + parser.add_option( "--verbosity", action="store", dest="verbosity", metavar='VERBOSITY', type="int", + help="Control the amount of detail in the log output, --verbosity=1 is the same as -v" ) + ( options, args ) = parser.parse_args() + try: + ini_file = args[ 0 ] + except IndexError: + print "Usage: python %s <tool shed .ini file> [options]" % sys.argv[ 0 ] + exit( 127 ) + config_parser = ConfigParser.ConfigParser( { 'here':os.getcwd() } ) + config_parser.read( ini_file ) + config_dict = {} + for key, value in config_parser.items( "app:main" ): + config_dict[ key ] = value + config = tool_shed_config.Configuration( **config_dict ) + config_section = options.section + + now = strftime( "%Y-%m-%d %H:%M:%S" ) + print "#############################################################################" + print "# %s - Validating repositories of type %s on %s..." % ( TOOL_DEPENDENCY_DEFINITION, now ) + print "# This tool shed is configured to listen on %s:%s" % ( config_parser.get( config_section, 'host' ), config_parser.get( config_section, 'port' ) ) + + app = RepositoriesApplication( config ) + + if options.info_only: + print "# Displaying info only ( --info_only )" + if options.verbosity: + print "# Displaying extra information ( --verbosity = %d )" % options.verbosity + validate_repositories( app, info_only=options.info_only, verbosity=options.verbosity ) + +def validate_repositories( app, info_only=False, verbosity=1 ): + """ + Inspect records in the repository_metadata table that are associated with repositories of type TOOL_DEPENDENCY_DEFINITION + to ensure they are valid and set the repository_metadata.do_not_test column value to True if the metadata is invalid. + Each repository's metadata should look something like: + "{"tool_dependencies": + {"libpng/1.2.5": {"name": "libpng", + "readme": "README content", + "type": "package", + "version": "1.2.5"}}}" + or: + "{"repository_dependencies": + {"description": null, + "repository_dependencies": + [["http://localhost:9009", "package_libpng_1_2", "iuc", "5788512d4c0a", "True", "False"]]}, + "tool_dependencies": + {"libgd/2.1.0": + {"name": "libgd", "readme": "text"}, + "libpng/1.2.5": + {"name": "libpng", "type": "package", "version": "1.2.5"}}}" + """ + invalid_metadata = 0 + records_checked = 0 + skip_metadata_ids = [] + start = time.time() + valid_metadata = 0 + # Restrict testing to repositories of type TOOL_DEPENDENCY_DEFINITION + tool_dependency_defintion_repository_ids = [] + for repository in app.sa_session.query( app.model.Repository ) \ + .filter( and_( app.model.Repository.table.c.deleted == False, + app.model.Repository.table.c.type == TOOL_DEPENDENCY_DEFINITION ) ): + tool_dependency_defintion_repository_ids.append( repository.id ) + # Do not check metadata records that have an entry in the skip_tool_tests table, since they won't be tested anyway. + skip_metadata_ids = select( [ app.model.SkipToolTest.table.c.repository_metadata_id ] ) + # Get the list of metadata records to check, restricting it to records that have not been flagged do_not_test. + for repository_metadata in app.sa_session.query( app.model.RepositoryMetadata ) \ + .filter( and_( app.model.RepositoryMetadata.table.c.downloadable == True, + app.model.RepositoryMetadata.table.c.do_not_test == False, + app.model.RepositoryMetadata.table.c.repository_id.in_( tool_dependency_defintion_repository_ids ), + not_( app.model.RepositoryMetadata.table.c.id.in_( skip_metadata_ids ) ) ) ): + records_checked += 1 + # Create the repository_status dictionary, using the dictionary from the previous test run if available. + if repository_metadata.tool_test_results: + repository_status = repository_metadata.tool_test_results + else: + repository_status = {} + # Initialize the repository_status dictionary with the information about the current test environment. + last_test_environment = repository_status.get( 'test_environment', None ) + if last_test_environment is None: + test_environment = get_test_environment() + else: + test_environment = get_test_environment( last_test_environment ) + test_environment[ 'tool_shed_database_version' ] = get_database_version( app ) + test_environment[ 'tool_shed_mercurial_version' ] = __version__.version + test_environment[ 'tool_shed_revision' ] = get_repository_current_revision( os.getcwd() ) + repository_status[ 'test_environment' ] = test_environment + # Check the next repository revision. + changeset_revision = str( repository_metadata.changeset_revision ) + name = repository.name + owner = repository.user.username + metadata = repository_metadata.metadata + repository = repository_metadata.repository + if verbosity >= 1: + print '# -------------------------------------------------------------------------------------------' + print '# Checking revision %s of %s owned by %s.' % ( changeset_revision, name, owner ) + if metadata: + # Valid metadata will undoubtedly have a tool_dependencies entry or repository_dependencies entry. + repository_dependencies = metadata.get( 'repository_dependencies', None ) + tool_dependencies = metadata.get( 'tool_dependencies', None ) + if repository_dependencies or tool_dependencies: + print 'Revision %s of %s owned by %s has valid metadata.' % ( changeset_revision, name, owner ) + valid_metadata += 1 + else: + if verbosity >= 1: + print 'Revision %s of %s owned by %s has invalid metadata.' % ( changeset_revision, name, owner ) + invalid_metadata += 1 + if not info_only: + repository_metadata.tool_test_results = repository_status + repository_metadata.time_last_tested = datetime.utcnow() + app.sa_session.add( repository_metadata ) + app.sa_session.flush() + stop = time.time() + print '# -------------------------------------------------------------------------------------------' + print '# Checked %d repository revisions.' % records_checked + print '# %d revisions found with valid tool dependency definition metadata.' % valid_metadata + print '# %d revisions found with valid tool dependency definition metadata.' % invalid_metadata + if info_only: + print '# Database not updated with any information from this run.' + print "# Elapsed time: ", stop - start + print "#############################################################################" + +if __name__ == "__main__": main() diff -r f8bebd9e12f53fca874739df55a34b73be224c0b -r d0fe08c597a40b32a5d6c9a57b4be07f610fdfc7 templates/webapps/tool_shed/repository/manage_repository.mako --- a/templates/webapps/tool_shed/repository/manage_repository.mako +++ b/templates/webapps/tool_shed/repository/manage_repository.mako @@ -203,7 +203,11 @@ %if includes_tools or repository.type == TOOL_DEPENDENCY_DEFINITION: <p/><div class="toolForm"> - <div class="toolFormTitle">Automated tool tests</div> + %if repository.type == TOOL_DEPENDENCY_DEFINITION: + <div class="toolFormTitle">Automated tool dependency test</div> + %else: + <div class="toolFormTitle">Automated tool tests</div> + %endif <div class="toolFormBody"><form name="skip_tool_tests" id="skip_tool_tests" action="${h.url_for( controller='repository', action='manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=repository_metadata.changeset_revision )}" method="post" ><div class="form-row"> diff -r f8bebd9e12f53fca874739df55a34b73be224c0b -r d0fe08c597a40b32a5d6c9a57b4be07f610fdfc7 test/base/util.py --- a/test/base/util.py +++ b/test/base/util.py @@ -1,6 +1,7 @@ -import os, sys, logging, platform - -log = logging.getLogger(__name__) +import logging +import os +import platform +import sys cwd = os.getcwd() if cwd not in sys.path: @@ -16,23 +17,10 @@ eggs.require( 'mercurial' ) -from mercurial import hg, ui, commands +from mercurial import hg +from mercurial import ui -def get_repository_current_revision( repo_path ): - ''' - This method uses the python mercurial API to get the current working directory's mercurial changeset hash. Note that if the author of mercurial - changes the API, this method will have to be updated or replaced. - ''' - # Initialize a mercurial repo object from the provided path. - repo = hg.repository( ui.ui(), repo_path ) - # Get the working directory's change context. - ctx = repo[ None ] - # Extract the changeset hash of the first parent of that change context (the most recent changeset to which the working directory was updated). - changectx = ctx.parents()[ 0 ] - # Also get the numeric revision, so we can return the customary id:hash changeset identifiers. - ctx_rev = changectx.rev() - hg_id = '%d:%s' % ( ctx_rev, str( changectx ) ) - return hg_id +log = logging.getLogger(__name__) def get_database_version( app ): ''' @@ -74,7 +62,25 @@ return None, repository_name, changeset_revision return last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision -def get_test_environment( current_environment={} ): +def get_repository_current_revision( repo_path ): + ''' + This method uses the python mercurial API to get the current working directory's mercurial changeset hash. Note that if the author of mercurial + changes the API, this method will have to be updated or replaced. + ''' + # Initialize a mercurial repo object from the provided path. + repo = hg.repository( ui.ui(), repo_path ) + # Get the working directory's change context. + ctx = repo[ None ] + # Extract the changeset hash of the first parent of that change context (the most recent changeset to which the working directory was updated). + changectx = ctx.parents()[ 0 ] + # Also get the numeric revision, so we can return the customary id:hash changeset identifiers. + ctx_rev = changectx.rev() + hg_id = '%d:%s' % ( ctx_rev, str( changectx ) ) + return hg_id + +def get_test_environment( current_environment=None ): + if current_environment is None: + current_environment = {} rval = current_environment rval[ 'python_version' ] = platform.python_version() rval[ 'architecture' ] = platform.machine() @@ -132,4 +138,3 @@ shed_tools_dict[ guid ] = galaxy_test_file_dir last_galaxy_test_file_dir = galaxy_test_file_dir return has_test_data, shed_tools_dict - Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Display the "Skip automated testing" checkbox on repositories of type tool_dependency_definition in the tool shed.
by commits-noreply＠bitbucket.org 20 Nov '13

20 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f8bebd9e12f5/ Changeset: f8bebd9e12f5 User: greg Date: 2013-11-20 21:24:14 Summary: Display the "Skip automated testing" checkbox on repositories of type tool_dependency_definition in the tool shed. Affected #: 1 file diff -r 3d8ecec9515505f276fdaaa54c2f1af1c1d390b4 -r f8bebd9e12f53fca874739df55a34b73be224c0b templates/webapps/tool_shed/repository/manage_repository.mako --- a/templates/webapps/tool_shed/repository/manage_repository.mako +++ b/templates/webapps/tool_shed/repository/manage_repository.mako @@ -7,6 +7,7 @@ <% from galaxy.web.framework.helpers import time_ago from tool_shed.util.shed_util_common import changeset_is_malicious + from tool_shed.repository_types.util import TOOL_DEPENDENCY_DEFINITION if repository.metadata_revisions: has_metadata = True @@ -199,17 +200,25 @@ </div></div> ${render_repository_items( metadata, containers_dict, can_set_metadata=True, render_repository_actions_for='tool_shed' )} -%if includes_tools: +%if includes_tools or repository.type == TOOL_DEPENDENCY_DEFINITION: <p/><div class="toolForm"><div class="toolFormTitle">Automated tool tests</div><div class="toolFormBody"><form name="skip_tool_tests" id="skip_tool_tests" action="${h.url_for( controller='repository', action='manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=repository_metadata.changeset_revision )}" method="post" ><div class="form-row"> - <label>Skip automated testing of tools in this revision:</label> + %if repository.type == TOOL_DEPENDENCY_DEFINITION: + <label>Skip automated testing of this tool dependency recipe</label> + %else: + <label>Skip automated testing of tools in this revision:</label> + %endif ${skip_tool_tests_check_box.get_html()} <div class="toolParamHelp" style="clear: both;"> - Check the box and click <b>Save</b> to skip automated testing of the tools in this revision. + %if repository.type == TOOL_DEPENDENCY_DEFINITION: + Check the box and click <b>Save</b> to skip automated testing of this tool dependency recipe. + %else: + Check the box and click <b>Save</b> to skip automated testing of the tools in this revision. + %endif </div></div><div style="clear: both"></div> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Enhance the tool shed repository's tool test inspection components to categorize empty <tests> or <test> tag sets defined in tool configs as "tool missing tests" instead of testing the tools with resulting test failures.
by commits-noreply＠bitbucket.org 20 Nov '13

20 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3d8ecec95155/ Changeset: 3d8ecec95155 User: greg Date: 2013-11-20 19:20:14 Summary: Enhance the tool shed repository's tool test inspection components to categorize empty <tests> or <test> tag sets defined in tool configs as "tool missing tests" instead of testing the tools with resulting test failures. Affected #: 1 file diff -r ec6c12a576850401bc502af2aa679a6c18568219 -r 3d8ecec9515505f276fdaaa54c2f1af1c1d390b4 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -284,18 +284,38 @@ # If there are no tests, this tool should not be tested, since the tool functional tests only report failure if the test itself fails, # not if it's missing or undefined. Filtering out those repositories at this step will reduce the number of "false negatives" the # automated functional test framework produces. - tool_has_tests = True - if 'tests' not in tool_metadata or not tool_metadata[ 'tests' ]: - tool_has_tests = False - if verbosity >= 2: - print '# No functional tests defined for %s.' % tool_id - no_tests += 1 - else: - tool_has_tests = True + tool_has_tests = False + defined_test_dicts = tool_metadata.get( 'tests', None ) + if defined_test_dicts is not None: + # We need to inspect the <test> tags because the following tags... + # <tests> + # </tests> + # ...will produce the following metadata: + # "tests": [] + # And the following tags... + # <tests> + # <test> + # </test> + # </tests> + # ...will produce the following metadata: + # "tests": + # [{"inputs": [], "name": "Test-1", "outputs": [], "required_files": []}] + for defined_test_dict in defined_test_dicts: + inputs = defined_test_dict.get( 'inputs', [] ) + outputs = defined_test_dict.get( 'outputs', [] ) + if inputs and outputs: + # At least one tool within the repository has a valid <test> tag. + tool_has_tests = True + break + if tool_has_tests: if verbosity >= 2: print "# Tool ID '%s' in changeset revision %s of %s has one or more valid functional tests defined." % \ ( tool_id, changeset_revision, name ) has_tests += 1 + else: + if verbosity >= 2: + print '# No functional tests defined for %s.' % tool_id + no_tests += 1 failure_reason = '' problem_found = False missing_test_files = [] Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: HDA UI: remove visualizations button for anon users, fix tooltip/popup when only one visualization is found
by commits-noreply＠bitbucket.org 20 Nov '13

20 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ec6c12a57685/ Changeset: ec6c12a57685 User: carlfeberhard Date: 2013-11-20 17:25:41 Summary: HDA UI: remove visualizations button for anon users, fix tooltip/popup when only one visualization is found Affected #: 2 files diff -r df7d45db5d2b284528ba9ca4a02ba10c953453a0 -r ec6c12a576850401bc502af2aa679a6c18568219 static/scripts/mvc/dataset/hda-edit.js --- a/static/scripts/mvc/dataset/hda-edit.js +++ b/static/scripts/mvc/dataset/hda-edit.js @@ -156,7 +156,8 @@ */ _render_visualizationsButton : function(){ var visualizations = this.model.get( 'visualizations' ); - if( ( !this.model.hasData() ) + if( ( !this.hasUser ) + || ( !this.model.hasData() ) || ( _.isEmpty( visualizations ) ) ){ return null; } @@ -209,8 +210,8 @@ } // No need for popup menu because there's a single visualization. - if ( visualizations.length === 1 ) { - $icon.attr( 'title', visualizations[0] ); + if( visualizations.length === 1 ){ + $icon.attr( 'data-original-title', visualizations[0] ); $icon.click( create_viz_action( visualizations[0] ) ); // >1: Populate menu dict with visualization fns, make the popupmenu diff -r df7d45db5d2b284528ba9ca4a02ba10c953453a0 -r ec6c12a576850401bc502af2aa679a6c18568219 static/scripts/packed/mvc/dataset/hda-edit.js --- a/static/scripts/packed/mvc/dataset/hda-edit.js +++ b/static/scripts/packed/mvc/dataset/hda-edit.js @@ -1,1 +1,1 @@ -define(["mvc/dataset/hda-model","mvc/dataset/hda-base"],function(d,a){var f=a.HDABaseView.extend(LoggableMixin).extend({initialize:function(g){a.HDABaseView.prototype.initialize.call(this,g);this.hasUser=g.hasUser;this.defaultPrimaryActionButtonRenderers=[this._render_showParamsButton,this._render_rerunButton]},_render_titleButtons:function(){return a.HDABaseView.prototype._render_titleButtons.call(this).concat([this._render_editButton(),this._render_deleteButton()])},_render_editButton:function(){if((this.model.get("state")===d.HistoryDatasetAssociation.STATES.NEW)||(this.model.get("state")===d.HistoryDatasetAssociation.STATES.DISCARDED)||(this.model.get("state")===d.HistoryDatasetAssociation.STATES.NOT_VIEWABLE)||(!this.model.get("accessible"))){return null}var i=this.model.get("purged"),g=this.model.get("deleted"),h={title:_l("Edit Attributes"),href:this.urls.edit,target:"galaxy_main",classes:"dataset-edit"};if(g||i){h.disabled=true;if(i){h.title=_l("Cannot edit attributes of datasets removed from disk")}else{if(g){h.title=_l("Undelete dataset to edit attributes")}}}else{if(this.model.get("state")===d.HistoryDatasetAssociation.STATES.UPLOAD){h.disabled=true;h.title=_l("This dataset must finish uploading before it can be edited")}}h.faIcon="fa-pencil";return faIconButton(h)},_render_deleteButton:function(){if((this.model.get("state")===d.HistoryDatasetAssociation.STATES.NEW)||(this.model.get("state")===d.HistoryDatasetAssociation.STATES.NOT_VIEWABLE)||(!this.model.get("accessible"))){return null}var g=this,h={title:_l("Delete"),classes:"dataset-delete",onclick:function(){g.$el.find(".icon-btn.dataset-delete").trigger("mouseout");g.model["delete"]()}};if(this.model.get("deleted")||this.model.get("purged")){h={title:_l("Dataset is already deleted"),disabled:true}}h.faIcon="fa-times";return faIconButton(h)},_render_errButton:function(){if(this.model.get("state")!==d.HistoryDatasetAssociation.STATES.ERROR){return null}return faIconButton({title:_l("View or report this error"),href:this.urls.report_error,target:"galaxy_main",faIcon:"fa-bug"})},_render_rerunButton:function(){return faIconButton({title:_l("Run this job again"),href:this.urls.rerun,target:"galaxy_main",faIcon:"fa-refresh"})},_render_visualizationsButton:function(){var g=this.model.get("visualizations");if((!this.model.hasData())||(_.isEmpty(g))){return null}if(_.isObject(g[0])){return this._render_visualizationsFrameworkButton(g)}if(!this.urls.visualization){return null}var i=this.model.get("dbkey"),l=this.urls.visualization,j={},m={dataset_id:this.model.get("id"),hda_ldda:"hda"};if(i){m.dbkey=i}var h=faIconButton({title:_l("Visualize"),href:this.urls.visualization,faIcon:"fa-bar-chart-o"});function k(n){switch(n){case"trackster":return b(l,m,i);case"scatterplot":return e(l,m);default:return function(){Galaxy.frame.add({title:"Visualization",type:"url",content:l+"/"+n+"?"+$.param(m)})}}}if(g.length===1){h.attr("title",g[0]);h.click(k(g[0]))}else{_.each(g,function(o){var n=o.charAt(0).toUpperCase()+o.slice(1);j[_l(n)]=k(o)});make_popupmenu(h,j)}return h},_render_visualizationsFrameworkButton:function(g){if(!(this.model.hasData())||!(g&&!_.isEmpty(g))){return null}var i=faIconButton({title:_l("Visualize"),faIcon:"fa-bar-chart-o"});i.addClass("visualize-icon");if(_.keys(g).length===1){i.attr("title",_.keys(g)[0]);i.attr("href",_.values(g)[0])}else{var j=[];_.each(g,function(k){j.push(k)});var h=new PopupMenu(i,j)}return i},_render_body_failed_metadata:function(){var h=$("<a/>").attr({href:this.urls.edit,target:"galaxy_main"}).text(_l("set it manually or retry auto-detection")),g=$("<span/>").text(". "+_l("You may be able to")+" ").append(h),i=a.HDABaseView.prototype._render_body_failed_metadata.call(this);i.find(".warningmessagesmall strong").append(g);return i},_render_body_error:function(){var g=a.HDABaseView.prototype._render_body_error.call(this);g.find(".dataset-actions .left").prepend(this._render_errButton());return g},_render_body_ok:function(){var g=a.HDABaseView.prototype._render_body_ok.call(this);if(this.model.isDeletedOrPurged()){return g}this.makeDbkeyEditLink(g);if(this.hasUser){g.find(".dataset-actions .left").append(this._render_visualizationsButton());this._renderTags(g);this._renderAnnotation(g)}return g},_renderTags:function(g){this.tagsEditor=new TagsEditor({model:this.model,el:g.find(".tags-display"),onshowFirstTime:function(){this.render()},$activator:faIconButton({title:_l("Edit dataset tags"),classes:"dataset-tag-btn",faIcon:"fa-tags"}).appendTo(g.find(".dataset-actions .right"))})},_renderAnnotation:function(g){this.annotationEditor=new AnnotationEditor({model:this.model,el:g.find(".annotation-display"),onshowFirstTime:function(){this.render()},$activator:faIconButton({title:_l("Edit dataset annotation"),classes:"dataset-annotate-btn",faIcon:"fa-comment"}).appendTo(g.find(".dataset-actions .right"))})},makeDbkeyEditLink:function(g){if(this.model.get("metadata_dbkey")==="?"&&!this.model.isDeletedOrPurged()){g.find(".dataset-dbkey .value").replaceWith($('<a target="galaxy_main">?</a>').attr("href",this.urls.edit))}},events:_.extend(_.clone(a.HDABaseView.prototype.events),{"click .dataset-undelete":function(g){this.model.undelete();return false},"click .dataset-unhide":function(g){this.model.unhide();return false},"click .dataset-purge":"confirmPurge"}),confirmPurge:function c(g){this.model.purge();return false},toString:function(){var g=(this.model)?(this.model+""):("(no model)");return"HDAView("+g+")"}});function e(g,h){action=function(){Galaxy.frame.add({title:"Scatterplot",type:"url",content:g+"/scatterplot?"+$.param(h),target:"galaxy_main",scratchbook:true});$("div.popmenu-wrapper").remove();return false};return action}function b(g,i,h){return function(){var j={};if(h){j["f-dbkey"]=h}$.ajax({url:g+"/list_tracks?"+$.param(j),dataType:"html",error:function(){alert(("Could not add this dataset to browser")+".")},success:function(k){var l=window.parent;l.Galaxy.modal.show({title:"View Data in a New or Saved Visualization",buttons:{Cancel:function(){l.Galaxy.modal.hide()},"View in saved visualization":function(){l.Galaxy.modal.show({title:"Add Data to Saved Visualization",body:k,buttons:{Cancel:function(){l.Galaxy.modal.hide()},"Add to visualization":function(){$(l.document).find("input[name=id]:checked").each(function(){l.Galaxy.modal.hide();var m=$(this).val();i.id=m;l.Galaxy.frame.add({title:"Trackster",type:"url",content:g+"/trackster?"+$.param(i),scratchbook:true})})}}})},"View in new visualization":function(){l.Galaxy.modal.hide();var m=g+"/trackster?"+$.param(i);l.Galaxy.frame.add({title:"Trackster",type:"url",content:m,scratchbook:true})}}})}});return false}}return{HDAEditView:f}}); \ No newline at end of file +define(["mvc/dataset/hda-model","mvc/dataset/hda-base"],function(d,a){var f=a.HDABaseView.extend(LoggableMixin).extend({initialize:function(g){a.HDABaseView.prototype.initialize.call(this,g);this.hasUser=g.hasUser;this.defaultPrimaryActionButtonRenderers=[this._render_showParamsButton,this._render_rerunButton]},_render_titleButtons:function(){return a.HDABaseView.prototype._render_titleButtons.call(this).concat([this._render_editButton(),this._render_deleteButton()])},_render_editButton:function(){if((this.model.get("state")===d.HistoryDatasetAssociation.STATES.NEW)||(this.model.get("state")===d.HistoryDatasetAssociation.STATES.DISCARDED)||(this.model.get("state")===d.HistoryDatasetAssociation.STATES.NOT_VIEWABLE)||(!this.model.get("accessible"))){return null}var i=this.model.get("purged"),g=this.model.get("deleted"),h={title:_l("Edit Attributes"),href:this.urls.edit,target:"galaxy_main",classes:"dataset-edit"};if(g||i){h.disabled=true;if(i){h.title=_l("Cannot edit attributes of datasets removed from disk")}else{if(g){h.title=_l("Undelete dataset to edit attributes")}}}else{if(this.model.get("state")===d.HistoryDatasetAssociation.STATES.UPLOAD){h.disabled=true;h.title=_l("This dataset must finish uploading before it can be edited")}}h.faIcon="fa-pencil";return faIconButton(h)},_render_deleteButton:function(){if((this.model.get("state")===d.HistoryDatasetAssociation.STATES.NEW)||(this.model.get("state")===d.HistoryDatasetAssociation.STATES.NOT_VIEWABLE)||(!this.model.get("accessible"))){return null}var g=this,h={title:_l("Delete"),classes:"dataset-delete",onclick:function(){g.$el.find(".icon-btn.dataset-delete").trigger("mouseout");g.model["delete"]()}};if(this.model.get("deleted")||this.model.get("purged")){h={title:_l("Dataset is already deleted"),disabled:true}}h.faIcon="fa-times";return faIconButton(h)},_render_errButton:function(){if(this.model.get("state")!==d.HistoryDatasetAssociation.STATES.ERROR){return null}return faIconButton({title:_l("View or report this error"),href:this.urls.report_error,target:"galaxy_main",faIcon:"fa-bug"})},_render_rerunButton:function(){return faIconButton({title:_l("Run this job again"),href:this.urls.rerun,target:"galaxy_main",faIcon:"fa-refresh"})},_render_visualizationsButton:function(){var g=this.model.get("visualizations");if((!this.hasUser)||(!this.model.hasData())||(_.isEmpty(g))){return null}if(_.isObject(g[0])){return this._render_visualizationsFrameworkButton(g)}if(!this.urls.visualization){return null}var i=this.model.get("dbkey"),l=this.urls.visualization,j={},m={dataset_id:this.model.get("id"),hda_ldda:"hda"};if(i){m.dbkey=i}var h=faIconButton({title:_l("Visualize"),href:this.urls.visualization,faIcon:"fa-bar-chart-o"});function k(n){switch(n){case"trackster":return b(l,m,i);case"scatterplot":return e(l,m);default:return function(){Galaxy.frame.add({title:"Visualization",type:"url",content:l+"/"+n+"?"+$.param(m)})}}}if(g.length===1){h.attr("data-original-title",g[0]);h.click(k(g[0]))}else{_.each(g,function(o){var n=o.charAt(0).toUpperCase()+o.slice(1);j[_l(n)]=k(o)});make_popupmenu(h,j)}return h},_render_visualizationsFrameworkButton:function(g){if(!(this.model.hasData())||!(g&&!_.isEmpty(g))){return null}var i=faIconButton({title:_l("Visualize"),faIcon:"fa-bar-chart-o"});i.addClass("visualize-icon");if(_.keys(g).length===1){i.attr("title",_.keys(g)[0]);i.attr("href",_.values(g)[0])}else{var j=[];_.each(g,function(k){j.push(k)});var h=new PopupMenu(i,j)}return i},_render_body_failed_metadata:function(){var h=$("<a/>").attr({href:this.urls.edit,target:"galaxy_main"}).text(_l("set it manually or retry auto-detection")),g=$("<span/>").text(". "+_l("You may be able to")+" ").append(h),i=a.HDABaseView.prototype._render_body_failed_metadata.call(this);i.find(".warningmessagesmall strong").append(g);return i},_render_body_error:function(){var g=a.HDABaseView.prototype._render_body_error.call(this);g.find(".dataset-actions .left").prepend(this._render_errButton());return g},_render_body_ok:function(){var g=a.HDABaseView.prototype._render_body_ok.call(this);if(this.model.isDeletedOrPurged()){return g}this.makeDbkeyEditLink(g);if(this.hasUser){g.find(".dataset-actions .left").append(this._render_visualizationsButton());this._renderTags(g);this._renderAnnotation(g)}return g},_renderTags:function(g){this.tagsEditor=new TagsEditor({model:this.model,el:g.find(".tags-display"),onshowFirstTime:function(){this.render()},$activator:faIconButton({title:_l("Edit dataset tags"),classes:"dataset-tag-btn",faIcon:"fa-tags"}).appendTo(g.find(".dataset-actions .right"))})},_renderAnnotation:function(g){this.annotationEditor=new AnnotationEditor({model:this.model,el:g.find(".annotation-display"),onshowFirstTime:function(){this.render()},$activator:faIconButton({title:_l("Edit dataset annotation"),classes:"dataset-annotate-btn",faIcon:"fa-comment"}).appendTo(g.find(".dataset-actions .right"))})},makeDbkeyEditLink:function(g){if(this.model.get("metadata_dbkey")==="?"&&!this.model.isDeletedOrPurged()){g.find(".dataset-dbkey .value").replaceWith($('<a target="galaxy_main">?</a>').attr("href",this.urls.edit))}},events:_.extend(_.clone(a.HDABaseView.prototype.events),{"click .dataset-undelete":function(g){this.model.undelete();return false},"click .dataset-unhide":function(g){this.model.unhide();return false},"click .dataset-purge":"confirmPurge"}),confirmPurge:function c(g){this.model.purge();return false},toString:function(){var g=(this.model)?(this.model+""):("(no model)");return"HDAView("+g+")"}});function e(g,h){action=function(){Galaxy.frame.add({title:"Scatterplot",type:"url",content:g+"/scatterplot?"+$.param(h),target:"galaxy_main",scratchbook:true});$("div.popmenu-wrapper").remove();return false};return action}function b(g,i,h){return function(){var j={};if(h){j["f-dbkey"]=h}$.ajax({url:g+"/list_tracks?"+$.param(j),dataType:"html",error:function(){alert(("Could not add this dataset to browser")+".")},success:function(k){var l=window.parent;l.Galaxy.modal.show({title:"View Data in a New or Saved Visualization",buttons:{Cancel:function(){l.Galaxy.modal.hide()},"View in saved visualization":function(){l.Galaxy.modal.show({title:"Add Data to Saved Visualization",body:k,buttons:{Cancel:function(){l.Galaxy.modal.hide()},"Add to visualization":function(){$(l.document).find("input[name=id]:checked").each(function(){l.Galaxy.modal.hide();var m=$(this).val();i.id=m;l.Galaxy.frame.add({title:"Trackster",type:"url",content:g+"/trackster?"+$.param(i),scratchbook:true})})}}})},"View in new visualization":function(){l.Galaxy.modal.hide();var m=g+"/trackster?"+$.param(i);l.Galaxy.frame.add({title:"Trackster",type:"url",content:m,scratchbook:true})}}})}});return false}}return{HDAEditView:f}}); \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0