1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/5e5c0cf930bb/ Changeset: 5e5c0cf930bb User: jmchilton Date: 2014-12-17 20:47:54+00:00 Summary: Merged in jmchilton/galaxy-central-fork-1 (pull request #617) Allow tool tests to assert properties about command, standard output, and standard error. Affected #: 6 files diff -r f24461d3e0b3f737f8e4c99b23475b571c32c61e -r 5e5c0cf930bb96bf8af0bd19daad1a999b40c374 lib/galaxy/tools/parser/xml.py --- a/lib/galaxy/tools/parser/xml.py +++ b/lib/galaxy/tools/parser/xml.py @@ -186,6 +186,11 @@ rval = dict( outputs=__parse_output_elems(test_elem), inputs=__parse_input_elems(test_elem, i), + command=__parse_assert_list_from_elem( test_elem.find("assert_command") ), + stdout=__parse_assert_list_from_elem( test_elem.find("assert_stdout") ), + stderr=__parse_assert_list_from_elem( test_elem.find("assert_stderr") ), + expect_exit_code=test_elem.get("expect_exit_code"), + expect_failure=string_as_bool(test_elem.get("expect_failure", False)), ) _copy_to_dict_if_present(test_elem, rval, ["interactor", "num_outputs"]) return rval @@ -222,6 +227,11 @@ return name, file, attributes +def __parse_command_elem( test_elem ): + assert_elem = test_elem.find("command") + return __parse_assert_list_from_elem( assert_elem ) + + def __parse_test_attributes( output_elem, attrib ): assert_list = __parse_assert_list( output_elem ) file = attrib.pop( 'file', None ) @@ -252,6 +262,10 @@ def __parse_assert_list( output_elem ): assert_elem = output_elem.find("assert_contents") + return __parse_assert_list_from_elem( assert_elem ) + + +def __parse_assert_list_from_elem( assert_elem ): assert_list = None def convert_elem(elem): diff -r f24461d3e0b3f737f8e4c99b23475b571c32c61e -r 5e5c0cf930bb96bf8af0bd19daad1a999b40c374 lib/galaxy/tools/parser/yaml.py --- a/lib/galaxy/tools/parser/yaml.py +++ b/lib/galaxy/tools/parser/yaml.py @@ -152,27 +152,47 @@ attributes["metadata"] = {} # TODO assert_list = [] - for key, assertion in attributes.get("asserts", {}).iteritems(): - # TODO: not handling nested assertions correctly, - # not sure these are used though. - children = [] - if "children" in assertion: - children = assertion["children"] - del assertion["children"] - assert_dict = dict( - tag=key, - attributes=assertion, - children=children, - ) - assert_list.append(assert_dict) + assert_list = __to_test_assert_list( attributes.get("asserts", [] ) ) attributes["assert_list"] = assert_list - _ensure_has(attributes, defaults) test_dict["outputs"] = new_outputs + test_dict["command"] = __to_test_assert_list( test_dict.get( "command", [] ) ) + test_dict["stdout"] = __to_test_assert_list( test_dict.get( "stdout", [] ) ) + test_dict["stderr"] = __to_test_assert_list( test_dict.get( "stderr", [] ) ) + test_dict["expect_exit_code"] = test_dict.get( "expect_exit_code", None ) + test_dict["expect_failure"] = test_dict.get( "expect_exit_code", False ) return test_dict +def __to_test_assert_list(assertions): + def expand_dict_form(item): + key, value = item + new_value = value.copy() + new_value["that"] = key + return new_value + + if isinstance( assertions, dict ): + assertions = map(expand_dict_form, assertions.items() ) + + assert_list = [] + for assertion in assertions: + # TODO: not handling nested assertions correctly, + # not sure these are used though. + children = [] + if "children" in assertion: + children = assertion["children"] + del assertion["children"] + assert_dict = dict( + tag=assertion["that"], + attributes=assertion, + children=children, + ) + assert_list.append(assert_dict) + + return assert_list or None # XML variant is None if no assertions made + + class YamlPageSource(PageSource): def __init__(self, inputs_list): diff -r f24461d3e0b3f737f8e4c99b23475b571c32c61e -r 5e5c0cf930bb96bf8af0bd19daad1a999b40c374 lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -127,6 +127,11 @@ if num_outputs: num_outputs = int( num_outputs ) self.num_outputs = num_outputs + self.command_line = test_dict.get("command", None) + self.stdout = test_dict.get("stdout", None) + self.stderr = test_dict.get("stderr", None) + self.expect_exit_code = test_dict.get("expect_exit_code", None) + self.expect_failure = test_dict.get("expect_failure", False) except Exception, e: self.error = True self.exception = e diff -r f24461d3e0b3f737f8e4c99b23475b571c32c61e -r 5e5c0cf930bb96bf8af0bd19daad1a999b40c374 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -1,6 +1,7 @@ import new import sys from base.twilltestcase import TwillTestCase +from base.asserts import verify_assertions from base.interactor import build_interactor, stage_data_in_history, RunToolException from base.instrument import register_job_data from galaxy.tools import DataManagerTool @@ -92,6 +93,9 @@ raise Exception( "Test parse failure" ) def _verify_outputs( self, testdef, history, jobs, shed_tool_id, data_list, galaxy_interactor ): + assert len(jobs) == 1, "Test framework logic error, somehow tool test resulted in more than one job." + job = jobs[ 0 ] + maxseconds = testdef.maxseconds if testdef.num_outputs is not None: expected = testdef.num_outputs @@ -102,7 +106,33 @@ raise Exception( message ) found_exceptions = [] - job_stdio = None + if testdef.expect_failure: + if testdef.outputs: + raise Exception("Cannot specify outputs in a test expecting failure.") + + # Wait for the job to complete and register expections if the final + # status was not what test was expecting. + job_failed = False + try: + galaxy_interactor.wait_for_job( job[ 'id' ], history, maxseconds ) + except Exception as e: + job_failed = True + if not testdef.expect_failure: + found_exceptions.append(e) + + if not job_failed and testdef.expect_failure: + error = AssertionError("Expected job to fail but Galaxy indicated the job successfully completed.") + found_exceptions.append(error) + + job_stdio = galaxy_interactor.get_job_stdio( job[ 'id' ] ) + + expect_exit_code = testdef.expect_exit_code + if expect_exit_code is not None: + exit_code = job_stdio["exit_code"] + if str(expect_exit_code) != str(exit_code): + error = AssertionError("Expected job to complete with exit code %s, found %s" % (expect_exit_code, exit_code)) + found_exceptions.append(error) + for output_index, output_tuple in enumerate(testdef.outputs): # Get the correct hid name, outfile, attributes = output_tuple @@ -123,14 +153,25 @@ except Exception as e: if not found_exceptions: # Only print this stuff out once. - for job in jobs: - job_stdio = galaxy_interactor.get_job_stdio( job[ 'id' ] ) - for stream in ['stdout', 'stderr']: - if stream in job_stdio: - print >>sys.stderr, self._format_stream( job_stdio[ stream ], stream=stream, format=True ) + for stream in ['stdout', 'stderr']: + if stream in job_stdio: + print >>sys.stderr, self._format_stream( job_stdio[ stream ], stream=stream, format=True ) found_exceptions.append(e) - if job_stdio is None: - job_stdio = galaxy_interactor.get_job_stdio( jobs[0][ 'id' ] ) + + other_checks = { + "command_line": "Command produced by the job", + "stdout": "Standard output of the job", + "stderr": "Standard error of the job", + } + for what, description in other_checks.items(): + if getattr( testdef, what, None ) is not None: + try: + data = job_stdio[what] + verify_assertions( data, getattr( testdef, what ) ) + except AssertionError, err: + errmsg = '%s different than expected\n' % description + errmsg += str( err ) + found_exceptions.append( AssertionError( errmsg ) ) if found_exceptions: raise JobOutputsError(found_exceptions, job_stdio) diff -r f24461d3e0b3f737f8e4c99b23475b571c32c61e -r 5e5c0cf930bb96bf8af0bd19daad1a999b40c374 test/functional/tools/job_properties.xml --- /dev/null +++ b/test/functional/tools/job_properties.xml @@ -0,0 +1,62 @@ +<tool id="job_properties" name="Test Job Properties"> + <command> + #if $thebool + echo "The bool is true"; + echo "The bool is really true" 1>&2; + echo "This is a line of text." > $out_file1 + #else + echo "The bool is not true"; + echo "The bool is very not true" 1>&2; + echo "This is a different line of text." > $out_file1; + sh -c "exit 2" + #end if + #if $failbool + ; sh -c "exit 127" + #end if + + </command> + <inputs> + <param name="thebool" type="boolean" label="The boolean property" /> + <param name="failbool" type="boolean" label="The failure property" checked="false" /> + </inputs> + <outputs> + <data name="out_file1" /> + </outputs> + <stdio> + <exit_code range="127" level="fatal" description="Failing exit code." /> + </stdio> + <tests> + <test expect_exit_code="0"> + <param name="thebool" value="true" /> + <output name="out_file1" file="simple_line.txt" /> + <assert_command> + <has_text text="really" /> + </assert_command> + <assert_stdout> + <has_line line="The bool is true" /> + </assert_stdout> + <assert_stderr> + <has_line line="The bool is really true" /> + </assert_stderr> + </test> + <test expect_exit_code="2"> + <param name="thebool" value="false" /> + <output name="out_file1" file="simple_line_alternative.txt" /> + <assert_command> + <has_text text="very not" /> + </assert_command> + <assert_stdout> + <has_line line="The bool is not true" /> + </assert_stdout> + <assert_stderr> + <has_line line="The bool is very not true" /> + </assert_stderr> + </test> + <test expect_exit_coded="127" expect_failure="true"> + <param name="thebool" value="true" /> + <param name="failbool" value="true" /> + </test> + </tests> + <help> + </help> +</tool> diff -r f24461d3e0b3f737f8e4c99b23475b571c32c61e -r 5e5c0cf930bb96bf8af0bd19daad1a999b40c374 test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -13,6 +13,7 @@ <tool file="multi_output_assign_primary.xml" /><tool file="composite_output.xml" /><tool file="metadata.xml" /> + <tool file="job_properties.xml" /><tool file="gzipped_inputs.xml" /><tool file="output_order.xml" /><tool file="output_format.xml" /> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.