details: http://www.bx.psu.edu/hg/galaxy/rev/e5ff0ca1fc95 changeset: 3491:e5ff0ca1fc95 user: Dan Blankenberg <dan@bx.psu.edu> date: Mon Mar 08 14:35:04 2010 -0500 description: Update Test framework to allow testing of contents of extra_files_path. This is done using a <extra_files> tag; this tag can be used to compare an entire directory or individual files. Compare types of 'diff', 're_match' and 're_match_multiline' are available. e.g. Individual files: <output name="output_html_file" file="peakcalling_macs/macs_test_1_out.html" compare="re_match" > <extra_files type="file" name="Galaxy_Test_Run_model.pdf" value="peakcalling_macs/test2/Galaxy_Test_Run_model.pdf" compare="re_match"/> <extra_files type="file" name="Galaxy_Test_Run_model.r" value="peakcalling_macs/test2/Galaxy_Test_Run_model.r" compare="re_match"/> <extra_files type="file" name="Galaxy_Test_Run_model.r.log" value="peakcalling_macs/test2/Galaxy_Test_Run_model.r.log"/> <extra_files type="file" name="Galaxy_Test_Run_negative_peaks.xls" value="peakcalling_macs/test2/Galaxy_Test_Run_negative_peaks.xls" compare="re_match"/> <extra_files type="file" name="Galaxy_Test_Run_peaks.xls" value="peakcalling_macs/test2/Galaxy_Test_Run_peaks.xls" compare="re_match"/> </output> By a directory: <output name="output_html_file" file="peakcalling_macs/macs_test_1_out.html" compare="re_match" > <extra_files type="directory" value="peakcalling_macs/test2/" compare="re_match"/> </output> diffstat: lib/galaxy/tools/__init__.py | 20 +++++++++-- test/base/twilltestcase.py | 75 ++++++++++++++++++++++++++++++------------- 2 files changed, 68 insertions(+), 27 deletions(-) diffs (169 lines): diff -r 97fe4a44b3bf -r e5ff0ca1fc95 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py Mon Mar 08 11:37:35 2010 -0500 +++ b/lib/galaxy/tools/__init__.py Mon Mar 08 14:35:04 2010 -0500 @@ -532,10 +532,22 @@ file = attrib.pop( 'file', None ) if file is None: raise Exception( "Test output does not have a 'file'") - attributes = Bunch() - attributes.compare = attrib.pop( 'compare', 'diff' ).lower() #method of comparison - attributes.lines_diff = int( attrib.pop( 'lines_diff', '0' ) ) # allow a few lines (dates etc) to vary in logs - attributes.sort = util.string_as_bool( attrib.pop( 'sort', False ) ) + attributes = {} + attributes['compare'] = attrib.pop( 'compare', 'diff' ).lower() #method of comparison + attributes['lines_diff'] = int( attrib.pop( 'lines_diff', '0' ) ) # allow a few lines (dates etc) to vary in logs + attributes['sort'] = util.string_as_bool( attrib.pop( 'sort', False ) ) + attributes['extra_files'] = [] + for extra in output_elem.findall( 'extra_files' ): + extra_type = extra.get( 'type', 'file' ) #file or directory, when directory, compare basename by basename + extra_name = extra.get( 'name', None ) + assert extra_type == 'directory' or extra_name is not None, 'extra_files type (%s) requires a name attribute' % extra_type + extra_value = extra.get( 'value', None ) + assert extra_value is not None, 'extra_files requires a value attribute' + extra_attributes = {} + extra_attributes['compare'] = extra.get( 'compare', 'diff' ).lower() #method of comparison + extra_attributes['lines_diff'] = int( extra.get( 'lines_diff', '0' ) ) # allow a few lines (dates etc) to vary in logs + extra_attributes['sort'] = util.string_as_bool( extra.get( 'sort', False ) ) + attributes['extra_files'].append( ( extra_type, extra_value, extra_name, extra_attributes ) ) test.add_output( name, file, attributes ) except Exception, e: test.error = True diff -r 97fe4a44b3bf -r e5ff0ca1fc95 test/base/twilltestcase.py --- a/test/base/twilltestcase.py Mon Mar 08 11:37:35 2010 -0500 +++ b/test/base/twilltestcase.py Mon Mar 08 14:35:04 2010 -0500 @@ -48,7 +48,9 @@ files_differ = False local_file = open( file1, 'U' ).readlines() history_data = open( file2, 'U' ).readlines() - if attributes and attributes.sort: + if attributes is None: + attributes = {} + if attributes.get( 'sort', False ): history_data.sort() ##Why even bother with the check loop below, why not just use the diff output? This seems wasteful. if len( local_file ) == len( history_data ): @@ -59,10 +61,7 @@ else: files_differ = True if files_differ: - if attributes and attributes.lines_diff: - allowed_diff_count = attributes.lines_diff - else: - allowed_diff_count = 0 + allowed_diff_count = attributes.get( 'lines_diff', 0 ) diff = list( difflib.unified_diff( local_file, history_data, "local_file", "history_data" ) ) diff_lines = get_lines_diff( diff ) if diff_lines > allowed_diff_count: @@ -106,12 +105,11 @@ local_file = open( file1, 'U' ).readlines() #regex file history_data = open( file2, 'U' ).readlines() assert len( local_file ) == len( history_data ), 'Data File and Regular Expression File contain a different number of lines (%s != %s)' % ( len( local_file ), len( history_data ) ) - if attributes and attributes.sort: + if attributes is None: + attributes = {} + if attributes.get( 'sort', False ): history_data.sort() - if attributes and attributes.lines_diff: - lines_diff = attributes.lines_diff - else: - lines_diff = 0 + lines_diff = attributes.get( 'lines_diff', 0 ) line_diff_count = 0 diffs = [] for i in range( len( history_data ) ): @@ -124,7 +122,9 @@ def files_re_match_multiline( self, file1, file2, attributes=None ): """Checks the contents of 2 files for differences using re.match in multiline mode""" local_file = open( file1, 'U' ).read() #regex file - if attributes and attributes.sort: + if attributes is None: + attributes = {} + if attributes.get( 'sort', False ): history_data = open( file2, 'U' ).readlines() history_data.sort() history_data = ''.join( history_data ) @@ -619,16 +619,16 @@ raise AssertionError( errmsg ) else: local_name = self.get_filename( filename ) - temp_name = self.get_filename( 'temp_%s' % filename ) + temp_name = self.get_filename( '%s_temp' % filename ) #This is a terrible way to generate a temp name self.home() self.visit_page( "display?hid=" + hid ) data = self.last_page() file( temp_name, 'wb' ).write(data) try: - if attributes and 'compare' in attributes: - compare = attributes.compare - else: - compare = 'diff' + if attributes is None: + attributes = {} + compare = attributes.get( 'compare', 'diff' ) + extra_files = attributes.get( 'extra_files', None ) if compare == 'diff': self.files_diff( local_name, temp_name, attributes=attributes ) elif compare == 're_match': @@ -637,26 +637,55 @@ self.files_re_match_multiline( local_name, temp_name, attributes=attributes ) else: raise Exception, 'Unimplemented Compare type: %s' % compare + if extra_files: + self.verify_extra_files_content( extra_files, elem.get( 'id' ) ) except AssertionError, err: errmsg = 'History item %s different than expected, difference (using %s):\n' % ( hid, compare ) errmsg += str( err ) raise AssertionError( errmsg ) finally: os.remove( temp_name ) - def verify_composite_datatype_file_content( self, file_name, hda_id ): + + def verify_extra_files_content( self, extra_files, hda_id ): + files_list = [] + for extra_type, extra_value, extra_name, extra_attributes in extra_files: + if extra_type == 'file': + files_list.append( ( extra_name, extra_value, extra_attributes ) ) + elif extra_type == 'directory': + for filename in os.listdir( self.get_filename( extra_value ) ): + files_list.append( ( filename, os.path.join( extra_value, filename ), extra_attributes ) ) + else: + raise ValueError, 'unknown extra_files type: %s' % extra_type + for filename, filepath, attributes in files_list: + self.verify_composite_datatype_file_content( filepath, hda_id, base_name = filename, attributes = attributes ) + + def verify_composite_datatype_file_content( self, file_name, hda_id, base_name = None, attributes = None ): local_name = self.get_filename( file_name ) - temp_name = self.get_filename( 'temp_%s' % file_name ) - self.visit_url( "%s/datasets/%s/display/%s" % ( self.url, self.security.encode_id( hda_id ), file_name ) ) + if base_name is None: + base_name = file_name + temp_name = self.get_filename( '%s_temp' % file_name ) #This is a terrible way to generate a temp name + self.visit_url( "%s/datasets/%s/display/%s" % ( self.url, self.security.encode_id( hda_id ), base_name ) ) data = self.last_page() file( temp_name, 'wb' ).write( data ) try: - self.files_diff( local_name, temp_name ) + if attributes is None: + attributes = {} + compare = attributes.get( 'compare', 'diff' ) + if compare == 'diff': + self.files_diff( local_name, temp_name, attributes=attributes ) + elif compare == 're_match': + self.files_re_match( local_name, temp_name, attributes=attributes ) + elif compare == 're_match_multiline': + self.files_re_match_multiline( local_name, temp_name, attributes=attributes ) + else: + raise Exception, 'Unimplemented Compare type: %s' % compare except AssertionError, err: - os.remove( temp_name ) - errmsg = 'History item %s different than expected, difference:\n' % str( hda_id ) + errmsg = 'Composite file (%s) of History item %s different than expected, difference (using %s):\n' % ( base_name, hda_id, compare ) errmsg += str( err ) raise AssertionError( errmsg ) - os.remove( temp_name ) + finally: + os.remove( temp_name ) + def is_zipped( self, filename ): if not zipfile.is_zipfile( filename ): return False