2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/cbfae86c6def/ Changeset: cbfae86c6def User: jmchilton Date: 2013-04-23 16:57:14 Summary: Add exception handling for Binary sniffers. Image sniffers can fail for large files because signed integers are used internally, this catches that and other potential problems. Traceback (most recent call last): File "/opt/galaxy/web/tools/data_source/upload.py", line 432, in <module> __main__() File "/opt/galaxy/web/tools/data_source/upload.py", line 421, in __main__ add_file( dataset, registry, json_file, output_path ) File "/opt/galaxy/web/tools/data_source/upload.py", line 155, in add_file type_info = Binary.is_sniffable_binary( dataset.path ) File "/opt/galaxy/web/lib/galaxy/datatypes/binary.py", line 38, in is_sniffable_binary if format["class"]().sniff(filename): File "/opt/galaxy/web/lib/galaxy/datatypes/images.py", line 203, in sniff headers = get_headers(filename, None, 1) File "/opt/galaxy/web/lib/galaxy/datatypes/sniff.py", line 179, in get_headers for idx, line in enumerate(file(fname)): SystemError: Negative size passed to PyString_FromStringAndSize Affected #: 1 file diff -r 1e3d92ec22b2794bfa2294edb74695c580791406 -r cbfae86c6def520dff8d922066054ede3072ebfb lib/galaxy/datatypes/binary.py --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -33,11 +33,20 @@ Binary.unsniffable_binary_formats.append(ext) @staticmethod - def is_sniffable_binary(filename): + def is_sniffable_binary( filename ): + format_information = None for format in Binary.sniffable_binary_formats: - if format["class"]().sniff(filename): - return (format["type"], format["ext"]) - return None + format_instance = format[ "class" ]() + try: + if format_instance.sniff(filename): + format_information = ( format["type"], format[ "ext" ] ) + break + except Exception: + # Sniffer raised exception, could be any number of + # reasons for this so there is not much to do besides + # trying next sniffer. + pass + return format_information @staticmethod def is_ext_unsniffable(ext): https://bitbucket.org/galaxy/galaxy-central/commits/4f400cb23860/ Changeset: 4f400cb23860 User: jmchilton Date: 2013-04-23 19:24:16 Summary: Rework checking binary files. Why read in line by line if only going to check 100 characters? This approach to just read first 100 characters is cleaner, more efficient, and hopefully less error prone. Should avoid the following exception caused when checking large files: Traceback (most recent call last): File "/opt/galaxy/web/tools/data_source/upload.py", line 432, in <module> __main__() File "/opt/galaxy/web/tools/data_source/upload.py", line 421, in __main__ add_file( dataset, registry, json_file, output_path ) File "/opt/galaxy/web/tools/data_source/upload.py", line 283, in add_file if check_binary( dataset.path ): File "/opt/galaxy/web/lib/galaxy/datatypes/checkers.py", line 58, in check_binary for chars in temp: SystemError: Negative size passed to PyString_FromStringAndSize Affected #: 1 file diff -r cbfae86c6def520dff8d922066054ede3072ebfb -r 4f400cb238607d7563a001ab0cf0d3b24bb10f17 lib/galaxy/datatypes/checkers.py --- a/lib/galaxy/datatypes/checkers.py +++ b/lib/galaxy/datatypes/checkers.py @@ -1,5 +1,6 @@ import os, gzip, re, gzip, zipfile, binascii, bz2, imghdr from galaxy import util +from StringIO import StringIO try: import Image as PIL @@ -53,20 +54,15 @@ if file_path: temp = open( name, "U" ) else: - temp = name + temp = StringIO( name ) chars_read = 0 - for chars in temp: - for char in chars: - chars_read += 1 + try: + for char in temp.read( 100 ): if ord( char ) > 128: is_binary = True break - if chars_read > 100: - break - if chars_read > 100: - break - if file_path: - temp.close() + finally: + temp.close( ) return is_binary def check_gzip( file_path ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.