5 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/060677b04984/ Changeset: 060677b04984 Branch: upload_binary_files_archive User: ghuls Date: 2013-05-27 20:55:52 Summary: Avoid corruption of binary files embedded in gzip, bz2 and zip archives in the upload tool. Add an option in the upload tool to disable the conversion of universal line endings to Posix line endings. This is useful for avoiding corruption of uploaded files when a binary file is contained inside a gzip, bz2 and zip archive. This fixes bug report: https://trello.com/card/issue-with-uploaded-2bit-gz-files/506338ce32ae458f6d... Affected #: 5 files diff -r 31714646a7b441f34a43748065278a1b08940c3c -r 060677b04984c5d171c34330879507c9edca979e lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -498,13 +498,14 @@ def before_setting_metadata( self, dataset ): """This function is called on the dataset before metadata is set.""" dataset.clear_associated_files( metadata_safe = True ) - def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, space_to_tab = False, **kwds ): + def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, to_posix_lines = True, space_to_tab = False, **kwds ): kwds[ 'name' ] = name kwds[ 'optional' ] = optional kwds[ 'mimetype' ] = mimetype kwds[ 'description' ] = description kwds[ 'substitute_name_with_metadata' ] = substitute_name_with_metadata kwds[ 'is_binary' ] = is_binary + kwds[ 'to_posix_lines' ] = to_posix_lines kwds[ 'space_to_tab' ] = space_to_tab return Bunch( **kwds ) def add_composite_file( self, name, **kwds ): diff -r 31714646a7b441f34a43748065278a1b08940c3c -r 060677b04984c5d171c34330879507c9edca979e lib/galaxy/tools/actions/upload_common.py --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -307,6 +307,7 @@ is_binary = is_binary, link_data_only = link_data_only, uuid = uuid_str, + to_posix_lines = uploaded_dataset.to_posix_lines, space_to_tab = uploaded_dataset.space_to_tab, in_place = trans.app.config.external_chown_script is None, path = uploaded_dataset.path ) diff -r 31714646a7b441f34a43748065278a1b08940c3c -r 060677b04984c5d171c34330879507c9edca979e lib/galaxy/tools/parameters/grouping.py --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -254,6 +254,9 @@ name = context.get( 'NAME', None ) info = context.get( 'INFO', None ) warnings = [] + to_posix_lines = False + if context.get( 'to_posix_lines', None ) not in [ "None", None, False ]: + to_posix_lines = True space_to_tab = False if context.get( 'space_to_tab', None ) not in [ "None", None, False ]: space_to_tab = True @@ -286,6 +289,7 @@ break if file_bunch.path: break + file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab return file_bunch, warnings def get_filenames( context ): @@ -295,16 +299,21 @@ ftp_files = context['ftp_files'] name = context.get( 'NAME', None ) info = context.get( 'INFO', None ) + to_posix_lines = False + if context.get( 'to_posix_lines', None ) not in [ "None", None, False ]: + to_posix_lines = True space_to_tab = False if context.get( 'space_to_tab', None ) not in [ "None", None, False ]: space_to_tab = True warnings = [] file_bunch = get_data_file_filename( data_file, override_name = name, override_info = info ) if file_bunch.path: + file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab rval.append( file_bunch ) for file_bunch in get_url_paste_urls_or_filename( context, override_name = name, override_info = info ): if file_bunch.path: + file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab rval.append( file_bunch ) # look for files uploaded via FTP @@ -378,11 +387,13 @@ #replace sniff here with just creating an empty file temp_name, is_multi_byte = sniff.stream_to_file( StringIO.StringIO( d_type.generate_primary_file( dataset ) ), prefix='upload_auto_primary_file' ) dataset.primary_file = temp_name + dataset.to_posix_lines = True dataset.space_to_tab = False else: file_bunch, warnings = get_one_filename( groups_incoming[ 0 ] ) writable_files_offset = 1 dataset.primary_file = file_bunch.path + dataset.to_posix_lines = file_bunch.to_posix_lines dataset.space_to_tab = file_bunch.space_to_tab dataset.warnings.extend( warnings ) if dataset.primary_file is None:#remove this before finish, this should create an empty dataset diff -r 31714646a7b441f34a43748065278a1b08940c3c -r 060677b04984c5d171c34330879507c9edca979e tools/data_source/upload.py --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -264,10 +264,14 @@ if link_data_only == 'copy_files': if dataset.type in ( 'server_dir', 'path_paste' ) and data_type not in [ 'gzip', 'bz2', 'zip' ]: in_place = False - if dataset.space_to_tab: - line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place ) - else: - line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place ) + # Convert universal line endings to Posix line endings, but allow the user to turn it off, + # so that is becomes possible to upload gzip, bz2 or zip files with binary data without + # corrupting the content of those files. + if dataset.to_posix_lines: + if dataset.space_to_tab: + line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place ) + else: + line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place ) if dataset.file_type == 'auto': ext = sniff.guess_ext( dataset.path, registry.sniff_order ) else: diff -r 31714646a7b441f34a43748065278a1b08940c3c -r 060677b04984c5d171c34330879507c9edca979e tools/data_source/upload.xml --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -1,6 +1,6 @@ <?xml version="1.0"?> -<tool name="Upload File" id="upload1" version="1.1.3" workflow_compatible="false"> +<tool name="Upload File" id="upload1" version="1.1.4" workflow_compatible="false"><description> from your computer </description> @@ -38,6 +38,9 @@ </param><param name="url_paste" type="text" area="true" size="5x35" label="URL/Text" help="Here you may specify a list of URLs (one per line) or paste the contents of a file."/><param name="ftp_files" type="ftpfile" label="Files uploaded via FTP"/> + <param name="to_posix_lines" type="select" display="checkboxes" multiple="True" label="Convert universal line endings to Posix line endings" help="Turn this option off if you upload a gzip, bz2 or zip archive which contains a binary file."> + <option value="Yes" selected="true">Yes</option> + </param><param name="space_to_tab" type="select" display="checkboxes" multiple="True" label="Convert spaces to tabs" help="Use this option if you are entering intervals by hand."><option value="Yes">Yes</option></param> https://bitbucket.org/galaxy/galaxy-central/commits/8d69bfa4c4e6/ Changeset: 8d69bfa4c4e6 User: jmchilton Date: 2014-01-17 06:57:43 Summary: Merge pull request 171. Affected #: 5 files diff -r 41cd069aa739f735add1b7c6c4bdb5881dc7ed3b -r 8d69bfa4c4e602acf4c7cda0164d9f5e52876dd4 lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -512,13 +512,14 @@ def before_setting_metadata( self, dataset ): """This function is called on the dataset before metadata is set.""" dataset.clear_associated_files( metadata_safe = True ) - def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, space_to_tab = False, **kwds ): + def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, to_posix_lines = True, space_to_tab = False, **kwds ): kwds[ 'name' ] = name kwds[ 'optional' ] = optional kwds[ 'mimetype' ] = mimetype kwds[ 'description' ] = description kwds[ 'substitute_name_with_metadata' ] = substitute_name_with_metadata kwds[ 'is_binary' ] = is_binary + kwds[ 'to_posix_lines' ] = to_posix_lines kwds[ 'space_to_tab' ] = space_to_tab return Bunch( **kwds ) def add_composite_file( self, name, **kwds ): diff -r 41cd069aa739f735add1b7c6c4bdb5881dc7ed3b -r 8d69bfa4c4e602acf4c7cda0164d9f5e52876dd4 lib/galaxy/tools/actions/upload_common.py --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -332,6 +332,7 @@ is_binary=is_binary, link_data_only=link_data_only, uuid=uuid_str, + to_posix_lines=uploaded_dataset.to_posix_lines, space_to_tab=uploaded_dataset.space_to_tab, in_place=trans.app.config.external_chown_script is None, path=uploaded_dataset.path ) diff -r 41cd069aa739f735add1b7c6c4bdb5881dc7ed3b -r 8d69bfa4c4e602acf4c7cda0164d9f5e52876dd4 lib/galaxy/tools/parameters/grouping.py --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -266,6 +266,9 @@ name = context.get( 'NAME', None ) info = context.get( 'INFO', None ) warnings = [] + to_posix_lines = False + if context.get( 'to_posix_lines', None ) not in [ "None", None, False ]: + to_posix_lines = True space_to_tab = False if context.get( 'space_to_tab', None ) not in [ "None", None, False ]: space_to_tab = True @@ -298,6 +301,7 @@ break if file_bunch.path: break + file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab return file_bunch, warnings def get_filenames( context ): @@ -307,16 +311,21 @@ ftp_files = context['ftp_files'] name = context.get( 'NAME', None ) info = context.get( 'INFO', None ) + to_posix_lines = False + if context.get( 'to_posix_lines', None ) not in [ "None", None, False ]: + to_posix_lines = True space_to_tab = False if context.get( 'space_to_tab', None ) not in [ "None", None, False ]: space_to_tab = True warnings = [] file_bunch = get_data_file_filename( data_file, override_name = name, override_info = info ) if file_bunch.path: + file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab rval.append( file_bunch ) for file_bunch in get_url_paste_urls_or_filename( context, override_name = name, override_info = info ): if file_bunch.path: + file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab rval.append( file_bunch ) # look for files uploaded via FTP @@ -390,11 +399,13 @@ #replace sniff here with just creating an empty file temp_name, is_multi_byte = sniff.stream_to_file( StringIO.StringIO( d_type.generate_primary_file( dataset ) ), prefix='upload_auto_primary_file' ) dataset.primary_file = temp_name + dataset.to_posix_lines = True dataset.space_to_tab = False else: file_bunch, warnings = get_one_filename( groups_incoming[ 0 ] ) writable_files_offset = 1 dataset.primary_file = file_bunch.path + dataset.to_posix_lines = file_bunch.to_posix_lines dataset.space_to_tab = file_bunch.space_to_tab dataset.warnings.extend( warnings ) if dataset.primary_file is None:#remove this before finish, this should create an empty dataset diff -r 41cd069aa739f735add1b7c6c4bdb5881dc7ed3b -r 8d69bfa4c4e602acf4c7cda0164d9f5e52876dd4 tools/data_source/upload.py --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -264,10 +264,14 @@ if link_data_only == 'copy_files': if dataset.type in ( 'server_dir', 'path_paste' ) and data_type not in [ 'gzip', 'bz2', 'zip' ]: in_place = False - if dataset.space_to_tab: - line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place ) - else: - line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place ) + # Convert universal line endings to Posix line endings, but allow the user to turn it off, + # so that is becomes possible to upload gzip, bz2 or zip files with binary data without + # corrupting the content of those files. + if dataset.to_posix_lines: + if dataset.space_to_tab: + line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place ) + else: + line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place ) if dataset.file_type == 'auto': ext = sniff.guess_ext( dataset.path, registry.sniff_order ) else: diff -r 41cd069aa739f735add1b7c6c4bdb5881dc7ed3b -r 8d69bfa4c4e602acf4c7cda0164d9f5e52876dd4 tools/data_source/upload.xml --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -1,6 +1,6 @@ <?xml version="1.0"?> -<tool name="Upload File" id="upload1" version="1.1.3" workflow_compatible="false"> +<tool name="Upload File" id="upload1" version="1.1.4" workflow_compatible="false"><description> from your computer </description> @@ -38,6 +38,9 @@ </param><param name="url_paste" type="text" area="true" size="5x35" label="URL/Text" help="Here you may specify a list of URLs (one per line) or paste the contents of a file."/><param name="ftp_files" type="ftpfile" label="Files uploaded via FTP"/> + <param name="to_posix_lines" type="select" display="checkboxes" multiple="True" label="Convert universal line endings to Posix line endings" help="Turn this option off if you upload a gzip, bz2 or zip archive which contains a binary file."> + <option value="Yes" selected="true">Yes</option> + </param><param name="space_to_tab" type="select" display="checkboxes" multiple="True" label="Convert spaces to tabs" help="Use this option if you are entering intervals by hand."><option value="Yes">Yes</option></param> https://bitbucket.org/galaxy/galaxy-central/commits/daeb94f97da7/ Changeset: daeb94f97da7 Branch: upload_binary_files_archive User: jmchilton Date: 2014-01-17 06:58:44 Summary: Close branch upload_binary_files_archive. Affected #: 0 files https://bitbucket.org/galaxy/galaxy-central/commits/9acf83083258/ Changeset: 9acf83083258 User: jmchilton Date: 2014-01-17 07:03:26 Summary: Functionally test various upload options via API. Affected #: 1 file diff -r 8d69bfa4c4e602acf4c7cda0164d9f5e52876dd4 -r 9acf83083258e6b6c9ea867aed97281b27991d0a test/functional/api/test_tools.py --- a/test/functional/api/test_tools.py +++ b/test/functional/api/test_tools.py @@ -33,6 +33,27 @@ create_response = self._post( "tools", data=payload ) self._assert_has_keys( create_response.json(), 'outputs' ) + def test_upload_posix_newline_fixes( self ): + windows_content = "1\t2\t3\r4\t5\t6\r" + posix_content = windows_content.replace("\r", "\n") + result_content = self._upload_and_get_content( windows_content ) + self.assertEquals( result_content, posix_content ) + + def test_upload_disable_posix_fix( self ): + windows_content = "1\t2\t3\r4\t5\t6\r" + result_content = self._upload_and_get_content( windows_content, to_posix_lines=None ) + self.assertEquals( result_content, windows_content ) + + def test_upload_tab_to_space( self ): + table = "1 2 3\n4 5 6\n" + result_content = self._upload_and_get_content( table, space_to_tab="Yes" ) + self.assertEquals( result_content, "1\t2\t3\n4\t5\t6\n" ) + + def test_upload_tab_to_space_off_by_default( self ): + table = "1 2 3\n4 5 6\n" + result_content = self._upload_and_get_content( table ) + self.assertEquals( result_content, table ) + def test_run_cat1( self ): history_id = self._new_history() new_dataset = self._new_dataset( history_id ) @@ -52,6 +73,14 @@ self._assert_has_keys( create_response.json(), 'outputs' ) self._wait_for_history( history_id, assert_ok=True ) + def _upload_and_get_content( self, content, **upload_kwds ): + history_id = self._new_history() + new_dataset = self._new_dataset( history_id, content=content, **upload_kwds ) + self._wait_for_history( history_id, assert_ok=True ) + display_response = self._get( "histories/%s/contents/%s/display" % ( history_id, new_dataset[ "id" ] ) ) + self._assert_status_code_is( display_response, 200 ) + return display_response.content + def _new_dataset( self, history_id, content='TestData123', **kwds ): payload = self._upload_payload( history_id, content, **kwds ) run_response = self._post( "tools", data=payload ) @@ -86,6 +115,10 @@ 'dbkey': dbkey, 'file_type': file_type, } + if "to_posix_lines" in kwds: + upload_params[ "files_0|to_posix_lines"] = kwds[ "to_posix_lines" ] + if "space_to_tab" in kwds: + upload_params[ "files_0|space_to_tab" ] = kwds[ "space_to_tab" ] return self._run_tool_payload( tool_id='upload1', inputs=upload_params, https://bitbucket.org/galaxy/galaxy-central/commits/b1121a315205/ Changeset: b1121a315205 User: jmchilton Date: 2014-01-17 07:03:26 Summary: Hide to_posix_lines in UI by default. Galaxy still completely supports this option on backend - API tests still completely work and will hopefully ensure this functionality continues running. Any Galaxy deployements that wish to enable this option simply have to switch this one param from hidden to select. This is a ugly workaround - but it is a very small workaround - and will hopefully alleviate any potential fears Gert Hulselmans has of Galaxy diverging from his changes. This is only a stop gap until the Galaxy upload UI supports more advanced options. I have previously outlined my concerns with the visual clutter on the upload.xml page - this is why I am hiding it by default. If anyone with commit access disagrees - please by all means backout of this individual changeset and restore the full functionality of pull request 171 (I have a terrible eye for this sort of GUI design stuff). Affected #: 1 file diff -r 9acf83083258e6b6c9ea867aed97281b27991d0a -r b1121a315205e8152d9c59199443c492340e9f0b tools/data_source/upload.xml --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -38,7 +38,10 @@ </param><param name="url_paste" type="text" area="true" size="5x35" label="URL/Text" help="Here you may specify a list of URLs (one per line) or paste the contents of a file."/><param name="ftp_files" type="ftpfile" label="Files uploaded via FTP"/> - <param name="to_posix_lines" type="select" display="checkboxes" multiple="True" label="Convert universal line endings to Posix line endings" help="Turn this option off if you upload a gzip, bz2 or zip archive which contains a binary file."> + <!-- Change the following parameter from hidden to select to enable + modifying this option via the Web GUI. See Bitbucket pull request + 171 for more information. --> + <param name="to_posix_lines" type="hidden" display="checkboxes" multiple="True" label="Convert universal line endings to Posix line endings" help="Turn this option off if you upload a gzip, bz2 or zip archive which contains a binary file." value="Yes"><option value="Yes" selected="true">Yes</option></param><param name="space_to_tab" type="select" display="checkboxes" multiple="True" label="Convert spaces to tabs" help="Use this option if you are entering intervals by hand."> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.