1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6292ada3115b/ Changeset: 6292ada3115b User: jmchilton Date: 2014-06-10 16:36:28 Summary: Bugfix: Fix shutil.move for converted files in upload.py. Affected #: 2 files diff -r 44081a39d8f0f36d92991a6faece6636914c1169 -r 6292ada3115b4d2b658d1e574d20b08d887e7bec lib/galaxy/datatypes/sniff.py --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -90,7 +90,7 @@ f.close() return False -def convert_newlines( fname, in_place=True ): +def convert_newlines( fname, in_place=True, tmp_dir=None, tmp_prefix=None ): """ Converts in place a file from universal line endings to Posix line endings. @@ -102,7 +102,7 @@ >>> file(fname).read() '1 2\\n3 4\\n' """ - fd, temp_name = tempfile.mkstemp() + fd, temp_name = tempfile.mkstemp( prefix=tmp_prefix, dir=tmp_dir ) fp = os.fdopen( fd, "wt" ) i = None for i, line in enumerate( file( fname, "U" ) ): @@ -150,7 +150,7 @@ else: return ( i, temp_name ) -def convert_newlines_sep2tabs( fname, in_place=True, patt="\\s+" ): +def convert_newlines_sep2tabs( fname, in_place=True, patt="\\s+", tmp_dir=None, tmp_prefix=None ): """ Combines above methods: convert_newlines() and sep2tabs() so that files do not need to be read twice @@ -163,7 +163,7 @@ '1\\t2\\n3\\t4\\n' """ regexp = re.compile( patt ) - fd, temp_name = tempfile.mkstemp() + fd, temp_name = tempfile.mkstemp( prefix=tmp_prefix, dir=tmp_dir ) fp = os.fdopen( fd, "wt" ) for i, line in enumerate( file( fname, "U" ) ): line = line.rstrip( '\r\n' ) diff -r 44081a39d8f0f36d92991a6faece6636914c1169 -r 6292ada3115b4d2b658d1e574d20b08d887e7bec tools/data_source/upload.py --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -272,10 +272,12 @@ # so that is becomes possible to upload gzip, bz2 or zip files with binary data without # corrupting the content of those files. if dataset.to_posix_lines: + tmpdir = output_adjacent_tmpdir( output_path ) + tmp_prefix = 'data_id_%s_convert_' % dataset.dataset_id if dataset.space_to_tab: - line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place ) + line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place, tmp_dir=tmpdir, tmp_prefix=tmp_prefix ) else: - line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place ) + line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place, tmp_dir=tmpdir, tmp_prefix=tmp_prefix ) if dataset.file_type == 'auto': ext = sniff.guess_ext( dataset.path, registry.sniff_order ) else: @@ -343,10 +345,12 @@ dataset.path = temp_name dp = temp_name if not value.is_binary: + tmpdir = output_adjacent_tmpdir( output_path ) + tmp_prefix = 'data_id_%s_convert_' % dataset.dataset_id if dataset.composite_file_paths[ value.name ].get( 'space_to_tab', value.space_to_tab ): - sniff.convert_newlines_sep2tabs( dp ) + sniff.convert_newlines_sep2tabs( dp, tmp_dir=tmpdir, tmp_prefix=tmp_prefix ) else: - sniff.convert_newlines( dp ) + sniff.convert_newlines( dp, tmp_dir=tmpdir, tmp_prefix=tmp_prefix ) shutil.move( dp, os.path.join( files_path, name ) ) # Move the dataset to its "real" path shutil.move( dataset.primary_file, output_path ) @@ -356,6 +360,15 @@ stdout = 'uploaded %s file' % dataset.file_type ) json_file.write( to_json_string( info ) + "\n" ) + +def output_adjacent_tmpdir( output_path ): + """ For temp files that will ultimately be moved to output_path anyway + just create the file directly in output_path's directory so shutil.move + will work optimially. + """ + return os.path.dirname( output_path ) + + def __main__(): if len( sys.argv ) < 4: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.