2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/510b063792de/ Changeset: 510b063792de Branch: catless-file-merge User: BjoernGruening Date: 2013-03-22 17:05:51 Summary: Change file-merging to use copy.copyfileobj() rather than cat and mv. Affected #: 1 file diff -r d629a346b859c91ba0da27f865952580d1054d80 -r 510b063792de89332428ed12ea1025155a7769c9 lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -544,21 +544,21 @@ return False - def merge( split_files, output_file): """ - TODO: Do we need to merge gzip files using gzjoin? cat seems to work, - but might be brittle. Need to revisit this. + Merge files with copy.copyfileobj() will not hit the + max argument limitation of cat. gz and bz2 files are also working. """ if not split_files: raise ValueError('Asked to merge zero files as %s' % output_file) elif len(split_files) == 1: - cmd = 'mv -f %s %s' % ( split_files[0], output_file ) + shutil.copyfileobj(open(split_files[0], 'rb'), open(output_file, 'wb')) else: - cmd = 'cat %s > %s' % ( ' '.join(split_files), output_file ) - result = os.system(cmd) - if result != 0: - raise Exception('Result %s from %s' % (result, cmd)) + fdst = open(output_file, 'wb') + for fsrc in split_files: + shutil.copyfileobj(open(fsrc, 'rb'), fdst) + fdst.close() + merge = staticmethod(merge) def get_visualizations( self, dataset ): https://bitbucket.org/galaxy/galaxy-central/commits/686781f9a8de/ Changeset: 686781f9a8de User: dannon Date: 2013-04-15 16:03:15 Summary: Merged in BjoernGruening/galaxy-central-bgruening/catless-file-merge (pull request #141) Change file-merging to use copy.copyfileobj() rather than cat and mv. Affected #: 1 file diff -r 3dd4b4a9221c8a839ad80f61cade10ac6480192b -r 686781f9a8de43b74c19eb8282e47e6bc75342dd lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -539,21 +539,21 @@ return False - def merge( split_files, output_file): """ - TODO: Do we need to merge gzip files using gzjoin? cat seems to work, - but might be brittle. Need to revisit this. + Merge files with copy.copyfileobj() will not hit the + max argument limitation of cat. gz and bz2 files are also working. """ if not split_files: raise ValueError('Asked to merge zero files as %s' % output_file) elif len(split_files) == 1: - cmd = 'mv -f %s %s' % ( split_files[0], output_file ) + shutil.copyfileobj(open(split_files[0], 'rb'), open(output_file, 'wb')) else: - cmd = 'cat %s > %s' % ( ' '.join(split_files), output_file ) - result = os.system(cmd) - if result != 0: - raise Exception('Result %s from %s' % (result, cmd)) + fdst = open(output_file, 'wb') + for fsrc in split_files: + shutil.copyfileobj(open(fsrc, 'rb'), fdst) + fdst.close() + merge = staticmethod(merge) def get_visualizations( self, dataset ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.