1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8f2b1130bc52/ Changeset: 8f2b1130bc52 User: Dave Bouvier Date: 2013-10-03 20:17:19 Summary: Enhance handling of downloaded compressed files based on their internal directory structure. Add rename_to attribute to the move_file action. Affected #: 3 files diff -r 84781f00ff8e57848397177511b77a400055b59f -r 8f2b1130bc523ba60731eb4c5cecfb004e7f9b1b lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -284,7 +284,8 @@ elif action_type == 'move_file': td_common_util.move_file( current_dir=current_dir, source=os.path.join( action_dict[ 'source' ] ), - destination_dir=os.path.join( action_dict[ 'destination' ] ) ) + destination=os.path.join( action_dict[ 'destination' ] ), + rename_to=action_dict[ 'rename_to' ] ) elif action_type == 'set_environment': # Currently the only action supported in this category is "environment_variable". # Build a command line from the prior_installation_required, in case an environment variable is referenced diff -r 84781f00ff8e57848397177511b77a400055b59f -r 8f2b1130bc523ba60731eb4c5cecfb004e7f9b1b lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -464,7 +464,7 @@ is_binary_download = True else: is_binary_download = False - elif action_elem: + elif action_elem is not None: # We were provided with a single <action> element to perform certain actions after a platform-specific tarball was downloaded. elems = [ action_elem ] else: @@ -550,11 +550,7 @@ action_dict[ 'directory' ] = action_elem.text else: continue - elif action_type in [ 'move_directory_files', 'move_file' ]: - # <action type="move_file"> - # <source>misc/some_file</source> - # <destination>$INSTALL_DIR/bin</destination> - # </action> + elif action_type == 'move_directory_files': # <action type="move_directory_files"> # <source_directory>bin</source_directory> # <destination_directory>$INSTALL_DIR/bin</destination_directory> @@ -563,6 +559,14 @@ move_elem_text = evaluate_template( move_elem.text ) if move_elem_text: action_dict[ move_elem.tag ] = move_elem_text + elif action_type == 'move_file': + # <action type="move_file" rename_to="new_file_name"> + # <source>misc/some_file</source> + # <destination>$INSTALL_DIR/bin</destination> + # </action> + action_dict[ 'source' ] = evaluate_template( action_elem.find( 'source' ).text ) + action_dict[ 'destination' ] = evaluate_template( action_elem.find( 'destination' ).text ) + action_dict[ 'rename_to' ] = action_elem.get( 'rename_to' ) elif action_type == 'set_environment': # <action type="set_environment"> # <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable> diff -r 84781f00ff8e57848397177511b77a400055b59f -r 8f2b1130bc523ba60731eb4c5cecfb004e7f9b1b lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py @@ -12,6 +12,109 @@ log = logging.getLogger( __name__ ) + +class CompressedFile( object ): + + def __init__( self, file_path, mode='r' ): + if istar( file_path ): + self.file_type = 'tar' + elif iszip( file_path ) and not isjar( file_path ): + self.file_type = 'zip' + self.file_name = os.path.splitext( file_path )[ 0 ] + if self.file_name.endswith( '.tar' ): + self.file_name = os.path.splitext( self.file_name )[ 0 ] + self.type = self.file_type + method = 'open_%s' % self.file_type + if hasattr( self, method ): + self.archive = getattr( self, method )( file_path, mode ) + else: + raise NameError( 'File type %s specified, no open method found.' % self.file_type ) + + def extract( self, path ): + full_path = self.extraction_path( path ) + self.archive.extractall( full_path ) + return full_path + + def extraction_path( self, path ): + '''Determine the path to which the archive should be extracted.''' + contents = self.getmembers() + extraction_path = path + if len( contents ) == 1: + # The archive contains a single file, return the extraction path. + if self.isfile( contents[ 0 ] ): + extraction_path = os.path.join( path, self.file_name ) + else: + # Sort filenames within the archive by length, because if the shortest path entry in the archive is a directory, + # and the next entry has the directory prepended, then everything following that entry must be within that directory. + # For example, consider tarball for BWA 0.5.9: + # bwa-0.5.9.tar.bz2: + # bwa-0.5.9/ + # bwa-0.5.9/bwt.c + # bwa-0.5.9/bwt.h + # bwa-0.5.9/Makefile + # bwa-0.5.9/bwt_gen/ + # bwa-0.5.9/bwt_gen/Makefile + # bwa-0.5.9/bwt_gen/bwt_gen.c + # bwa-0.5.9/bwt_gen/bwt_gen.h + # When sorted by length, one sees a directory at the root of the tarball, and all other tarball contents as + # children of that directory. + filenames = sorted( [ self.getname( item ) for item in contents ], cmp=lambda a,b: cmp( len( a ), len( b ) ) ) + parent_name = filenames[ 0 ] + parent = self.getmember( parent_name ) + first_child = filenames[ 1 ] + if first_child.startswith( parent_name ) and parent is not None and self.isdir( parent ): + extraction_path = os.path.join( path, self.getname( parent ) ) + else: + extraction_path = os.path.join( path, self.file_name ) + if not os.path.exists( extraction_path ): + os.makedirs( extraction_path ) + return os.path.abspath( extraction_path ) + + def getmembers_tar( self ): + return self.archive.getmembers() + + def getmembers_zip( self ): + return self.archive.infolist() + + def getname_tar( self, item ): + return item.name + + def getname_zip( self, item ): + return item.filename + + def getmember( self, name ): + for member in self.getmembers(): + if self.getname( member ) == name: + return member + + def getmembers( self ): + return getattr( self, 'getmembers_%s' % self.type )() + + def getname( self, member ): + return getattr( self, 'getname_%s' % self.type )( member ) + + def isdir( self, member ): + return getattr( self, 'isdir_%s' % self.type )( member ) + + def isdir_tar( self, member ): + return member.isdir() + + def isdir_zip( self, member ): + if member.filename.endswith( os.sep ): + return True + return False + + def isfile( self, member ): + if not self.isdir( member ): + return True + return False + + def open_tar( self, filepath, mode ): + return tarfile.open( filepath, mode ) + + def open_zip( self, filepath, mode ): + return zipfile.ZipFile( filepath, mode ) + def clean_tool_shed_url( base_url ): if base_url: if base_url.find( '://' ) > -1: @@ -92,30 +195,6 @@ dir = url_download( work_dir, downloaded_filename, url, extract=False ) return downloaded_filename -def extract_tar( file_name, file_path ): - if isgzip( file_name ) or isbz2( file_name ): - # Open for reading with transparent compression. - tar = tarfile.open( file_name, 'r:*', errorlevel=0 ) - else: - tar = tarfile.open( file_name, errorlevel=0 ) - tar.extractall( path=file_path ) - tar.close() - -def extract_zip( archive_path, extraction_path ): - # TODO: change this method to use zipfile.Zipfile.extractall() when we stop supporting Python 2.5. - if not zipfile_ok( archive_path ): - return False - zip_archive = zipfile.ZipFile( archive_path, 'r' ) - for name in zip_archive.namelist(): - uncompressed_path = os.path.join( extraction_path, name ) - if uncompressed_path.endswith( '/' ): - if not os.path.isdir( uncompressed_path ): - os.makedirs( uncompressed_path ) - else: - file( uncompressed_path, 'wb' ).write( zip_archive.read( name ) ) - zip_archive.close() - return True - def format_traceback(): ex_type, ex, tb = sys.exc_info() return ''.join( traceback.format_tb( tb ) ) @@ -232,12 +311,18 @@ destination_file = os.path.join( destination_directory, file_name ) shutil.move( source_file, destination_file ) -def move_file( current_dir, source, destination_dir ): +def move_file( current_dir, source, destination, rename_to=None ): source_file = os.path.abspath( os.path.join( current_dir, source ) ) - destination_directory = os.path.join( destination_dir ) - if not os.path.isdir( destination_directory ): + if rename_to is not None: + destination_file = rename_to + destination_directory = os.path.join( destination ) + destination_path = os.path.join( destination_directory, destination_file ) + else: + destination_directory = os.path.join( destination ) + destination_path = os.path.join( destination_directory, source_file ) + if not os.path.exists( destination_directory ): os.makedirs( destination_directory ) - shutil.move( source_file, destination_directory ) + shutil.move( source_file, destination_path ) def parse_package_elem( package_elem, platform_info_dict=None, include_after_install_actions=True ): """ @@ -340,19 +425,6 @@ continue return actions_elem_tuples -def tar_extraction_directory( file_path, file_name ): - """Try to return the correct extraction directory.""" - file_name = file_name.strip() - extensions = [ '.tar.gz', '.tgz', '.tar.bz2', '.tar', '.zip' ] - for extension in extensions: - if file_name.find( extension ) > 0: - dir_name = file_name[ :-len( extension ) ] - if os.path.exists( os.path.abspath( os.path.join( file_path, dir_name ) ) ): - return dir_name - if os.path.exists( os.path.abspath( os.path.join( file_path, file_name ) ) ): - return os.path.abspath( file_path ) - raise ValueError( 'Could not find path to file %s' % os.path.abspath( os.path.join( file_path, file_name ) ) ) - def url_download( install_dir, downloaded_file_name, download_url, extract=True ): file_path = os.path.join( install_dir, downloaded_file_name ) src = None @@ -374,32 +446,14 @@ if dst: dst.close() if extract: - if istar( file_path ): - # <action type="download_by_url">http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2</action> - extract_tar( file_path, install_dir ) - dir = tar_extraction_directory( install_dir, downloaded_file_name ) - elif isjar( file_path ): - dir = os.path.curdir - elif iszip( file_path ): - # <action type="download_by_url">http://downloads.sourceforge.net/project/picard/picard-tools/1.56/picard-tools-1.56.zip</action> - zip_archive_extracted = extract_zip( file_path, install_dir ) - dir = zip_extraction_directory( install_dir, downloaded_file_name ) + if istar( file_path ) or iszip( file_path ): + archive = CompressedFile( file_path ) + extraction_path = archive.extract( install_dir ) else: - dir = os.path.abspath( install_dir ) + extraction_path = os.path.abspath( install_dir ) else: - dir = os.path.abspath( install_dir ) - return dir - -def zip_extraction_directory( file_path, file_name ): - """Try to return the correct extraction directory.""" - files = [ filename for filename in os.listdir( file_path ) if not filename.endswith( '.zip' ) ] - if len( files ) > 1: - return os.path.abspath( file_path ) - elif len( files ) == 1: - # If there is only on file it should be a directory. - if os.path.isdir( os.path.join( file_path, files[ 0 ] ) ): - return os.path.abspath( os.path.join( file_path, files[ 0 ] ) ) - raise ValueError( 'Could not find directory for the extracted file %s' % os.path.abspath( os.path.join( file_path, file_name ) ) ) + extraction_path = os.path.abspath( install_dir ) + return extraction_path def zipfile_ok( path_to_archive ): """ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.