1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6e3602150252/ Changeset: 6e3602150252 User: Dave Bouvier Date: 2013-08-01 17:05:34 Summary: Implement baseline support for defining a tool dependency that downloads a precompiled binary for the target platform. Affected #: 4 files diff -r fe5867b89ec01bfde18591df8809526d85fb821e -r 6e36021502522213aca87e0b49f0bf0384eddfa1 lib/tool_shed/galaxy_install/tool_dependencies/common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py @@ -8,6 +8,7 @@ import zipfile import tool_shed.util.shed_util_common as suc from galaxy.datatypes import checkers +from urllib2 import HTTPError log = logging.getLogger( __name__ ) @@ -70,6 +71,23 @@ __shellquote(env_shell_file_path)) return cmd +def download_binary_from_url( url, work_dir, install_dir ): + ''' + Download a pre-compiled binary from the specified URL. If the downloaded file is an archive, + extract it into install_dir and delete the archive. + ''' + downloaded_filename = os.path.split( url )[ -1 ] + try: + dir = url_download( work_dir, downloaded_filename, url, extract=True ) + downloaded_filepath = os.path.join( work_dir, downloaded_filename ) + if is_compressed( downloaded_filepath ): + os.remove( downloaded_filepath ) + move_directory_files( current_dir=work_dir, + source_dir=dir, + destination_dir=install_dir ) + return True + except HTTPError: + return False def extract_tar( file_name, file_path ): if isgzip( file_name ) or isbz2( file_name ): @@ -190,6 +208,12 @@ def iszip( file_path ): return checkers.check_zip( file_path ) +def is_compressed( file_path ): + if isjar( file_path ): + return False + else: + return iszip( file_path ) or isgzip( file_path ) or istar( file_path ) or isbz2( file_path ) + def make_directory( full_path ): if not os.path.exists( full_path ): os.makedirs( full_path ) diff -r fe5867b89ec01bfde18591df8809526d85fb821e -r 6e36021502522213aca87e0b49f0bf0384eddfa1 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -165,12 +165,29 @@ actions = actions_dict.get( 'actions', None ) filtered_actions = [] env_shell_file_paths = [] + # Default to false so that the install process will default to compiling. + binary_found = False if actions: with make_tmp_dir() as work_dir: with lcd( work_dir ): # The first action in the list of actions will be the one that defines the installation process. There # are currently only two supported processes; download_by_url and clone via a "shell_command" action type. action_type, action_dict = actions[ 0 ] + if action_type == 'download_binary': + # Eliminate the download_binary action so remaining actions can be processed correctly. + filtered_actions = actions[ 1: ] + url = action_dict[ 'url' ] + # Attempt to download a binary from the specified URL. + log.debug( 'Attempting to download from %s', url ) + binary_found = common_util.download_binary_from_url( url, work_dir, install_dir ) + if binary_found: + # If the attempt succeeded, set the action_type to binary_found, in order to skip any download_by_url or shell_command actions. + actions = filtered_actions + action_type = 'binary_found' + else: + # No binary exists, or there was an error downloading the binary from the generated URL. Proceed with the remaining actions. + del actions[ 0 ] + action_type, action_dict = actions[ 0 ] if action_type == 'download_by_url': # Eliminate the download_by_url action so remaining actions can be processed correctly. filtered_actions = actions[ 1: ] @@ -220,6 +237,9 @@ current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) with lcd( current_dir ): action_type, action_dict = action_tup + # If a binary was found, we only need to process environment variables, file permissions, and any other binary downloads. + if binary_found and action_type not in [ 'set_environment', 'chmod', 'download_binary' ]: + continue if action_type == 'make_directory': common_util.make_directory( full_path=action_dict[ 'full_path' ] ) elif action_type == 'move_directory_files': @@ -348,6 +368,18 @@ dir = target_directory.replace( os.path.realpath( work_dir ), '' ).lstrip( '/' ) else: log.error( 'Invalid or nonexistent directory %s specified, ignoring change_directory action.', target_directory ) + elif action_type == 'chmod': + for target_file, mode in action_dict[ 'change_modes' ]: + if os.path.exists( target_file ): + os.chmod( target_file, mode ) + elif action_type == 'download_binary': + url = action_dict[ 'url' ] + binary_found = common_util.download_binary_from_url( url, work_dir, install_dir ) + if binary_found: + log.debug( 'Successfully downloaded binary from %s', url ) + else: + log.error( 'Unable to download binary from %s', url ) + def log_results( command, fabric_AttributeString, file_path ): """ diff -r fe5867b89ec01bfde18591df8809526d85fb821e -r 6e36021502522213aca87e0b49f0bf0384eddfa1 lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -1,6 +1,7 @@ import logging import os import sys +import stat import subprocess import tempfile from string import Template @@ -379,7 +380,22 @@ for action_elem in actions_elem.findall( 'action' ): action_dict = {} action_type = action_elem.get( 'type', 'shell_command' ) - if action_type == 'shell_command': + if action_type == 'download_binary': + platform_info_dict = tool_dependency_util.get_platform_info_dict() + platform_info_dict[ 'name' ] = tool_dependency.name + platform_info_dict[ 'version' ] = tool_dependency.version + url_template_elems = action_elem.findall( 'url_template' ) + # Check if there are multiple url_template elements, each with attrib entries for a specific platform. + if len( url_template_elems ) > 1: + # <base_url os="darwin" extract="false">http://hgdownload.cse.ucsc.edu/admin/exe/macOSX.${architecture}/faToTwoBit</base_url> + # This method returns the url_elem that best matches the current platform as received from os.uname(). + # Currently checked attributes are os and architecture. + # These correspond to the values sysname and processor from the Python documentation for os.uname(). + url_template_elem = tool_dependency_util.get_download_url_for_platform( url_template_elems, platform_info_dict ) + else: + url_template_elem = url_template_elems[ 0 ] + action_dict[ 'url' ] = Template( url_template_elem.text ).safe_substitute( platform_info_dict ) + elif action_type == 'shell_command': # <action type="shell_command">make</action> action_elem_text = evaluate_template( action_elem.text ) if action_elem_text: @@ -492,6 +508,27 @@ # lxml==2.3.0</action> ## Manually specify contents of requirements.txt file to create dynamically. action_dict[ 'requirements' ] = evaluate_template( action_elem.text or 'requirements.txt' ) + elif action_type == 'chmod': + # Change the read, write, and execute bits on a file. + file_elems = action_elem.findall( 'file' ) + chmod_actions = [] + # A unix octal mode is the sum of the following values: + # Owner: + # 400 Read 200 Write 100 Execute + # Group: + # 040 Read 020 Write 010 Execute + # World: + # 004 Read 002 Write 001 Execute + for file_elem in file_elems: + # So by the above table, owner read/write/execute and group read permission would be 740. + # Python's os.chmod uses base 10 modes, convert received unix-style octal modes to base 10. + received_mode = int( file_elem.get( 'mode', 600 ), base=8 ) + # For added security, ensure that the setuid and setgid bits are not set. + mode = received_mode & ~( stat.S_ISUID | stat.S_ISGID ) + file = evaluate_template( file_elem.text ) + chmod_tuple = ( file, mode ) + chmod_actions.append( chmod_tuple ) + action_dict[ 'change_modes' ] = chmod_actions else: log.debug( "Unsupported action type '%s'. Not proceeding." % str( action_type ) ) raise Exception( "Unsupported action type '%s' in tool dependency definition." % str( action_type ) ) diff -r fe5867b89ec01bfde18591df8809526d85fb821e -r 6e36021502522213aca87e0b49f0bf0384eddfa1 lib/tool_shed/util/tool_dependency_util.py --- a/lib/tool_shed/util/tool_dependency_util.py +++ b/lib/tool_shed/util/tool_dependency_util.py @@ -39,6 +39,37 @@ tool_dependencies[ dependency_key ] = requirements_dict return tool_dependencies +def get_download_url_for_platform( url_templates, platform_info_dict ): + ''' + Compare the dict returned by get_platform_info() with the values specified in the base_url element. Return + true if and only if all defined attributes match the corresponding dict entries. If an entry is not + defined in the base_url element, it is assumed to be irrelevant at this stage. For example, + <base_url os="darwin">http://hgdownload.cse.ucsc.edu/admin/exe/macOSX.${architecture}/faToTwoBit</base_url> + where the OS must be 'darwin', but the architecture is filled in later using string.Template. + ''' + os_ok = False + architecture_ok = False + for url_template in url_templates: + os_name = url_template.get( 'os', None ) + architecture = url_template.get( 'architecture', None ) + if os_name: + if os_name.lower() == platform_info_dict[ 'os' ]: + os_ok = True + else: + os_ok = False + else: + os_ok = True + if architecture: + if architecture.lower() == platform_info_dict[ 'architecture' ]: + architecture_ok = True + else: + architecture_ok = False + else: + architecture_ok = True + if os_ok and architecture_ok: + return url_template + return None + def create_or_update_tool_dependency( app, tool_shed_repository, name, version, type, status, set_status=True ): # Called from Galaxy (never the tool shed) when a new repository is being installed or when an uninstalled repository is being reinstalled. sa_session = app.model.context.current @@ -204,6 +235,14 @@ missing_tool_dependencies = None return tool_dependencies, missing_tool_dependencies +def get_platform_info_dict(): + '''Return a dict with information about the current platform.''' + platform_dict = {} + sysname, nodename, release, version, machine = os.uname() + platform_dict[ 'os' ] = sysname.lower() + platform_dict[ 'architecture' ] = machine.lower() + return platform_dict + def get_tool_dependency( trans, id ): """Get a tool_dependency from the database via id""" return trans.sa_session.query( trans.model.ToolDependency ).get( trans.security.decode_id( id ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.