[galaxy-commits] commit/galaxy-central: greg: Enhance the tool dependency package installation framework's InstallEnvironment object to have pointers to the root installation directory of both the repository and the tool dependency package. Enhance the prepare_step()_ functions in the various tool dependency package installation classes to receive the install_environment rather than just the package install_dir. This allows these functions to substitute reserved words like $REPOSITORY_INSTALL_DIR and $INSTA

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/5582e58fa7a2/ Changeset: 5582e58fa7a2 User: greg Date: 2014-04-30 21:49:31 Summary: Enhance the tool dependency package installation framework's InstallEnvironment object to have pointers to the root installation directory of both the repository and the tool dependency package. Enhance the prepare_step()_ functions in the various tool dependency package installation classes to receive the install_environment rather than just the package install_dir. This allows these functions to substitute reserved words like $REPOSITORY_INSTALL_DIR and $INSTALL_DIR used in recipes with the values of these paths retrieved from the install_environment. These reserved words can now hopefully be safely used in any action type, including "move_file" and "move_directory". Affected #: 5 files diff -r c05ab2ae3a58938e07c33193a7a52a89cbd43056 -r 5582e58fa7a27364f25cd66f54f373ed2e108a11 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -24,14 +24,19 @@ if int( version.split( "." )[ 0 ] ) < 1: raise NotImplementedError( "Install Fabric version 1.0 or later." ) -def install_and_build_package( app, tool_dependency, actions_dict ): +def get_tool_shed_repository_install_dir( app, tool_shed_repository ): + return os.path.abspath( tool_shed_repository.repo_files_directory( app ) ) + +def install_and_build_package( app, tool_shed_repository, tool_dependency, actions_dict ): """Install a Galaxy tool dependency package either via a url or a mercurial or git clone command.""" + tool_shed_repository_install_dir = get_tool_shed_repository_install_dir( app, tool_shed_repository ) install_dir = actions_dict[ 'install_dir' ] package_name = actions_dict[ 'package_name' ] actions = actions_dict.get( 'actions', None ) filtered_actions = [] env_file_builder = EnvFileBuilder( install_dir ) - install_environment = InstallEnvironment() + install_environment = InstallEnvironment( tool_shed_repository_install_dir=tool_shed_repository_install_dir, + install_dir=install_dir ) recipe_manager = RecipeManager() if actions: with install_environment.make_tmp_dir() as work_dir: @@ -58,7 +63,6 @@ env_file_builder=env_file_builder, install_environment=install_environment, work_dir=work_dir, - install_dir=install_dir, current_dir=None, initial_download=True ) else: @@ -85,7 +89,6 @@ env_file_builder=env_file_builder, install_environment=install_environment, work_dir=work_dir, - install_dir=install_dir, current_dir=current_dir, initial_download=False ) if tool_dependency.status in [ app.install_model.ToolDependency.installation_status.ERROR ]: diff -r c05ab2ae3a58938e07c33193a7a52a89cbd43056 -r 5582e58fa7a27364f25cd66f54f373ed2e108a11 lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -8,6 +8,7 @@ import fabric_util import td_common_util from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import EnvFileBuilder +from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import InstallEnvironment from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import RecipeManager import tool_shed.util.shed_util_common as suc from tool_shed.util import common_util @@ -161,9 +162,6 @@ return tool_shed_repository return None -def get_tool_shed_repository_install_dir( app, tool_shed_repository ): - return os.path.abspath( tool_shed_repository.repo_files_directory( app ) ) - def get_updated_changeset_revisions_from_tool_shed( app, tool_shed_url, name, owner, changeset_revision ): """ Get all appropriate newer changeset revisions for the repository defined by @@ -291,11 +289,11 @@ raise Exception( message ) return handled_tool_dependencies -def install_and_build_package_via_fabric( app, tool_dependency, actions_dict ): +def install_and_build_package_via_fabric( app, tool_shed_repository, tool_dependency, actions_dict ): sa_session = app.install_model.context try: # There is currently only one fabric method. - tool_dependency = fabric_util.install_and_build_package( app, tool_dependency, actions_dict ) + tool_dependency = fabric_util.install_and_build_package( app, tool_shed_repository, tool_dependency, actions_dict ) except Exception, e: log.exception( 'Error installing tool dependency %s version %s.', str( tool_dependency.name ), str( tool_dependency.version ) ) # Since there was an installation error, update the tool dependency status to Error. The remove_installation_path option must @@ -369,13 +367,14 @@ can_install_tool_dependency = True if can_install_tool_dependency: package_install_version = package_elem.get( 'version', '1.0' ) + status = app.install_model.ToolDependency.installation_status.INSTALLING tool_dependency = \ tool_dependency_util.create_or_update_tool_dependency( app=app, tool_shed_repository=tool_shed_repository, name=package_name, version=package_version, type='package', - status=app.install_model.ToolDependency.installation_status.INSTALLING, + status=status, set_status=True ) # Get the information about the current platform in case the tool dependency definition includes tag sets # for installing compiled binaries. @@ -412,11 +411,12 @@ if binary_installed: continue # No platform-specific <actions> recipe has yet resulted in a successful installation. - tool_dependency = install_via_fabric( app, - tool_dependency, - install_dir, - package_name=package_name, - actions_elem=actions_elem, + tool_dependency = install_via_fabric( app, + tool_shed_repository, + tool_dependency, + install_dir, + package_name=package_name, + actions_elem=actions_elem, action_elem=None ) if tool_dependency.status == app.install_model.ToolDependency.installation_status.INSTALLED: # If an <actions> tag was found that matches the current platform, and the @@ -453,31 +453,34 @@ # tag set that defines the recipe to install and compile from source. log.debug( 'Proceeding with install and compile recipe for tool dependency %s.' % \ str( tool_dependency.name ) ) - tool_dependency = install_via_fabric( app, - tool_dependency, - install_dir, - package_name=package_name, - actions_elem=actions_elem, + tool_dependency = install_via_fabric( app, + tool_shed_repository, + tool_dependency, + install_dir, + package_name=package_name, + actions_elem=actions_elem, action_elem=None ) if actions_elem.tag == 'action' and tool_dependency.status != app.install_model.ToolDependency.installation_status.ERROR: # If the tool dependency is not in an error state, perform any final actions that have been # defined within the actions_group tag set, but outside of an <actions> tag, which defines # the recipe for installing and compiling from source. - tool_dependency = install_via_fabric( app, - tool_dependency, - install_dir, - package_name=package_name, - actions_elem=None, + tool_dependency = install_via_fabric( app, + tool_shed_repository, + tool_dependency, + install_dir, + package_name=package_name, + actions_elem=None, action_elem=actions_elem ) else: # Checks for "os" and "architecture" attributes are not made for any <actions> tag sets outside of # an <actions_group> tag set. If the attributes are defined, they will be ignored. All <actions> tags # outside of an <actions_group> tag set will always be processed. - tool_dependency = install_via_fabric( app, - tool_dependency, - install_dir, - package_name=package_name, - actions_elem=actions_elems, + tool_dependency = install_via_fabric( app, + tool_shed_repository, + tool_dependency, + install_dir, + package_name=package_name, + actions_elem=actions_elems, action_elem=None ) if tool_dependency.status != app.install_model.ToolDependency.installation_status.ERROR: log.debug( 'Tool dependency %s version %s has been installed in %s.' % \ @@ -509,7 +512,7 @@ # print 'Installing tool dependencies via fabric script ', custom_fabfile_path return tool_dependency -def install_via_fabric( app, tool_dependency, install_dir, package_name=None, custom_fabfile_path=None, +def install_via_fabric( app, tool_shed_repository, tool_dependency, install_dir, package_name=None, custom_fabfile_path=None, actions_elem=None, action_elem=None, **kwd ): """ Parse a tool_dependency.xml file's <actions> tag set to gather information for installation using the @@ -534,25 +537,28 @@ else: elems = [] recipe_manager = RecipeManager() + tool_shed_repository_install_dir = fabric_util.get_tool_shed_repository_install_dir( app, tool_shed_repository ) + install_environment = InstallEnvironment( tool_shed_repository_install_dir, install_dir ) for action_elem in elems: # Make sure to skip all comments, since they are now included in the XML tree. if action_elem.tag != 'action': continue action_dict = {} - action_type = action_elem.get( 'type', 'shell_command' ) - action_dict = recipe_manager.prepare_step( app=app, - tool_dependency=tool_dependency, - action_type=action_type, - action_elem=action_elem, - action_dict=action_dict, - install_dir=install_dir, - is_binary_download=is_binary_download ) - action_tuple = ( action_type, action_dict ) - if action_type == 'set_environment': - if action_tuple not in actions: + action_type = action_elem.get( 'type', None ) + if action_type is not None: + action_dict = recipe_manager.prepare_step( app=app, + tool_dependency=tool_dependency, + action_type=action_type, + action_elem=action_elem, + action_dict=action_dict, + install_environment=install_environment, + is_binary_download=is_binary_download ) + action_tuple = ( action_type, action_dict ) + if action_type == 'set_environment': + if action_tuple not in actions: + actions.append( action_tuple ) + else: actions.append( action_tuple ) - else: - actions.append( action_tuple ) if actions: actions_dict[ 'actions' ] = actions if custom_fabfile_path is not None: @@ -560,7 +566,7 @@ # execute_custom_fabric_script( app, elem, custom_fabfile_path, install_dir, package_name=package_name ) raise Exception( 'Tool dependency installation using proprietary fabric scripts is not yet supported.' ) else: - tool_dependency = install_and_build_package_via_fabric( app, tool_dependency, actions_dict ) + tool_dependency = install_and_build_package_via_fabric( app, tool_shed_repository, tool_dependency, actions_dict ) return tool_dependency def execute_custom_fabric_script( app, elem, custom_fabfile_path, install_dir, package_name=None, **kwd ): @@ -622,8 +628,9 @@ def set_environment( app, elem, tool_shed_repository, attr_tups_of_dependencies_for_install ): """ - Create a ToolDependency to set an environment variable. This is different from the process used to set an environment variable that is associated - with a package. An example entry in a tool_dependencies.xml file is:: + Create a ToolDependency to set an environment variable. This is different from the process used to + set an environment variable that is associated with a package. An example entry in a tool_dependencies.xml + file is:: <set_environment version="1.0"><environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> @@ -655,8 +662,10 @@ tool_dependency_type='set_environment', tool_dependency_name=env_var_name, tool_dependency_version=None ) - tool_shed_repository_install_dir = get_tool_shed_repository_install_dir( app, tool_shed_repository ) - env_var_dict = td_common_util.create_env_var_dict( env_var_elem, tool_shed_repository_install_dir=tool_shed_repository_install_dir ) + tool_shed_repository_install_dir = fabric_util.get_tool_shed_repository_install_dir( app, tool_shed_repository ) + env_var_dict = td_common_util.create_env_var_dict( elem=env_var_elem, + tool_dependency_install_dir=install_dir, + tool_shed_repository_install_dir=tool_shed_repository_install_dir ) if env_var_dict: if not os.path.exists( install_dir ): os.makedirs( install_dir ) diff -r c05ab2ae3a58938e07c33193a7a52a89cbd43056 -r 5582e58fa7a27364f25cd66f54f373ed2e108a11 lib/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py @@ -153,11 +153,18 @@ """Object describing the environment built up as part of the process of building and installing a package.""" - def __init__( self ): + def __init__( self, tool_shed_repository_install_dir, install_dir ): + """ + The value of the received tool_shed_repository_install_dir is the root installation directory + of the repository containing the tool dependency, and the value of the received install_dir is + the root installation directory of the tool dependency. + """ self.env_shell_file_paths = [] + self.install_dir = install_dir + self.tool_shed_repository_install_dir = tool_shed_repository_install_dir - def __call__( self, install_dir ): - with settings( warn_only=True, **td_common_util.get_env_var_values( install_dir ) ): + def __call__( self ): + with settings( warn_only=True, **td_common_util.get_env_var_values( self ) ): with prefix( self.__setup_environment() ): yield @@ -233,12 +240,12 @@ log.debug( 'Invalid file %s specified, ignoring template_command action.' % str( env_shell_file_path ) ) return env_vars - def handle_command( self, app, tool_dependency, install_dir, cmd, return_output=False ): + def handle_command( self, app, tool_dependency, cmd, return_output=False ): """Handle a command and log the results.""" context = app.install_model.context command = str( cmd ) output = self.handle_complex_command( command ) - self.log_results( cmd, output, os.path.join( install_dir, td_common_util.INSTALLATION_LOG ) ) + self.log_results( cmd, output, os.path.join( self.install_dir, td_common_util.INSTALLATION_LOG ) ) stdout = output.stdout stderr = output.stderr if len( stdout ) > DATABASE_MAX_STRING_SIZE: @@ -395,7 +402,7 @@ return self.step_handlers_by_type.get( type, None ) def execute_step( self, app, tool_dependency, package_name, actions, action_type, action_dict, filtered_actions, - env_file_builder, install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + env_file_builder, install_environment, work_dir, current_dir=None, initial_download=False ): if actions: step_handler = self.get_step_handler_by_type( action_type ) tool_dependency, filtered_actions, dir = step_handler.execute_step( app=app, @@ -407,7 +414,6 @@ env_file_builder=env_file_builder, install_environment=install_environment, work_dir=work_dir, - install_dir=install_dir, current_dir=current_dir, initial_download=initial_download ) else: @@ -439,7 +445,7 @@ template_command=step_handler.TemplateCommand() ) return step_handlers_by_type - def prepare_step( self, app, tool_dependency, action_type, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_type, action_elem, action_dict, install_environment, is_binary_download ): """ Prepare the recipe step for later execution. This generally alters the received action_dict with new information needed during this step's execution. @@ -450,7 +456,7 @@ tool_dependency=tool_dependency, action_elem=action_elem, action_dict=action_dict, - install_dir=install_dir, + install_environment=install_environment, is_binary_download=is_binary_download ) return action_dict diff -r c05ab2ae3a58938e07c33193a7a52a89cbd43056 -r 5582e58fa7a27364f25cd66f54f373ed2e108a11 lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -30,10 +30,10 @@ """Abstract class that defines a standard format for handling recipe steps when installing packages.""" def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): raise "Unimplemented Method" - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): raise "Unimplemented Method" @@ -43,7 +43,7 @@ self.type = 'assert_directory_executable' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Make sure a symbolic link or directory on disk exists and is executable, but is not a file. Since this class is not used in the initial download stage, no recipe step filtering is @@ -63,10 +63,10 @@ remove_from_disk=False ) return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="assert_executable">$INSTALL_DIR/mira/my_file</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_dir ) + action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -76,7 +76,7 @@ self.type = 'assert_directory_exists' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Make sure a a symbolic link or directory on disk exists, but is not a file. Since this class is not used in the initial download stage, no recipe step filtering is performed @@ -96,10 +96,10 @@ remove_from_disk=False ) return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="make_directory">$INSTALL_DIR/mira</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_dir ) + action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -109,7 +109,7 @@ self.type = 'assert_file_executable' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Make sure a symbolic link or file on disk exists and is executable, but is not a directory. Since this class is not used in the initial download stage, no recipe step filtering is @@ -129,10 +129,10 @@ remove_from_disk=False ) return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="assert_executable">$INSTALL_DIR/mira/my_file</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_dir ) + action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -142,7 +142,7 @@ self.type = 'assert_file_exists' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Make sure a symbolic link or file on disk exists, but is not a directory. Since this class is not used in the initial download stage, no recipe step filtering is performed @@ -162,10 +162,10 @@ remove_from_disk=False ) return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="assert_on_path">$INSTALL_DIR/mira/my_file</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_dir ) + action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -175,7 +175,7 @@ self.type = 'autoconf' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Handle configure, make and make install in a shell, allowing for configuration options. Since this class is not used in the initial download stage, no recipe step filtering is performed here, and None @@ -187,16 +187,19 @@ pre_cmd = './configure %s && make && make install' % configure_opts else: pre_cmd = './configure --prefix=$INSTALL_DIR %s && make && make install' % configure_opts - cmd = install_environment.build_command( td_common_util.evaluate_template( pre_cmd, install_dir ) ) - return_code = install_environment.handle_command( app, tool_dependency, install_dir, cmd ) + cmd = install_environment.build_command( td_common_util.evaluate_template( pre_cmd, install_environment ) ) + return_code = install_environment.handle_command( app=app, + tool_dependency=tool_dependency, + cmd=cmd, + return_output=False ) # The caller should check the status of the returned tool_dependency since this function # does nothing with the return_code. return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # Handle configure, make and make install allow providing configuration options if action_elem.text: - configure_opts = td_common_util.evaluate_template( action_elem.text, install_dir ) + configure_opts = td_common_util.evaluate_template( action_elem.text, install_environment ) action_dict[ 'configure_opts' ] = configure_opts return action_dict @@ -207,7 +210,7 @@ self.type = 'change_directory' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Change the working directory in a shell. Since this class is not used in the initial download stage, no recipe step filtering is performed here and a None value is return for filtered_actions. However, @@ -225,7 +228,7 @@ dir = current_dir return tool_dependency, None, dir - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="change_directory">PHYLIP-3.6b</action> if action_elem.text: action_dict[ 'directory' ] = action_elem.text @@ -238,7 +241,7 @@ self.type = 'chmod' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Change the mode setting for certain files in the installation environment. Since this class is not used in the initial download stage, no recipe step filtering is performed here, and None values are @@ -251,7 +254,7 @@ log.debug( 'Invalid file %s specified, ignoring %s action.', target_file, action_type ) return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # Change the read, write, and execute bits on a file. # <action type="chmod"> # <file mode="750">$INSTALL_DIR/bin/faToTwoBit</file> @@ -271,7 +274,7 @@ received_mode = int( file_elem.get( 'mode', 600 ), base=8 ) # For added security, ensure that the setuid and setgid bits are not set. mode = received_mode & ~( stat.S_ISUID | stat.S_ISGID ) - file = td_common_util.evaluate_template( file_elem.text, install_dir ) + file = td_common_util.evaluate_template( file_elem.text, install_environment ) chmod_tuple = ( file, mode ) chmod_actions.append( chmod_tuple ) if chmod_actions: @@ -294,7 +297,7 @@ return filtered_actions def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Download a binary file. If the value of initial_download is True, the recipe steps will be filtered and returned and the installation directory (i.e., dir) will be defined and returned. @@ -326,14 +329,15 @@ # any errors in the move step are correctly sent to the tool dependency error handler. if downloaded_filename and os.path.exists( os.path.join( work_dir, downloaded_filename ) ): if target_directory: - target_directory = os.path.realpath( os.path.normpath( os.path.join( install_dir, target_directory ) ) ) + target_directory = os.path.realpath( os.path.normpath( os.path.join( install_environment.install_dir, + target_directory ) ) ) # Make sure the target directory is not outside of $INSTALL_DIR. - if target_directory.startswith( os.path.realpath( install_dir ) ): - full_path_to_dir = os.path.abspath( os.path.join( install_dir, target_directory ) ) + if target_directory.startswith( os.path.realpath( install_environment.install_dir ) ): + full_path_to_dir = os.path.abspath( os.path.join( install_environment.install_dir, target_directory ) ) else: - full_path_to_dir = os.path.abspath( install_dir ) + full_path_to_dir = os.path.abspath( install_environment.install_dir ) else: - full_path_to_dir = os.path.abspath( install_dir ) + full_path_to_dir = os.path.abspath( install_environment.install_dir ) td_common_util.move_file( current_dir=work_dir, source=downloaded_filename, destination=full_path_to_dir ) @@ -343,7 +347,7 @@ return tool_dependency, filtered_actions, dir return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): platform_info_dict = tool_dependency_util.get_platform_info_dict() platform_info_dict[ 'name' ] = str( tool_dependency.name ) platform_info_dict[ 'version' ] = str( tool_dependency.version ) @@ -370,7 +374,7 @@ self.type = 'download_by_url' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Download a file via HTTP. If the value of initial_download is True, the recipe steps will be filtered and returned and the installation directory (i.e., dir) will be defined and returned. @@ -392,7 +396,7 @@ downloaded_filename = os.path.split( url )[ -1 ] dir = td_common_util.url_download( work_dir, downloaded_filename, url, extract=True ) if is_binary: - log_file = os.path.join( install_dir, td_common_util.INSTALLATION_LOG ) + log_file = os.path.join( install_environment.install_dir, td_common_util.INSTALLATION_LOG ) if os.path.exists( log_file ): logfile = open( log_file, 'ab' ) else: @@ -404,7 +408,7 @@ return tool_dependency, filtered_actions, dir return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="download_by_url"> # http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.... # </action> @@ -424,7 +428,7 @@ self.type = 'download_file' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Download a file. If the value of initial_download is True, the recipe steps will be filtered and returned and the installation directory (i.e., dir) will be defined and returned. @@ -449,7 +453,7 @@ return tool_dependency, filtered_actions, dir return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="download_file">http://effectors.org/download/version/TTSS_GUI-1.0.1.jar</action> if action_elem.text: action_dict[ 'url' ] = action_elem.text @@ -466,7 +470,7 @@ self.type = 'make_directory' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Make a directory on disk. Since this class is not used in the initial download stage, no recipe step filtering is performed here, and None values are always returned for filtered_actions and dir. @@ -478,10 +482,10 @@ td_common_util.make_directory( full_path=full_path ) return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="make_directory">$INSTALL_DIR/lib/python</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_dir ) + action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -491,7 +495,7 @@ self.type = 'make_install' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Execute a make_install command in a shell. Since this class is not used in the initial download stage, no recipe step filtering is performed here, and None values are always returned for filtered_actions and dir. @@ -500,15 +504,18 @@ with settings( warn_only=True ): make_opts = action_dict.get( 'make_opts', '' ) cmd = install_environment.build_command( 'make %s && make install' % make_opts ) - return_code = install_environment.handle_command( app, tool_dependency, install_dir, cmd ) + return_code = install_environment.handle_command( app=app, + tool_dependency=tool_dependency, + cmd=cmd, + return_output=False ) # The caller should check the status of the returned tool_dependency since this function # does nothing with the return_code. return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # make; make install; allow providing make options if action_elem.text: - make_opts = td_common_util.evaluate_template( action_elem.text, install_dir ) + make_opts = td_common_util.evaluate_template( action_elem.text, install_environment ) action_dict[ 'make_opts' ] = make_opts return action_dict @@ -519,7 +526,7 @@ self.type = 'move_directory_files' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Move a directory of files. Since this class is not used in the initial download stage, no recipe step filtering is performed here, and None values are always returned for filtered_actions and dir. @@ -529,13 +536,13 @@ destination_dir=os.path.join( action_dict[ 'destination_directory' ] ) ) return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="move_directory_files"> # <source_directory>bin</source_directory> # <destination_directory>$INSTALL_DIR/bin</destination_directory> # </action> for move_elem in action_elem: - move_elem_text = td_common_util.evaluate_template( move_elem.text, install_dir ) + move_elem_text = td_common_util.evaluate_template( move_elem.text, install_environment ) if move_elem_text: action_dict[ move_elem.tag ] = move_elem_text return action_dict @@ -547,7 +554,7 @@ self.type = 'move_file' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Move a file on disk. Since this class is not used in the initial download stage, no recipe step filtering is performed here, and None values are always returned for filtered_actions and dir. @@ -558,13 +565,13 @@ rename_to=action_dict[ 'rename_to' ] ) return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="move_file" rename_to="new_file_name"> # <source>misc/some_file</source> # <destination>$INSTALL_DIR/bin</destination> # </action> - action_dict[ 'source' ] = td_common_util.evaluate_template( action_elem.find( 'source' ).text, install_dir ) - action_dict[ 'destination' ] = td_common_util.evaluate_template( action_elem.find( 'destination' ).text, install_dir ) + action_dict[ 'source' ] = td_common_util.evaluate_template( action_elem.find( 'source' ).text, install_environment ) + action_dict[ 'destination' ] = td_common_util.evaluate_template( action_elem.find( 'destination' ).text, install_environment ) action_dict[ 'rename_to' ] = action_elem.get( 'rename_to' ) return action_dict @@ -575,7 +582,7 @@ self.type = 'set_environment' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Configure an install environment. Since this class is not used in the initial download stage, no recipe step filtering is performed here, and None values are always returned for filtered_actions @@ -583,13 +590,12 @@ """ # Currently the only action supported in this category is "environment_variable". cmds = install_environment.environment_commands( 'set_environment' ) - env_var_dicts = action_dict[ 'environment_variable' ] + env_var_dicts = action_dict.get( 'environment_variable', [] ) for env_var_dict in env_var_dicts: # Check for the presence of the $ENV[] key string and populate it if possible. env_var_dict = self.handle_environment_variables( app=app, install_environment=install_environment, tool_dependency=tool_dependency, - install_dir=install_dir, env_var_dict=env_var_dict, set_prior_environment_commands=cmds ) env_file_builder.append_line( **env_var_dict ) @@ -598,7 +604,7 @@ return_code = env_file_builder.return_code return tool_dependency, None, None - def handle_environment_variables( self, app, install_environment, tool_dependency, install_dir, env_var_dict, + def handle_environment_variables( self, app, install_environment, tool_dependency, env_var_dict, set_prior_environment_commands ): """ This method works with with a combination of three tool dependency definition tag sets, which are defined @@ -681,7 +687,10 @@ set_prior_environment_commands.append( 'echo %s: $%s' % ( inherited_env_var_name, inherited_env_var_name ) ) command = ' ; '.join( set_prior_environment_commands ) # Run the command and capture the output. - command_return = install_environment.handle_command( app, tool_dependency, install_dir, command, return_output=True ) + command_return = install_environment.handle_command( app=app, + tool_dependency=tool_dependency, + cmd=command, + return_output=True ) # And extract anything labeled with the name of the environment variable we're populating here. if '%s: ' % inherited_env_var_name in command_return: environment_variable_value = command_return.split( '\n' ) @@ -697,7 +706,7 @@ env_var_dict[ 'value' ] = env_var_value return env_var_dict - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="set_environment"> # <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable> # <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable> @@ -705,7 +714,10 @@ env_var_dicts = [] for env_elem in action_elem: if env_elem.tag == 'environment_variable': - env_var_dict = td_common_util.create_env_var_dict( env_elem, tool_dependency_install_dir=install_dir ) + env_var_dict = \ + td_common_util.create_env_var_dict( elem=env_elem, + tool_dependency_install_dir=install_environment.install_dir, + tool_shed_repository_install_dir=install_environment.tool_shed_repository_install_dir ) if env_var_dict: env_var_dicts.append( env_var_dict ) if env_var_dicts: @@ -720,7 +732,7 @@ self.type = 'set_environment_for_install' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Configure an environment for compiling a package. Since this class is not used in the initial download stage, no recipe step filtering is performed here, and None values are always returned @@ -730,10 +742,11 @@ # dependency env.sh files, the environment setting in each of which will be injected into the # environment for all <action type="shell_command"> tags that follow this # <action type="set_environment_for_install"> tag set in the tool_dependencies.xml file. - install_environment.add_env_shell_file_paths( action_dict[ 'env_shell_file_paths' ] ) + env_shell_file_paths = action_dict.get( 'env_shell_file_paths', [] ) + install_environment.add_env_shell_file_paths( env_shell_file_paths ) return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="set_environment_for_install"> # <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056"> # <package name="numpy" version="1.7.1" /> @@ -751,8 +764,7 @@ env_shell_file_paths = td_common_util.get_env_shell_file_paths( app, env_elem ) if env_shell_file_paths: all_env_shell_file_paths.extend( env_shell_file_paths ) - if all_env_shell_file_paths: - action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths + action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths return action_dict @@ -762,7 +774,7 @@ self.type = 'setup_purl_environment' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Initialize the environment for installing Perl packages. The class is called during the initial download stage when installing packages, so the value of initial_download will generally be True. @@ -819,8 +831,11 @@ return tool_dependency, filtered_actions, dir return tool_dependency, None, None with lcd( tmp_work_dir ): - cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_dir ) ) - return_code = install_environment.handle_command( app, tool_dependency, install_dir, cmd ) + cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_environment ) ) + return_code = install_environment.handle_command( app=app, + tool_dependency=tool_dependency, + cmd=cmd, + return_output=False ) if return_code: if initial_download: return tool_dependency, filtered_actions, dir @@ -829,8 +844,11 @@ # perl package from CPAN without version number. # cpanm should be installed with the parent perl distribution, otherwise this will not work. cmd += '''cpanm --local-lib=$INSTALL_DIR %s''' % ( perl_package ) - cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_dir ) ) - return_code = install_environment.handle_command( app, tool_dependency, install_dir, cmd ) + cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_environment ) ) + return_code = install_environment.handle_command( app=app, + tool_dependency=tool_dependency, + cmd=cmd, + return_output=False ) if return_code: if initial_download: return tool_dependency, filtered_actions, dir @@ -838,8 +856,12 @@ # Pull in perl dependencies (runtime). env_file_builder.handle_action_shell_file_paths( env_file_builder, action_dict ) # Recursively add dependent PERL5LIB and PATH to env.sh & anything else needed. - env_file_builder.append_line( name="PERL5LIB", action="prepend_to", value=os.path.join( install_dir, 'lib', 'perl5' ) ) - env_file_builder.append_line( name="PATH", action="prepend_to", value=os.path.join( install_dir, 'bin' ) ) + env_file_builder.append_line( name="PERL5LIB", + action="prepend_to", + value=os.path.join( install_environment.install_dir, 'lib', 'perl5' ) ) + env_file_builder.append_line( name="PATH", + action="prepend_to", + value=os.path.join( install_environment.install_dir, 'bin' ) ) return_code = env_file_builder.return_code if return_code: if initial_download: @@ -849,7 +871,7 @@ return tool_dependency, filtered_actions, dir return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # setup a Perl environment. # <action type="setup_perl_environment"> # <repository name="package_perl_5_18" owner="bgruening"> @@ -887,7 +909,7 @@ self.type = 'setup_r_environment' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Initialize the environment for installing R packages. The class is called during the initial download stage when installing packages, so the value of initial_download will generally be True. @@ -931,8 +953,11 @@ cmd = r'''PATH=$PATH:$R_HOME/bin; export PATH; R_LIBS=$INSTALL_DIR; export R_LIBS; Rscript -e "install.packages(c('%s'),lib='$INSTALL_DIR', repos=NULL, dependencies=FALSE)"''' % \ ( str( tarball_name ) ) - cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_dir ) ) - return_code = install_environment.handle_command( app, tool_dependency, install_dir, cmd ) + cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_environment ) ) + return_code = install_environment.handle_command( app=app, + tool_dependency=tool_dependency, + cmd=cmd, + return_output=False ) if return_code: if initial_download: return tool_dependency, filtered_actions, dir @@ -940,7 +965,7 @@ # R libraries are installed to $INSTALL_DIR (install_dir), we now set the R_LIBS path to that directory # Pull in R environment (runtime). env_file_builder.handle_action_shell_file_paths( action_dict ) - env_file_builder.append_line( name="R_LIBS", action="prepend_to", value=install_dir ) + env_file_builder.append_line( name="R_LIBS", action="prepend_to", value=install_environment.install_dir ) return_code = env_file_builder.return_code if return_code: if initial_download: @@ -950,7 +975,7 @@ return tool_dependency, filtered_actions, dir return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # setup an R environment. # <action type="setup_r_environment"> # <repository name="package_r_3_0_1" owner="bgruening"> @@ -982,7 +1007,7 @@ self.type = 'setup_ruby_environment' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Initialize the environment for installing Ruby packages. The class is called during the initial download stage when installing packages, so the value of initial_download will generally be True. @@ -1043,16 +1068,23 @@ # no version number given cmd = '''PATH=$PATH:$RUBY_HOME/bin; export PATH; GEM_HOME=$INSTALL_DIR; export GEM_HOME; gem install %s''' % ( gem ) - cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_dir ) ) - return_code = install_environment.handle_command( app, tool_dependency, install_dir, cmd ) + cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_environment ) ) + return_code = install_environment.handle_command( app=app, + tool_dependency=tool_dependency, + cmd=cmd, + return_output=False ) if return_code: if initial_download: return tool_dependency, filtered_actions, dir return tool_dependency, None, None # Pull in ruby dependencies (runtime). env_file_builder.handle_action_shell_file_paths( env_file_builder, action_dict ) - env_file_builder.append_line( name="GEM_PATH", action="prepend_to", value=install_dir ) - env_file_builder.append_line( name="PATH", action="prepend_to", value=os.path.join(install_dir, 'bin') ) + env_file_builder.append_line( name="GEM_PATH", + action="prepend_to", + value=install_environment.install_dir ) + env_file_builder.append_line( name="PATH", + action="prepend_to", + value=os.path.join( install_environment.install_dir, 'bin' ) ) return_code = env_file_builder.return_code if return_code: if initial_download: @@ -1062,7 +1094,7 @@ return tool_dependency, filtered_actions, dir return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # setup a Ruby environment. # <action type="setup_ruby_environment"> # <repository name="package_ruby_2_0" owner="bgruening"> @@ -1109,7 +1141,7 @@ self.type = 'setup_virtualenv' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Initialize a virtual environment for installing packages. If initial_download is True, the recipe steps will be filtered and returned and the installation directory (i.e., dir) will be defined and @@ -1124,15 +1156,15 @@ log.debug( 'Unable to install virtualenv' ) return tool_dependency, None, None requirements = action_dict[ 'requirements' ] - if os.path.exists( os.path.join( install_dir, requirements ) ): + if os.path.exists( os.path.join( install_environment.install_dir, requirements ) ): # requirements specified as path to a file requirements_path = requirements else: # requirements specified directly in XML, create a file with these for pip. - requirements_path = os.path.join( install_dir, "requirements.txt" ) + requirements_path = os.path.join( install_environment.install_dir, "requirements.txt" ) with open( requirements_path, "w" ) as f: f.write( requirements ) - venv_directory = os.path.join( install_dir, "venv" ) + venv_directory = os.path.join( install_environment.install_dir, "venv" ) python_cmd = action_dict[ 'python' ] # TODO: Consider making --no-site-packages optional. setup_command = "%s %s/virtualenv.py --no-site-packages '%s'" % ( python_cmd, venv_src_directory, venv_directory ) @@ -1140,7 +1172,10 @@ # and well defined behavior in bash/zsh. activate_command = "POSIXLY_CORRECT=1; . %s" % os.path.join( venv_directory, "bin", "activate" ) if action_dict[ 'use_requirements_file' ]: - install_command = "python '%s' install -r '%s' --log '%s'" % ( os.path.join( venv_directory, "bin", "pip" ), requirements_path, os.path.join( install_dir, 'pip_install.log' ) ) + install_command = "python '%s' install -r '%s' --log '%s'" % \ + ( os.path.join( venv_directory, "bin", "pip" ), + requirements_path, + os.path.join( install_environment.install_dir, 'pip_install.log' ) ) else: install_command = '' with open( requirements_path, "rb" ) as f: @@ -1150,18 +1185,29 @@ break line = line.strip() if line: - line_install_command = "python '%s' install %s --log '%s'" % ( os.path.join( venv_directory, "bin", "pip" ), line, os.path.join( install_dir, 'pip_install_%s.log' % ( line ) ) ) + line_install_command = "python '%s' install %s --log '%s'" % \ + ( os.path.join( venv_directory, "bin", "pip" ), + line, + os.path.join( install_environment.install_dir, 'pip_install_%s.log' % ( line ) ) ) if not install_command: install_command = line_install_command else: install_command = "%s && %s" % ( install_command, line_install_command ) full_setup_command = "%s; %s; %s" % ( setup_command, activate_command, install_command ) - return_code = install_environment.handle_command( app, tool_dependency, install_dir, full_setup_command ) + return_code = install_environment.handle_command( app=app, + tool_dependency=tool_dependency, + cmd=full_setup_command, + return_output=False ) if return_code: log.error( "Failed to do setup_virtualenv install, exit code='%s'", return_code ) # would it be better to try to set env variables anway, instead of returning here? return tool_dependency, None, None - site_packages_directory, site_packages_directory_list = self.__get_site_packages_directory( install_environment, app, tool_dependency, install_dir, python_cmd, venv_directory ) + site_packages_directory, site_packages_directory_list = \ + self.__get_site_packages_directory( install_environment, + app, + tool_dependency, + python_cmd, + venv_directory ) env_file_builder.append_line( name="PATH", action="prepend_to", value=os.path.join( venv_directory, "bin" ) ) if site_packages_directory is None: log.error( "virtualenv's site-packages directory '%s' does not exist", site_packages_directory_list ) @@ -1185,7 +1231,7 @@ shutil.move( full_path_to_dir, venv_dir ) return True - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="setup_virtualenv" /> ## Install requirements from file requirements.txt of downloaded bundle - or - # <action type="setup_virtualenv">tools/requirements.txt</action> @@ -1194,11 +1240,11 @@ # lxml==2.3.0</action> ## Manually specify contents of requirements.txt file to create dynamically. action_dict[ 'use_requirements_file' ] = asbool( action_elem.get( 'use_requirements_file', True ) ) - action_dict[ 'requirements' ] = td_common_util.evaluate_template( action_elem.text or 'requirements.txt', install_dir ) + action_dict[ 'requirements' ] = td_common_util.evaluate_template( action_elem.text or 'requirements.txt', install_environment ) action_dict[ 'python' ] = action_elem.get( 'python', 'python' ) return action_dict - def __get_site_packages_directory( self, install_environment, app, tool_dependency, install_dir, python_cmd, venv_directory ): + def __get_site_packages_directory( self, install_environment, app, tool_dependency, python_cmd, venv_directory ): lib_dir = os.path.join( venv_directory, "lib" ) rval = os.path.join( lib_dir, python_cmd, 'site-packages' ) site_packages_directory_list = [ rval ] @@ -1220,7 +1266,10 @@ os.path.join( venv_directory, "bin", "python" ), r"""%s -c 'import os, sys; print os.path.join( sys.prefix, "lib", "python" + sys.version[:3], "site-packages" )'""" % \ os.path.join( venv_directory, "bin", "python" ) ]: - output = install_environment.handle_command( app, tool_dependency, install_dir, site_packages_command, return_output=True ) + output = install_environment.handle_command( app=app, + tool_dependency=tool_dependency, + cmd=site_packages_command, + return_output=True ) site_packages_directory_list.append( output.stdout ) if not output.return_code and os.path.exists( output.stdout ): return ( output.stdout, site_packages_directory_list ) @@ -1232,7 +1281,7 @@ self.type = 'shell_command' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Execute a command in a shell. If the value of initial_download is True, the recipe steps will be filtered and returned and the installation directory (i.e., dir) will be defined and returned. @@ -1252,14 +1301,17 @@ with settings( warn_only=True ): # The caller should check the status of the returned tool_dependency since this function # does nothing with return_code. - return_code = install_environment.handle_command( app, tool_dependency, install_dir, cmd ) + return_code = install_environment.handle_command( app=app, + tool_dependency=tool_dependency, + cmd=cmd, + return_output=False ) if initial_download: return tool_dependency, filtered_actions, dir return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="shell_command">make</action> - action_elem_text = td_common_util.evaluate_template( action_elem.text, install_dir ) + action_elem_text = td_common_util.evaluate_template( action_elem.text, install_environment ) if action_elem_text: action_dict[ 'command' ] = action_elem_text return action_dict @@ -1271,7 +1323,7 @@ self.type = 'template_command' def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, - install_environment, work_dir, install_dir, current_dir=None, initial_download=False ): + install_environment, work_dir, current_dir=None, initial_download=False ): """ Execute a template command in a shell. If the value of initial_download is True, the recipe steps will be filtered and returned and the installation directory (i.e., dir) will be defined and returned. @@ -1280,7 +1332,8 @@ """ env_vars = dict() env_vars = install_environment.environment_dict() - env_vars.update( td_common_util.get_env_var_values( install_dir ) ) + tool_shed_repository = tool_depenedncy.tool_shed_repository + env_vars.update( td_common_util.get_env_var_values( install_environment ) ) language = action_dict[ 'language' ] with settings( warn_only=True, **env_vars ): if language == 'cheetah': @@ -1288,10 +1341,13 @@ cmd = fill_template( '#from fabric.api import env\n%s' % action_dict[ 'command' ], context=env_vars ) # The caller should check the status of the returned tool_dependency since this function # does nothing with return_code. - return_code = install_environment.handle_command( app, tool_dependency, install_dir, cmd ) + return_code = install_environment.handle_command( app=app, + tool_dependency=tool_dependency, + cmd=cmd, + return_output=False ) return tool_dependency, None, None - def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_dir, is_binary_download ): + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # Default to Cheetah as it's the first template language supported. language = action_elem.get( 'language', 'cheetah' ).lower() if language == 'cheetah': diff -r c05ab2ae3a58938e07c33193a7a52a89cbd43056 -r 5582e58fa7a27364f25cd66f54f373ed2e108a11 lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py @@ -187,9 +187,10 @@ env_var_text = elem.text.replace( '$INSTALL_DIR', tool_shed_repository_install_dir ) return dict( name=env_var_name, action=env_var_action, value=env_var_text ) if elem.text: - # Allow for environment variables that contain neither REPOSITORY_INSTALL_DIR nor INSTALL_DIR since there may be command line - # parameters that are tuned for a Galaxy instance. Allowing them to be set in one location rather than being hard coded into - # each tool config is the best approach. For example: + # Allow for environment variables that contain neither REPOSITORY_INSTALL_DIR nor INSTALL_DIR + # since there may be command line parameters that are tuned for a Galaxy instance. Allowing them + # to be set in one location rather than being hard coded into each tool config is the best approach. + # For example: # <environment_variable name="GATK2_SITE_OPTIONS" action="set_to"> # "--num_threads 4 --num_cpu_threads_per_data_thread 3 --phone_home STANDARD" # </environment_variable> @@ -213,9 +214,14 @@ regex = regex.replace( r"\'", "'" ) return regex -def evaluate_template( text, install_dir ): - """ Substitute variables defined in XML blocks from dependencies file.""" - return Template( text ).safe_substitute( get_env_var_values( install_dir ) ) +def evaluate_template( text, install_environment ): + """ + Substitute variables defined in XML blocks from dependencies file. The value of the received + repository_install_dir is the root installation directory of the repository that contains the + tool dependency. The value of the received install_dir is the root installation directory of + the tool_dependency. + """ + return Template( text ).safe_substitute( get_env_var_values( install_environment ) ) def format_traceback(): ex_type, ex, tb = sys.exc_info() @@ -320,10 +326,18 @@ action_dict[ 'action_shell_file_paths' ] = env_shell_file_paths return action_dict -def get_env_var_values( install_dir ): +def get_env_var_values( install_environment ): + """ + Return a dictionary of values, some of which enable substitution of reserved words for the values. + The received install_enviroment object has 2 important attributes for reserved word substitution: + install_environment.tool_shed_repository_install_dir is the root installation directory of the repository + that contains the tool dependency being installed, and install_environment.install_dir is the root + installation directory of the tool dependency. + """ env_var_dict = {} - env_var_dict[ 'INSTALL_DIR' ] = install_dir - env_var_dict[ 'system_install' ] = install_dir + env_var_dict[ 'REPOSITORY_INSTALL_DIR' ] = install_environment.tool_shed_repository_install_dir + env_var_dict[ 'INSTALL_DIR' ] = install_environment.install_dir + env_var_dict[ 'system_install' ] = install_environment.install_dir # If the Python interpreter is 64bit then we can safely assume that the underlying system is also 64bit. env_var_dict[ '__is64bit__' ] = sys.maxsize > 2**32 return env_var_dict Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.