6 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/67be8479c89d/ Changeset: 67be8479c89d User: BjoernGruening Date: 2013-09-16 10:54:21 Summary: move evaluate_template() to td_common_util and add the install_dir parameter Affected #: 2 files diff -r b65602624e257d96d90fe94de13b483fd70d2f7e -r 67be8479c89db565816a71edd41d69228b4e87ee lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -445,10 +445,6 @@ def install_via_fabric( app, tool_dependency, install_dir, package_name=None, proprietary_fabfile_path=None, actions_elem=None, action_elem=None, **kwd ): """Parse a tool_dependency.xml file's <actions> tag set to gather information for the installation via fabric.""" - def evaluate_template( text ): - """ Substitute variables defined in XML blocks from dependencies file.""" - return Template( text ).safe_substitute( td_common_util.get_env_var_values( install_dir ) ) - sa_session = app.model.context.current if not os.path.exists( install_dir ): os.makedirs( install_dir ) @@ -493,7 +489,7 @@ action_dict[ 'target_directory' ] = action_elem.get( 'target_directory', None ) elif action_type == 'shell_command': # <action type="shell_command">make</action> - action_elem_text = evaluate_template( action_elem.text ) + action_elem_text = td_common_util.evaluate_template( action_elem.text, install_dir ) if action_elem_text: action_dict[ 'command' ] = action_elem_text else: @@ -541,7 +537,7 @@ elif action_type == 'make_directory': # <action type="make_directory">$INSTALL_DIR/lib/python</action> if action_elem.text: - action_dict[ 'full_path' ] = evaluate_template( action_elem.text ) + action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_dir ) else: continue elif action_type == 'change_directory': @@ -560,7 +556,7 @@ # <destination_directory>$INSTALL_DIR/bin</destination_directory> # </action> for move_elem in action_elem: - move_elem_text = evaluate_template( move_elem.text ) + move_elem_text = td_common_util.evaluate_template( move_elem.text, install_dir ) if move_elem_text: action_dict[ move_elem.tag ] = move_elem_text elif action_type == 'set_environment': @@ -605,9 +601,12 @@ # <action type="setup_virtualenv">pyyaml==3.2.0 # lxml==2.3.0</action> ## Manually specify contents of requirements.txt file to create dynamically. - action_dict[ 'requirements' ] = evaluate_template( action_elem.text or 'requirements.txt' ) + action_dict[ 'requirements' ] = td_common_util.evaluate_template( action_elem.text or 'requirements.txt', install_dir ) elif action_type == 'chmod': # Change the read, write, and execute bits on a file. + # <action type="chmod"> + # <file mode="750">$INSTALL_DIR/bin/faToTwoBit</file> + # </action> file_elems = action_elem.findall( 'file' ) chmod_actions = [] # A unix octal mode is the sum of the following values: @@ -623,7 +622,7 @@ received_mode = int( file_elem.get( 'mode', 600 ), base=8 ) # For added security, ensure that the setuid and setgid bits are not set. mode = received_mode & ~( stat.S_ISUID | stat.S_ISGID ) - file = evaluate_template( file_elem.text ) + file = td_common_util.evaluate_template( file_elem.text, install_dir ) chmod_tuple = ( file, mode ) chmod_actions.append( chmod_tuple ) action_dict[ 'change_modes' ] = chmod_actions diff -r b65602624e257d96d90fe94de13b483fd70d2f7e -r 67be8479c89db565816a71edd41d69228b4e87ee lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py @@ -6,6 +6,7 @@ import traceback import urllib2 import zipfile +from string import Template import tool_shed.util.shed_util_common as suc from galaxy.datatypes import checkers from urllib2 import HTTPError @@ -415,3 +416,8 @@ def __shellquote(s): """Quote and escape the supplied string for use in shell expressions.""" return "'" + s.replace( "'", "'\\''" ) + "'" + +def evaluate_template( text, install_dir ): + """ Substitute variables defined in XML blocks from dependencies file.""" + return Template( text ).safe_substitute( get_env_var_values( install_dir ) ) + https://bitbucket.org/galaxy/galaxy-central/commits/d4e5ea66bd74/ Changeset: d4e5ea66bd74 Branch: action_type_misc User: BjoernGruening Date: 2013-09-16 10:59:47 Summary: touch Affected #: 1 file diff -r 67be8479c89db565816a71edd41d69228b4e87ee -r d4e5ea66bd74385e7eb2e76191566ebe5dd3109a lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py @@ -420,4 +420,3 @@ def evaluate_template( text, install_dir ): """ Substitute variables defined in XML blocks from dependencies file.""" return Template( text ).safe_substitute( get_env_var_values( install_dir ) ) - https://bitbucket.org/galaxy/galaxy-central/commits/5b17d89780b3/ Changeset: 5b17d89780b3 Branch: action_type_misc User: BjoernGruening Date: 2013-09-16 11:42:33 Summary: Add John Chilton's InstallEnvrionment() abstraction. Setup abstraction (InstallEnvrionment) in tool shed client for capturing the environment setup to during building and installations of packages. This simplifies install_and_build_package and provides a clear extension point for other customizations to the install process. Affected #: 1 file diff -r d4e5ea66bd74385e7eb2e76191566ebe5dd3109a -r 5b17d89780b381bfd53712f964e0b53137adc322 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -164,6 +164,54 @@ shutil.move( full_path_to_dir, venv_dir ) return True + +class InstallEnvironment( object ): + """ + Object describing the environment built up as part of the process of building + and installing a package. + """ + + def __init__( self ): + self.env_shell_file_paths = [] + + def build_command( self, command, action_type='shell_command' ): + """ + Build command line for execution from simple command, but + configuring environment described by this object. + """ + env_cmds = self.environment_commands(action_type) + return '\n'.join(env_cmds + [command]) + + def environment_commands(self, action_type): + """ + Build a list of commands used to construct the environment described by + this object. + """ + cmds = [] + for env_shell_file_path in self.env_shell_file_paths: + if os.path.exists( env_shell_file_path ): + for env_setting in open( env_shell_file_path ): + cmds.append( env_setting.strip( '\n' ) ) + else: + log.debug( 'Invalid file %s specified, ignoring %s action.', env_shell_file_path, action_type ) + return cmds + + def environment_dict(self, action_type='template_command'): + env_vars = dict() + for env_shell_file_path in self.env_shell_file_paths: + if os.path.exists( env_shell_file_path ): + for env_setting in open( env_shell_file_path ): + env_string = env_setting.split( ';' )[ 0 ] + env_name, env_path = env_string.split( '=' ) + env_vars[ env_name ] = env_path + else: + log.debug( 'Invalid file %s specified, ignoring template_command action.', env_shell_file_path ) + return env_vars + + def add_env_shell_file_paths(self, paths): + self.env_shell_file_paths.extend(paths) + + def install_and_build_package( app, tool_dependency, actions_dict ): """Install a Galaxy tool dependency package either via a url or a mercurial or git clone command.""" sa_session = app.model.context.current @@ -171,7 +219,7 @@ package_name = actions_dict[ 'package_name' ] actions = actions_dict.get( 'actions', None ) filtered_actions = [] - env_shell_file_paths = [] + install_environment = InstallEnvironment() if actions: with make_tmp_dir() as work_dir: with lcd( work_dir ): @@ -286,13 +334,7 @@ # Currently the only action supported in this category is "environment_variable". # Build a command line from the prior_installation_required, in case an environment variable is referenced # in the set_environment action. - cmds = [] - for env_shell_file_path in env_shell_file_paths: - if os.path.exists( env_shell_file_path ): - for env_setting in open( env_shell_file_path ): - cmds.append( env_setting.strip( '\n' ) ) - else: - log.debug( 'Invalid file %s specified, ignoring set_environment action.', env_shell_file_path ) + cmds = install_environment.environment_commands( 'set_environment' ) env_var_dicts = action_dict[ 'environment_variable' ] for env_var_dict in env_var_dicts: # Check for the presence of the $ENV[] key string and populate it if possible. @@ -305,7 +347,7 @@ # Currently the only action supported in this category is a list of paths to one or more tool dependency env.sh files, # the environment setting in each of which will be injected into the environment for all <action type="shell_command"> # tags that follow this <action type="set_environment_for_install"> tag set in the tool_dependencies.xml file. - env_shell_file_paths = action_dict[ 'env_shell_file_paths' ] + install_environment.add_env_shell_file_paths( action_dict[ 'env_shell_file_paths' ] ) elif action_type == 'setup_virtualenv': # TODO: maybe should be configurable venv_src_directory = os.path.abspath( os.path.join( app.config.tool_dependency_dir, '__virtualenv_src' ) ) @@ -351,27 +393,13 @@ return elif action_type == 'shell_command': with settings( warn_only=True ): - cmd = '' - for env_shell_file_path in env_shell_file_paths: - if os.path.exists( env_shell_file_path ): - for env_setting in open( env_shell_file_path ): - cmd += '%s\n' % env_setting - else: - log.debug( 'Invalid file %s specified, ignoring shell_command action.', env_shell_file_path ) - cmd += action_dict[ 'command' ] + install_environment.add_env_shell_file_paths( action_dict[ 'env_shell_file_paths' ] ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: return elif action_type == 'template_command': env_vars = dict() - for env_shell_file_path in env_shell_file_paths: - if os.path.exists( env_shell_file_path ): - for env_setting in open( env_shell_file_path ): - env_string = env_setting.split( ';' )[ 0 ] - env_name, env_path = env_string.split( '=' ) - env_vars[ env_name ] = env_path - else: - log.debug( 'Invalid file %s specified, ignoring template_command action.', env_shell_file_path ) + env_vars = install_environment.environment_dict() env_vars.update( td_common_util.get_env_var_values( install_dir ) ) language = action_dict[ 'language' ] with settings( warn_only=True, **env_vars ): https://bitbucket.org/galaxy/galaxy-central/commits/318dc55bd485/ Changeset: 318dc55bd485 Branch: action_type_misc User: BjoernGruening Date: 2013-09-16 14:30:11 Summary: bugfix Affected #: 1 file diff -r 5b17d89780b381bfd53712f964e0b53137adc322 -r 318dc55bd4859105a315d0d52de2e2ba032260eb lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -393,7 +393,7 @@ return elif action_type == 'shell_command': with settings( warn_only=True ): - install_environment.add_env_shell_file_paths( action_dict[ 'env_shell_file_paths' ] ) + install_environment.add_env_shell_file_paths( action_dict[ 'command' ] ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: return https://bitbucket.org/galaxy/galaxy-central/commits/223f968a3b3c/ Changeset: 223f968a3b3c Branch: action_type_misc User: BjoernGruening Date: 2013-09-16 14:37:25 Summary: forgot the cmd variable Affected #: 1 file diff -r 318dc55bd4859105a315d0d52de2e2ba032260eb -r 223f968a3b3c35c709b7588bec33f4e183c8d906 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -393,7 +393,7 @@ return elif action_type == 'shell_command': with settings( warn_only=True ): - install_environment.add_env_shell_file_paths( action_dict[ 'command' ] ) + cmd = install_environment.build_command( action_dict[ 'command' ] ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: return https://bitbucket.org/galaxy/galaxy-central/commits/86573eb35eec/ Changeset: 86573eb35eec User: inithello Date: 2013-10-15 17:15:57 Summary: Merged in BjoernGruening/galaxy-central-bgruening/action_type_misc (pull request #216) Move the evaluate_template function to the td_common_util file. Affected #: 3 files diff -r 00d3bdc6e73d65126b97eb38289dc52688f810a3 -r 86573eb35eec9474ac4121875da321811f343994 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -199,6 +199,54 @@ shutil.move( full_path_to_dir, venv_dir ) return True + +class InstallEnvironment( object ): + """ + Object describing the environment built up as part of the process of building + and installing a package. + """ + + def __init__( self ): + self.env_shell_file_paths = [] + + def build_command( self, command, action_type='shell_command' ): + """ + Build command line for execution from simple command, but + configuring environment described by this object. + """ + env_cmds = self.environment_commands(action_type) + return '\n'.join(env_cmds + [command]) + + def environment_commands(self, action_type): + """ + Build a list of commands used to construct the environment described by + this object. + """ + cmds = [] + for env_shell_file_path in self.env_shell_file_paths: + if os.path.exists( env_shell_file_path ): + for env_setting in open( env_shell_file_path ): + cmds.append( env_setting.strip( '\n' ) ) + else: + log.debug( 'Invalid file %s specified, ignoring %s action.', env_shell_file_path, action_type ) + return cmds + + def environment_dict(self, action_type='template_command'): + env_vars = dict() + for env_shell_file_path in self.env_shell_file_paths: + if os.path.exists( env_shell_file_path ): + for env_setting in open( env_shell_file_path ): + env_string = env_setting.split( ';' )[ 0 ] + env_name, env_path = env_string.split( '=' ) + env_vars[ env_name ] = env_path + else: + log.debug( 'Invalid file %s specified, ignoring template_command action.', env_shell_file_path ) + return env_vars + + def add_env_shell_file_paths(self, paths): + self.env_shell_file_paths.extend(paths) + + def install_and_build_package( app, tool_dependency, actions_dict ): """Install a Galaxy tool dependency package either via a url or a mercurial or git clone command.""" sa_session = app.model.context.current @@ -206,7 +254,7 @@ package_name = actions_dict[ 'package_name' ] actions = actions_dict.get( 'actions', None ) filtered_actions = [] - env_shell_file_paths = [] + install_environment = InstallEnvironment() if actions: with make_tmp_dir() as work_dir: with lcd( work_dir ): @@ -326,13 +374,7 @@ # Currently the only action supported in this category is "environment_variable". # Build a command line from the prior_installation_required, in case an environment variable is referenced # in the set_environment action. - cmds = [] - for env_shell_file_path in env_shell_file_paths: - if os.path.exists( env_shell_file_path ): - for env_setting in open( env_shell_file_path ): - cmds.append( env_setting.strip( '\n' ) ) - else: - log.debug( 'Invalid file %s specified, ignoring set_environment action.', env_shell_file_path ) + cmds = install_environment.environment_commands( 'set_environment' ) env_var_dicts = action_dict[ 'environment_variable' ] for env_var_dict in env_var_dicts: # Check for the presence of the $ENV[] key string and populate it if possible. @@ -345,7 +387,7 @@ # Currently the only action supported in this category is a list of paths to one or more tool dependency env.sh files, # the environment setting in each of which will be injected into the environment for all <action type="shell_command"> # tags that follow this <action type="set_environment_for_install"> tag set in the tool_dependencies.xml file. - env_shell_file_paths = action_dict[ 'env_shell_file_paths' ] + install_environment.add_env_shell_file_paths( action_dict[ 'env_shell_file_paths' ] ) elif action_type == 'setup_virtualenv': # TODO: maybe should be configurable venv_src_directory = os.path.abspath( os.path.join( app.config.tool_dependency_dir, '__virtualenv_src' ) ) @@ -391,27 +433,13 @@ return elif action_type == 'shell_command': with settings( warn_only=True ): - cmd = '' - for env_shell_file_path in env_shell_file_paths: - if os.path.exists( env_shell_file_path ): - for env_setting in open( env_shell_file_path ): - cmd += '%s\n' % env_setting - else: - log.debug( 'Invalid file %s specified, ignoring shell_command action.', env_shell_file_path ) - cmd += action_dict[ 'command' ] + cmd = install_environment.build_command( action_dict[ 'command' ] ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: return elif action_type == 'template_command': env_vars = dict() - for env_shell_file_path in env_shell_file_paths: - if os.path.exists( env_shell_file_path ): - for env_setting in open( env_shell_file_path ): - env_string = env_setting.split( ';' )[ 0 ] - env_name, env_path = env_string.split( '=' ) - env_vars[ env_name ] = env_path - else: - log.debug( 'Invalid file %s specified, ignoring template_command action.', env_shell_file_path ) + env_vars = install_environment.environment_dict() env_vars.update( td_common_util.get_env_var_values( install_dir ) ) language = action_dict[ 'language' ] with settings( warn_only=True, **env_vars ): diff -r 00d3bdc6e73d65126b97eb38289dc52688f810a3 -r 86573eb35eec9474ac4121875da321811f343994 lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -445,10 +445,6 @@ def install_via_fabric( app, tool_dependency, install_dir, package_name=None, proprietary_fabfile_path=None, actions_elem=None, action_elem=None, **kwd ): """Parse a tool_dependency.xml file's <actions> tag set to gather information for the installation via fabric.""" - def evaluate_template( text ): - """ Substitute variables defined in XML blocks from dependencies file.""" - return Template( text ).safe_substitute( td_common_util.get_env_var_values( install_dir ) ) - sa_session = app.model.context.current if not os.path.exists( install_dir ): os.makedirs( install_dir ) @@ -493,7 +489,7 @@ action_dict[ 'target_directory' ] = action_elem.get( 'target_directory', None ) elif action_type == 'shell_command': # <action type="shell_command">make</action> - action_elem_text = evaluate_template( action_elem.text ) + action_elem_text = td_common_util.evaluate_template( action_elem.text, install_dir ) if action_elem_text: action_dict[ 'command' ] = action_elem_text else: @@ -541,7 +537,7 @@ elif action_type == 'make_directory': # <action type="make_directory">$INSTALL_DIR/lib/python</action> if action_elem.text: - action_dict[ 'full_path' ] = evaluate_template( action_elem.text ) + action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_dir ) else: continue elif action_type == 'change_directory': @@ -556,7 +552,7 @@ # <destination_directory>$INSTALL_DIR/bin</destination_directory> # </action> for move_elem in action_elem: - move_elem_text = evaluate_template( move_elem.text ) + move_elem_text = td_common_util.evaluate_template( move_elem.text, install_dir ) if move_elem_text: action_dict[ move_elem.tag ] = move_elem_text elif action_type == 'move_file': @@ -609,9 +605,12 @@ # <action type="setup_virtualenv">pyyaml==3.2.0 # lxml==2.3.0</action> ## Manually specify contents of requirements.txt file to create dynamically. - action_dict[ 'requirements' ] = evaluate_template( action_elem.text or 'requirements.txt' ) + action_dict[ 'requirements' ] = td_common_util.evaluate_template( action_elem.text or 'requirements.txt', install_dir ) elif action_type == 'chmod': # Change the read, write, and execute bits on a file. + # <action type="chmod"> + # <file mode="750">$INSTALL_DIR/bin/faToTwoBit</file> + # </action> file_elems = action_elem.findall( 'file' ) chmod_actions = [] # A unix octal mode is the sum of the following values: @@ -627,7 +626,7 @@ received_mode = int( file_elem.get( 'mode', 600 ), base=8 ) # For added security, ensure that the setuid and setgid bits are not set. mode = received_mode & ~( stat.S_ISUID | stat.S_ISGID ) - file = evaluate_template( file_elem.text ) + file = td_common_util.evaluate_template( file_elem.text, install_dir ) chmod_tuple = ( file, mode ) chmod_actions.append( chmod_tuple ) action_dict[ 'change_modes' ] = chmod_actions diff -r 00d3bdc6e73d65126b97eb38289dc52688f810a3 -r 86573eb35eec9474ac4121875da321811f343994 lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py @@ -7,6 +7,7 @@ import traceback import urllib2 import zipfile +from string import Template import tool_shed.util.shed_util_common as suc from galaxy.datatypes import checkers @@ -470,3 +471,7 @@ def __shellquote(s): """Quote and escape the supplied string for use in shell expressions.""" return "'" + s.replace( "'", "'\\''" ) + "'" + +def evaluate_template( text, install_dir ): + """ Substitute variables defined in XML blocks from dependencies file.""" + return Template( text ).safe_substitute( get_env_var_values( install_dir ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.