2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d9b308ec77f6/ Changeset: d9b308ec77f6 User: jmchilton Date: 2013-05-20 15:25:03 Summary: Implement setting up tool dependency action (setup_virtualenv) to automatically setup python virtual environments. Install requirements from file requirements.txt of downloaded bundle: <action type="setup_virtualenv" /> - or - Install requirements from specified file from downloaded bundle: <action type="setup_virtualenv">tools/requirements.txt</action> - or - Manually specify contents of requirements.txt file to create dynamically. <action type="setup_virtualenv">pyyaml==3.2.0 lxml==2.3.0</action> Other small refactorings to support this. Including: - Rework expansion of $INSTALL_DIR in tool_dependency.xml files to use string.Template and allow additional variables. $system_install now expands to the install dir to mimic what what we do with CloudBioLinux Galaxy tool dependency creation. Ultimately, would like to have $tool_version to expand as well, since most downloads differ only based on this version which must be set anyway. - Refactor a more general function, create_or_update_env_shell_file_with_command, out of create_or_update_env_shell_file. Use this more general function to implement Python virtualenv activation in tool dependencies. - Implement shell quoting create_or_update_env_shell_file_with_command to handle special characters in paths. Affected #: 3 files diff -r 237209336f0337ea9f47df39548df3c11900f182 -r d9b308ec77f675f33d340d6f6e08000de374441c lib/tool_shed/galaxy_install/tool_dependencies/common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py @@ -47,19 +47,28 @@ changed_value = '%s' % env_var_value elif env_var_action == 'append_to': changed_value = '$%s:%s' % ( env_var_name, env_var_value ) + line = "%s=%s; export %s" % (env_var_name, changed_value, env_var_name) + return create_or_update_env_shell_file_with_command(install_dir, line) + + +def create_or_update_env_shell_file_with_command( install_dir, command ): + """ + Return a shell expression which when executed will create or update + a Galaxy env.sh dependency file in the specified install_dir containing + the supplied command. + """ env_shell_file_path = '%s/env.sh' % install_dir if os.path.exists( env_shell_file_path ): write_action = '>>' else: write_action = '>' - cmd = "echo '%s=%s; export %s' %s %s;chmod +x %s" % ( env_var_name, - changed_value, - env_var_name, - write_action, - env_shell_file_path, - env_shell_file_path ) + cmd = "echo %s %s %s;chmod +x %s" % ( __shellquote(command), + write_action, + __shellquote(env_shell_file_path), + __shellquote(env_shell_file_path)) return cmd + def extract_tar( file_name, file_path ): if isgzip( file_name ) or isbz2( file_name ): # Open for reading with transparent compression. @@ -233,3 +242,10 @@ if not member_path.startswith( basename ): return False return True + + +def __shellquote(s): + """ + Quote and escape the supplied string for use in shell expressions. + """ + return "'" + s.replace("'", "'\\''") + "'" diff -r 237209336f0337ea9f47df39548df3c11900f182 -r d9b308ec77f675f33d340d6f6e08000de374441c lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -140,6 +140,31 @@ # the environment setting in each of which will be injected into the environment for all <action type="shell_command"> # tags that follow this <action type="set_environment_for_install"> tag set in the tool_dependencies.xml file. env_shell_file_paths = action_dict[ 'env_shell_file_paths' ] + elif action_type == 'setup_virtualenv': + requirements = action_dict[ 'requirements' ] + if os.path.exists( os.path.join( dir, requirements ) ): + # requirements specified as path to a file + requirements_path = requirements + else: + # requirements specified directly in XML, create a file with these for pip. + requirements_path = os.path.join( install_dir, "requirements.txt" ) + with open( requirements_path, "w" ) as f: + f.write( requirements ) + venv_directory = os.path.join( install_dir, "venv" ) + # TODO: Consider making --no-site-packages optional. + setup_command = "virtualenv --no-site-packages '%s'" % venv_directory + # POSIXLY_CORRECT forces shell commands . and source to have the same + # and well defined behavior in bash/zsh. + activate_command = "POSIXLY_CORRECT=1; . %s" % os.path.join( venv_directory, "bin", "activate" ) + install_command = "pip install -r '%s'" % requirements_path + full_setup_command = "%s; %s; %s" % ( setup_command, activate_command, install_command ) + return_code = handle_command( app, tool_dependency, install_dir, full_setup_command ) + if return_code: + return + modify_env_command = common_util.create_or_update_env_shell_file_with_command( install_dir, activate_command ) + return_code = handle_command( app, tool_dependency, install_dir, modify_env_command ) + if return_code: + return elif action_type == 'shell_command': with settings( warn_only=True ): cmd = '' diff -r 237209336f0337ea9f47df39548df3c11900f182 -r d9b308ec77f675f33d340d6f6e08000de374441c lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -3,6 +3,7 @@ import sys import subprocess import tempfile +from string import Template import common_util import fabric_util import tool_shed.util.shed_util_common as suc @@ -317,6 +318,17 @@ def install_via_fabric( app, tool_dependency, actions_elem, install_dir, package_name=None, proprietary_fabfile_path=None, **kwd ): """Parse a tool_dependency.xml file's <actions> tag set to gather information for the installation via fabric.""" sa_session = app.model.context.current + + def evaluate_template( text ): + """ Substitute variables defined in XML blocks obtained loaded from + dependencies file. """ + # TODO: Add tool_version substitution for compat with CloudBioLinux. + substitutions = { + "INSTALL_DIR": install_dir, + "system_install": install_dir, # Added for compat with CloudBioLinux + } + return Template( text ).safe_substitute( substitutions ) + if not os.path.exists( install_dir ): os.makedirs( install_dir ) actions_dict = dict( install_dir=install_dir ) @@ -328,7 +340,7 @@ action_type = action_elem.get( 'type', 'shell_command' ) if action_type == 'shell_command': # <action type="shell_command">make</action> - action_elem_text = action_elem.text.replace( '$INSTALL_DIR', install_dir ) + action_elem_text = evaluate_template( action_elem.text ) if action_elem_text: action_dict[ 'command' ] = action_elem_text else: @@ -351,7 +363,7 @@ elif action_type == 'make_directory': # <action type="make_directory">$INSTALL_DIR/lib/python</action> if action_elem.text: - action_dict[ 'full_path' ] = action_elem.text.replace( '$INSTALL_DIR', install_dir ) + action_dict[ 'full_path' ] = evaluate_template( action_elem.text ) else: continue elif action_type in [ 'move_directory_files', 'move_file' ]: @@ -364,7 +376,7 @@ # <destination_directory>$INSTALL_DIR/bin</destination_directory> # </action> for move_elem in action_elem: - move_elem_text = move_elem.text.replace( '$INSTALL_DIR', install_dir ) + move_elem_text = evaluate_template( move_elem.text ) if move_elem_text: action_dict[ move_elem.tag ] = move_elem_text elif action_type == 'set_environment': @@ -402,6 +414,15 @@ action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths else: continue + elif action_type == 'setup_virtualenv': + # <action type="setup_virtualenv" /> + ## Install requirements from file requirements.txt of downloaded bundle - or - + # <action type="setup_virtualenv">tools/requirements.txt</action> + ## Install requirements from specified file from downloaded bundle -or - + # <action type="setup_virtualenv">pyyaml==3.2.0 + # lxml==2.3.0</action> + ## Manually specify contents of requirements.txt file to create dynamically. + action_dict[ 'requirements' ] = evaluate_template( action_elem.text or 'requirements.txt' ) else: log.debug( "Unsupported action type '%s'. Not proceeding." % str( action_type ) ) raise Exception( "Unsupported action type '%s' in tool dependency definition." % str( action_type ) ) https://bitbucket.org/galaxy/galaxy-central/commits/68f1441f04f5/ Changeset: 68f1441f04f5 User: natefoo Date: 2013-05-20 20:56:07 Summary: Merged in jmchilton/galaxy-central-library-refactoring (pull request #167) Implement setting up tool dependency action (setup_virtualenv) to automatically setup python virtual environments. Affected #: 3 files diff -r e620c575616470afaa3923bf5817db07e09b45bc -r 68f1441f04f557ba06d41426408dab4e98997d26 lib/tool_shed/galaxy_install/tool_dependencies/common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py @@ -47,19 +47,28 @@ changed_value = '%s' % env_var_value elif env_var_action == 'append_to': changed_value = '$%s:%s' % ( env_var_name, env_var_value ) + line = "%s=%s; export %s" % (env_var_name, changed_value, env_var_name) + return create_or_update_env_shell_file_with_command(install_dir, line) + + +def create_or_update_env_shell_file_with_command( install_dir, command ): + """ + Return a shell expression which when executed will create or update + a Galaxy env.sh dependency file in the specified install_dir containing + the supplied command. + """ env_shell_file_path = '%s/env.sh' % install_dir if os.path.exists( env_shell_file_path ): write_action = '>>' else: write_action = '>' - cmd = "echo '%s=%s; export %s' %s %s;chmod +x %s" % ( env_var_name, - changed_value, - env_var_name, - write_action, - env_shell_file_path, - env_shell_file_path ) + cmd = "echo %s %s %s;chmod +x %s" % ( __shellquote(command), + write_action, + __shellquote(env_shell_file_path), + __shellquote(env_shell_file_path)) return cmd + def extract_tar( file_name, file_path ): if isgzip( file_name ) or isbz2( file_name ): # Open for reading with transparent compression. @@ -233,3 +242,10 @@ if not member_path.startswith( basename ): return False return True + + +def __shellquote(s): + """ + Quote and escape the supplied string for use in shell expressions. + """ + return "'" + s.replace("'", "'\\''") + "'" diff -r e620c575616470afaa3923bf5817db07e09b45bc -r 68f1441f04f557ba06d41426408dab4e98997d26 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -140,6 +140,31 @@ # the environment setting in each of which will be injected into the environment for all <action type="shell_command"> # tags that follow this <action type="set_environment_for_install"> tag set in the tool_dependencies.xml file. env_shell_file_paths = action_dict[ 'env_shell_file_paths' ] + elif action_type == 'setup_virtualenv': + requirements = action_dict[ 'requirements' ] + if os.path.exists( os.path.join( dir, requirements ) ): + # requirements specified as path to a file + requirements_path = requirements + else: + # requirements specified directly in XML, create a file with these for pip. + requirements_path = os.path.join( install_dir, "requirements.txt" ) + with open( requirements_path, "w" ) as f: + f.write( requirements ) + venv_directory = os.path.join( install_dir, "venv" ) + # TODO: Consider making --no-site-packages optional. + setup_command = "virtualenv --no-site-packages '%s'" % venv_directory + # POSIXLY_CORRECT forces shell commands . and source to have the same + # and well defined behavior in bash/zsh. + activate_command = "POSIXLY_CORRECT=1; . %s" % os.path.join( venv_directory, "bin", "activate" ) + install_command = "pip install -r '%s'" % requirements_path + full_setup_command = "%s; %s; %s" % ( setup_command, activate_command, install_command ) + return_code = handle_command( app, tool_dependency, install_dir, full_setup_command ) + if return_code: + return + modify_env_command = common_util.create_or_update_env_shell_file_with_command( install_dir, activate_command ) + return_code = handle_command( app, tool_dependency, install_dir, modify_env_command ) + if return_code: + return elif action_type == 'shell_command': with settings( warn_only=True ): cmd = '' diff -r e620c575616470afaa3923bf5817db07e09b45bc -r 68f1441f04f557ba06d41426408dab4e98997d26 lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -3,6 +3,7 @@ import sys import subprocess import tempfile +from string import Template import common_util import fabric_util import tool_shed.util.shed_util_common as suc @@ -317,6 +318,17 @@ def install_via_fabric( app, tool_dependency, actions_elem, install_dir, package_name=None, proprietary_fabfile_path=None, **kwd ): """Parse a tool_dependency.xml file's <actions> tag set to gather information for the installation via fabric.""" sa_session = app.model.context.current + + def evaluate_template( text ): + """ Substitute variables defined in XML blocks obtained loaded from + dependencies file. """ + # TODO: Add tool_version substitution for compat with CloudBioLinux. + substitutions = { + "INSTALL_DIR": install_dir, + "system_install": install_dir, # Added for compat with CloudBioLinux + } + return Template( text ).safe_substitute( substitutions ) + if not os.path.exists( install_dir ): os.makedirs( install_dir ) actions_dict = dict( install_dir=install_dir ) @@ -328,7 +340,7 @@ action_type = action_elem.get( 'type', 'shell_command' ) if action_type == 'shell_command': # <action type="shell_command">make</action> - action_elem_text = action_elem.text.replace( '$INSTALL_DIR', install_dir ) + action_elem_text = evaluate_template( action_elem.text ) if action_elem_text: action_dict[ 'command' ] = action_elem_text else: @@ -351,7 +363,7 @@ elif action_type == 'make_directory': # <action type="make_directory">$INSTALL_DIR/lib/python</action> if action_elem.text: - action_dict[ 'full_path' ] = action_elem.text.replace( '$INSTALL_DIR', install_dir ) + action_dict[ 'full_path' ] = evaluate_template( action_elem.text ) else: continue elif action_type in [ 'move_directory_files', 'move_file' ]: @@ -364,7 +376,7 @@ # <destination_directory>$INSTALL_DIR/bin</destination_directory> # </action> for move_elem in action_elem: - move_elem_text = move_elem.text.replace( '$INSTALL_DIR', install_dir ) + move_elem_text = evaluate_template( move_elem.text ) if move_elem_text: action_dict[ move_elem.tag ] = move_elem_text elif action_type == 'set_environment': @@ -402,6 +414,15 @@ action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths else: continue + elif action_type == 'setup_virtualenv': + # <action type="setup_virtualenv" /> + ## Install requirements from file requirements.txt of downloaded bundle - or - + # <action type="setup_virtualenv">tools/requirements.txt</action> + ## Install requirements from specified file from downloaded bundle -or - + # <action type="setup_virtualenv">pyyaml==3.2.0 + # lxml==2.3.0</action> + ## Manually specify contents of requirements.txt file to create dynamically. + action_dict[ 'requirements' ] = evaluate_template( action_elem.text or 'requirements.txt' ) else: log.debug( "Unsupported action type '%s'. Not proceeding." % str( action_type ) ) raise Exception( "Unsupported action type '%s' in tool dependency definition." % str( action_type ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.