2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7b424e9c2e57/ Changeset: 7b424e9c2e57 Branch: setup_python_env User: Bjoern Gruening Date: 2014-06-29 20:19:03 Summary: Add new install_python_environment installation tag to the Tool Shed. It can be used like the following example: <action type="setup_python_environment"><repository name="package_python_2_7" owner="iuc"><package name="python" version="2.7" /></repository><!-- allow downloading and installing an Python package from https://pypi.org/ --><package>https://pypi.python.org/packages/source/d/deepTools/deepTools-1.5.8.tar.gz</package><package>kMer-0.2.0.tar.gz</package></action> Currently, you can specify one URL to a tarball or a name to a locally shiped tarball. Important: The underlying easy_install will skip all dependencies. The Tool Author need to specify every dependency explicitly. Test repository can be found on the Test Tool Shed: https://testtoolshed.g2.bx.psu.edu/view/bgruening/test_repository_setup_pyth... https://testtoolshed.g2.bx.psu.edu/view/bgruening/test_repository_setup_pyth... Affected #: 4 files diff -r 381a36069c14bf20c98955c01ab00e7838257943 -r 7b424e9c2e57ec12eca26f35c8f4259bbcc55cdb lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -44,7 +44,8 @@ def __init__( self, app ): self.app = app - self.INSTALL_ACTIONS = [ 'download_binary', 'download_by_url', 'download_file', 'setup_perl_environment', + self.INSTALL_ACTIONS = [ 'download_binary', 'download_by_url', 'download_file', + 'setup_perl_environment', 'setup_python_environment', 'setup_r_environment', 'setup_ruby_environment', 'shell_command' ] def format_traceback( self ): diff -r 381a36069c14bf20c98955c01ab00e7838257943 -r 7b424e9c2e57ec12eca26f35c8f4259bbcc55cdb lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py @@ -124,7 +124,7 @@ Parse an XML tag set to discover all child repository dependency tags and define the path to an env.sh file associated with the repository (this requires the repository dependency to be in an installed state). The received action_dict will be updated with these discovered paths and returned to the caller. This method handles tool dependency definition - tag sets <setup_r_environment>, <setup_ruby_environment> and <setup_perl_environment>. + tag sets <setup_r_environment>, <setup_ruby_environment>, <setup_python_environment> and <setup_perl_environment>. """ # An example elem is: # <action type="setup_perl_environment"> diff -r 381a36069c14bf20c98955c01ab00e7838257943 -r 7b424e9c2e57ec12eca26f35c8f4259bbcc55cdb lib/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py @@ -54,6 +54,7 @@ setup_perl_environment=step_handler.SetupPerlEnvironment( self.app ), setup_r_environment=step_handler.SetupREnvironment( self.app ), setup_ruby_environment=step_handler.SetupRubyEnvironment( self.app ), + setup_python_environment=step_handler.SetupPythonEnvironment( self.app ), setup_virtualenv=step_handler.SetupVirtualEnv( self.app ), shell_command=step_handler.ShellCommand( self.app ), template_command=step_handler.TemplateCommand( self.app ) ) diff -r 381a36069c14bf20c98955c01ab00e7838257943 -r 7b424e9c2e57ec12eca26f35c8f4259bbcc55cdb lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -1390,6 +1390,140 @@ return action_dict +class SetupPythonEnvironment( Download, RecipeStep ): + + def __init__( self, app ): + self.app = app + self.type = 'setup_python_environment' + + def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, + install_environment, work_dir, current_dir=None, initial_download=False ): + """ + Initialize the environment for installing Python packages. The class is called during the initial + download stage when installing packages, so the value of initial_download will generally be True. + However, the parameter value allows this class to also be used in the second stage of the installation, + although it may never be necessary. If initial_download is True, the recipe steps will be filtered + and returned and the installation directory (i.e., dir) will be defined and returned. If we're not + in the initial download stage, these actions will not occur, and None values will be returned for them. + + Warning: easy_install is configured that it will not be install any dependency, the tool developer needs + to specify every dependency explicitly + """ + # <action type="setup_python_environment"> + # <repository name="package_python_2_7" owner="bgruening"> + # <package name="python" version="2.7" /> + # </repository> + # <!-- allow downloading and installing a Python package from https://pypi.python.org/ --> + # <package>pysam.tar.gz</package> + # <package>http://url-to-some-python-package.de/pysam.tar.gz</package> + # </action> + dir = None + if initial_download: + filtered_actions = actions[ 1: ] + env_shell_file_paths = action_dict.get( 'env_shell_file_paths', None ) + log.debug( '\n%s\n' % env_shell_file_paths ) + if env_shell_file_paths is None: + log.debug( 'Missing Python environment, make sure your specified Python installation exists.' ) + if initial_download: + return tool_dependency, filtered_actions, dir + return tool_dependency, None, None + else: + install_environment.add_env_shell_file_paths( env_shell_file_paths ) + log.debug( 'Handling setup_python_environment for tool dependency %s with install_environment.env_shell_file_paths:\n%s' % \ + ( str( tool_dependency.name ), str( install_environment.env_shell_file_paths ) ) ) + dir = os.path.curdir + current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) + with lcd( current_dir ): + with settings( warn_only=True ): + python_package_tups = action_dict.get( 'python_package_tups', [] ) + for python_package_tup in python_package_tups: + package, package_version = python_package_tup + package_path = os.path.join( install_environment.tool_shed_repository_install_dir, package ) + if os.path.isfile( package_path ): + # we assume a local shipped python package + + cmd = r'''PATH=$PATH:$PYTHONHOME/bin; export PATH; + export PYTHONPATH=$PYTHONPATH:$INSTALL_DIR; + easy_install --no-deps --install-dir $INSTALL_DIR --script-dir $INSTALL_DIR/bin %s + ''' % ( package_path ) + elif package.find( '://' ) != -1: + # We assume a URL to a python package. + url = package + package_name = url.split( '/' )[ -1 ] + self.url_download( work_dir, package_name, url, extract=False ) + + cmd = r'''PATH=$PATH:$PYTHONHOME/bin; export PATH; + export PYTHONPATH=$PYTHONPATH:$INSTALL_DIR; + easy_install --no-deps --install-dir $INSTALL_DIR --script-dir $INSTALL_DIR/bin %s + ''' % ( package_name ) + else: + pass + # pypi can be implemented or for > python3.4 we can use the build-in system + cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) ) + return_code = install_environment.handle_command( tool_dependency=tool_dependency, + cmd=cmd, + return_output=False ) + if return_code: + if initial_download: + return tool_dependency, filtered_actions, dir + return tool_dependency, None, None + # Pull in python dependencies (runtime). + env_file_builder.handle_action_shell_file_paths( action_dict ) + env_file_builder.append_line( name="PYTHONPATH", + action="prepend_to", + value= os.path.join( install_environment.install_dir, 'lib', 'python') ) + env_file_builder.append_line( name="PATH", + action="prepend_to", + value=os.path.join( install_environment.install_dir, 'bin' ) ) + return_code = env_file_builder.return_code + if return_code: + if initial_download: + return tool_dependency, filtered_actions, dir + return tool_dependency, None, None + if initial_download: + return tool_dependency, filtered_actions, dir + return tool_dependency, None, None + + def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): + # setup a Python environment. + # <action type="setup_python_environment"> + # <repository name="package_python_2_7" owner="bgruening"> + # <package name="python" version="2.7" /> + # </repository> + # <!-- allow downloading and installing an Python package from https://pypi.org/ --> + # <package>pysam.tar.gz</package> + # <package>http://url-to-some-python-package.de/pysam.tar.gz</package> + # </action> + # Discover all child repository dependency tags and define the path to an env.sh file + # associated with each repository. This will potentially update the value of the + # 'env_shell_file_paths' entry in action_dict. + all_env_shell_file_paths = [] + env_manager = EnvManager( self.app ) + action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( all_env_shell_file_paths, + action_elem, + action_dict ) + python_package_tups = [] + for env_elem in action_elem: + if env_elem.tag == 'package': + #A valid package definitions can be: + # pysam.tar.gz -> locally shipped tarball + # ftp://ftp.gruening.de/pysam.tar.gz -> online tarball + python_token = env_elem.text.strip().split( '=' ) + if len( python_token ) == 2: + # version string + package_name = python_token[ 0 ] + package_version = python_token[ 1 ] + python_package_tups.append( ( package_name, package_version ) ) + else: + # package name for pypi.org without version number + package = env_elem.text.strip() + python_package_tups.append( ( package, None ) ) + if python_package_tups: + action_dict[ 'python_package_tups' ] = python_package_tups + return action_dict + + + class SetupVirtualEnv( Download, RecipeStep ): def __init__( self, app ): https://bitbucket.org/galaxy/galaxy-central/commits/4411a021a9fa/ Changeset: 4411a021a9fa User: dannon Date: 2014-07-09 17:35:51 Summary: Merged in BjoernGruening/galaxy-central-bgruening/setup_python_env (pull request #431) Add new install_python_environment installation tag to the Tool Shed. Affected #: 4 files diff -r d1d5fcd5bec6d4d48908f6da58a31ca9242e694e -r 4411a021a9fad918cef26e00adf6f49e247fb0bd lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -44,7 +44,8 @@ def __init__( self, app ): self.app = app - self.INSTALL_ACTIONS = [ 'download_binary', 'download_by_url', 'download_file', 'setup_perl_environment', + self.INSTALL_ACTIONS = [ 'download_binary', 'download_by_url', 'download_file', + 'setup_perl_environment', 'setup_python_environment', 'setup_r_environment', 'setup_ruby_environment', 'shell_command' ] def format_traceback( self ): diff -r d1d5fcd5bec6d4d48908f6da58a31ca9242e694e -r 4411a021a9fad918cef26e00adf6f49e247fb0bd lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py @@ -124,7 +124,7 @@ Parse an XML tag set to discover all child repository dependency tags and define the path to an env.sh file associated with the repository (this requires the repository dependency to be in an installed state). The received action_dict will be updated with these discovered paths and returned to the caller. This method handles tool dependency definition - tag sets <setup_r_environment>, <setup_ruby_environment> and <setup_perl_environment>. + tag sets <setup_r_environment>, <setup_ruby_environment>, <setup_python_environment> and <setup_perl_environment>. """ # An example elem is: # <action type="setup_perl_environment"> diff -r d1d5fcd5bec6d4d48908f6da58a31ca9242e694e -r 4411a021a9fad918cef26e00adf6f49e247fb0bd lib/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py @@ -54,6 +54,7 @@ setup_perl_environment=step_handler.SetupPerlEnvironment( self.app ), setup_r_environment=step_handler.SetupREnvironment( self.app ), setup_ruby_environment=step_handler.SetupRubyEnvironment( self.app ), + setup_python_environment=step_handler.SetupPythonEnvironment( self.app ), setup_virtualenv=step_handler.SetupVirtualEnv( self.app ), shell_command=step_handler.ShellCommand( self.app ), template_command=step_handler.TemplateCommand( self.app ) ) diff -r d1d5fcd5bec6d4d48908f6da58a31ca9242e694e -r 4411a021a9fad918cef26e00adf6f49e247fb0bd lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -1390,6 +1390,140 @@ return action_dict +class SetupPythonEnvironment( Download, RecipeStep ): + + def __init__( self, app ): + self.app = app + self.type = 'setup_python_environment' + + def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, + install_environment, work_dir, current_dir=None, initial_download=False ): + """ + Initialize the environment for installing Python packages. The class is called during the initial + download stage when installing packages, so the value of initial_download will generally be True. + However, the parameter value allows this class to also be used in the second stage of the installation, + although it may never be necessary. If initial_download is True, the recipe steps will be filtered + and returned and the installation directory (i.e., dir) will be defined and returned. If we're not + in the initial download stage, these actions will not occur, and None values will be returned for them. + + Warning: easy_install is configured that it will not be install any dependency, the tool developer needs + to specify every dependency explicitly + """ + # <action type="setup_python_environment"> + # <repository name="package_python_2_7" owner="bgruening"> + # <package name="python" version="2.7" /> + # </repository> + # <!-- allow downloading and installing a Python package from https://pypi.python.org/ --> + # <package>pysam.tar.gz</package> + # <package>http://url-to-some-python-package.de/pysam.tar.gz</package> + # </action> + dir = None + if initial_download: + filtered_actions = actions[ 1: ] + env_shell_file_paths = action_dict.get( 'env_shell_file_paths', None ) + log.debug( '\n%s\n' % env_shell_file_paths ) + if env_shell_file_paths is None: + log.debug( 'Missing Python environment, make sure your specified Python installation exists.' ) + if initial_download: + return tool_dependency, filtered_actions, dir + return tool_dependency, None, None + else: + install_environment.add_env_shell_file_paths( env_shell_file_paths ) + log.debug( 'Handling setup_python_environment for tool dependency %s with install_environment.env_shell_file_paths:\n%s' % \ + ( str( tool_dependency.name ), str( install_environment.env_shell_file_paths ) ) ) + dir = os.path.curdir + current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) + with lcd( current_dir ): + with settings( warn_only=True ): + python_package_tups = action_dict.get( 'python_package_tups', [] ) + for python_package_tup in python_package_tups: + package, package_version = python_package_tup + package_path = os.path.join( install_environment.tool_shed_repository_install_dir, package ) + if os.path.isfile( package_path ): + # we assume a local shipped python package + + cmd = r'''PATH=$PATH:$PYTHONHOME/bin; export PATH; + export PYTHONPATH=$PYTHONPATH:$INSTALL_DIR; + easy_install --no-deps --install-dir $INSTALL_DIR --script-dir $INSTALL_DIR/bin %s + ''' % ( package_path ) + elif package.find( '://' ) != -1: + # We assume a URL to a python package. + url = package + package_name = url.split( '/' )[ -1 ] + self.url_download( work_dir, package_name, url, extract=False ) + + cmd = r'''PATH=$PATH:$PYTHONHOME/bin; export PATH; + export PYTHONPATH=$PYTHONPATH:$INSTALL_DIR; + easy_install --no-deps --install-dir $INSTALL_DIR --script-dir $INSTALL_DIR/bin %s + ''' % ( package_name ) + else: + pass + # pypi can be implemented or for > python3.4 we can use the build-in system + cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) ) + return_code = install_environment.handle_command( tool_dependency=tool_dependency, + cmd=cmd, + return_output=False ) + if return_code: + if initial_download: + return tool_dependency, filtered_actions, dir + return tool_dependency, None, None + # Pull in python dependencies (runtime). + env_file_builder.handle_action_shell_file_paths( action_dict ) + env_file_builder.append_line( name="PYTHONPATH", + action="prepend_to", + value= os.path.join( install_environment.install_dir, 'lib', 'python') ) + env_file_builder.append_line( name="PATH", + action="prepend_to", + value=os.path.join( install_environment.install_dir, 'bin' ) ) + return_code = env_file_builder.return_code + if return_code: + if initial_download: + return tool_dependency, filtered_actions, dir + return tool_dependency, None, None + if initial_download: + return tool_dependency, filtered_actions, dir + return tool_dependency, None, None + + def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): + # setup a Python environment. + # <action type="setup_python_environment"> + # <repository name="package_python_2_7" owner="bgruening"> + # <package name="python" version="2.7" /> + # </repository> + # <!-- allow downloading and installing an Python package from https://pypi.org/ --> + # <package>pysam.tar.gz</package> + # <package>http://url-to-some-python-package.de/pysam.tar.gz</package> + # </action> + # Discover all child repository dependency tags and define the path to an env.sh file + # associated with each repository. This will potentially update the value of the + # 'env_shell_file_paths' entry in action_dict. + all_env_shell_file_paths = [] + env_manager = EnvManager( self.app ) + action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( all_env_shell_file_paths, + action_elem, + action_dict ) + python_package_tups = [] + for env_elem in action_elem: + if env_elem.tag == 'package': + #A valid package definitions can be: + # pysam.tar.gz -> locally shipped tarball + # ftp://ftp.gruening.de/pysam.tar.gz -> online tarball + python_token = env_elem.text.strip().split( '=' ) + if len( python_token ) == 2: + # version string + package_name = python_token[ 0 ] + package_version = python_token[ 1 ] + python_package_tups.append( ( package_name, package_version ) ) + else: + # package name for pypi.org without version number + package = env_elem.text.strip() + python_package_tups.append( ( package, None ) ) + if python_package_tups: + action_dict[ 'python_package_tups' ] = python_package_tups + return action_dict + + + class SetupVirtualEnv( Download, RecipeStep ): def __init__( self, app ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.