10 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/08ca76c1804d/ Changeset: 08ca76c1804d User: BjoernGruening Date: 2013-11-14 17:39:49 Summary: add setup_perl_environment as Tool Shed functionality Affected #: 2 files diff -r 516df888b97a164b7ff2d8235f5d4ea6462e185d -r 08ca76c1804d9307feb3db95fdcecfe0d74af20b lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -441,6 +441,99 @@ return_code = file_append( env_entry, env_file, skip_if_contained=True, make_executable=True ) if return_code: return + elif action_type == 'setup_perl_environment': + # setup an Perl environment + # <action type="setup_perl_environment"> + # <repository name="package_perl_5_18" owner="bgruening"> + # <package name="perl" version="5.18.1" /> + # </repository> + # <!-- allow downloading and installing an Perl package from cpan.org--> + # <package>XML::Parser</package> + # <package>http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz</package> + # </action> + filtered_actions = actions[ 1: ] + + if action_dict.get( 'env_shell_file_paths', False ): + install_environment.add_env_shell_file_paths( action_dict[ 'env_shell_file_paths' ] ) + else: + log.warning( 'Missing Rerl environment. Please check if your specified Rerl installation exists.' ) + return + + dir = os.path.curdir + current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) + with lcd( current_dir ): + with settings( warn_only=True ): + + for package in action_dict[ 'perl_packages' ]: + """ + If set to a true value then MakeMaker's prompt function will always return the default without waiting for user input. + """ + cmd = '''export PERL_MM_USE_DEFAULT=1 && ''' + + if package.find('://') != -1: + # we assume a URL to a gem file + url = package + package_name = url.split( '/' )[ -1 ] + dir = td_common_util.url_download( work_dir, package_name, url, extract=True ) + # search for Build.PL or Makefile.PL (ExtUtils::MakeMaker vs. Module::Build) + + tmp_work_dir = os.path.join( work_dir, dir) + if os.path.exists( os.path.join( tmp_work_dir, 'Makefile.PL' ) ): + + cmd += '''perl Makefile.PL INSTALL_BASE=$INSTALL_DIR && make && make install''' + elif os.path.exists( os.path.join( tmp_work_dir, 'Build.PL' ) ): + cmd += '''perl Build.PL --install_base $INSTALL_DIR && perl Build && perl Build install''' + else: + log.warning( 'No Makefile.PL or Build.PL file found in %s. Skip installation of %s.' % ( url, package_name ) ) + return + with lcd( tmp_work_dir ): + cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_dir ) ) + return_code = handle_command( app, tool_dependency, install_dir, cmd ) + if return_code: + return + else: + # perl package from CPAN without version number + # cpanm should be installed with the parent perl distribution, otherwise this will not work + cmd += '''cpanm --local-lib=$INSTALL_DIR %s''' % ( package ) + + cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_dir ) ) + return_code = handle_command( app, tool_dependency, install_dir, cmd ) + if return_code: + return + + # Ruby libraries are installed to $INSTALL_DIR (install_dir), we now set the GEM_PATH path to that directory + # TODO: That code is used a lot for the different environments and should be refactored, once the environments are integrated + installed_env_dict = install_environment.environment_dict() + perl5lib_path = installed_env_dict.get('PERL5LIB', False) + perlbin_path = installed_env_dict.get('PATH', False) + + if not perl5lib_path or not perlbin_path: + log.warning( 'Missing RRUBYLIB or/and RUBY_HOME environment variable. Please check if your specified Ruby installation is valid.' ) + + modify_env_command_dict = dict( name="PATH", action="set_to", value=perlbin_path ) + env_entry, env_file = td_common_util.create_or_update_env_shell_file( install_dir, modify_env_command_dict ) + return_code = file_append( env_entry, env_file, skip_if_contained=True, make_executable=True ) + if return_code: + return + + modify_env_command_dict = dict( name="PERL5LIB", action="set_to", value=perl5lib_path ) + env_entry, env_file = td_common_util.create_or_update_env_shell_file( install_dir, modify_env_command_dict ) + return_code = file_append( env_entry, env_file, skip_if_contained=True, make_executable=True ) + if return_code: + return + + modify_env_command_dict = dict( name="PERL5LIB", action="prepend_to", value=os.path.join(install_dir, 'lib', 'perl5') ) + env_entry, env_file = td_common_util.create_or_update_env_shell_file( install_dir, modify_env_command_dict ) + return_code = file_append( env_entry, env_file, skip_if_contained=True, make_executable=True ) + if return_code: + return + + modify_env_command_dict = dict( name="PATH", action="prepend_to", value=os.path.join(install_dir, 'bin') ) + env_entry, env_file = td_common_util.create_or_update_env_shell_file( install_dir, modify_env_command_dict ) + return_code = file_append( env_entry, env_file, skip_if_contained=True, make_executable=True ) + if return_code: + return + else: # We're handling a complex repository dependency where we only have a set_environment tag set. diff -r 516df888b97a164b7ff2d8235f5d4ea6462e185d -r 08ca76c1804d9307feb3db95fdcecfe0d74af20b lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -719,6 +719,36 @@ action_dict[ 'ruby_packages' ] = ruby_packages else: continue + elif action_type == 'setup_perl_environment': + # setup an Perl environment + # <action type="setup_perl_environment"> + # <repository name="package_perl_5_18" owner="bgruening"> + # <package name="perl" version="5.18.1" /> + # </repository> + # <!-- allow downloading and installing an Perl package from cpan.org--> + # <package>XML::Parser</package> + # <package>http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz</package> + # </action> + + env_shell_file_paths = td_common_util.get_env_shell_file_paths( app, action_elem.find('repository') ) + all_env_shell_file_paths.extend( env_shell_file_paths ) + if all_env_shell_file_paths: + action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths + perl_packages = list() + for env_elem in action_elem: + if env_elem.tag == 'package': + """ + A valid package definition can be: + XML::Parser + http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz + Unfortunately, CPAN does not support versioning. If you want real Reproducibility, + you need to specify the tarball path and the right order of different tarballs manually. + """ + perl_packages.append( env_elem.text.strip() ) + if perl_packages: + action_dict[ 'perl_packages' ] = perl_packages + else: + continue elif action_type == 'make_install': # make; make install; allow providing make options if action_elem.text: https://bitbucket.org/galaxy/galaxy-central/commits/bfc098707375/ Changeset: bfc098707375 User: BjoernGruening Date: 2013-11-14 17:39:49 Summary: fix c&p leftovers in comments Affected #: 1 file diff -r 08ca76c1804d9307feb3db95fdcecfe0d74af20b -r bfc098707375ff25cad706a9929ac29aef84e38c lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -501,14 +501,14 @@ if return_code: return - # Ruby libraries are installed to $INSTALL_DIR (install_dir), we now set the GEM_PATH path to that directory + # Perl libraries are installed to $INSTALL_DIR (install_dir), we now set the PERL5LIB path to that directory # TODO: That code is used a lot for the different environments and should be refactored, once the environments are integrated installed_env_dict = install_environment.environment_dict() perl5lib_path = installed_env_dict.get('PERL5LIB', False) perlbin_path = installed_env_dict.get('PATH', False) if not perl5lib_path or not perlbin_path: - log.warning( 'Missing RRUBYLIB or/and RUBY_HOME environment variable. Please check if your specified Ruby installation is valid.' ) + log.warning( 'Missing PERL5LIB or/and PATH environment variable. Please check if your specified Perl installation is valid.' ) modify_env_command_dict = dict( name="PATH", action="set_to", value=perlbin_path ) env_entry, env_file = td_common_util.create_or_update_env_shell_file( install_dir, modify_env_command_dict ) https://bitbucket.org/galaxy/galaxy-central/commits/78ebe94f1492/ Changeset: 78ebe94f1492 User: jmchilton Date: 2013-11-14 17:39:49 Summary: Introduce utility method parse_setup_environment_repositories. Use to reduce code duplication in install_util and provides target for unit tests of new functionality. Add unit tests. Add new functionality for parsing out which packages should be recursive runtime dependencies (not implemented on fabric side yet). Affected #: 4 files diff -r bfc098707375ff25cad706a9929ac29aef84e38c -r 78ebe94f1492e145b298539042a9d9f68e5f0ae1 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -137,6 +137,11 @@ ElementInclude.include(root) return tree + +def parse_xml_string(xml_string): + tree = ElementTree.fromstring(xml_string) + return tree + def xml_to_string( elem, pretty=False ): """Returns a string from an xml tree""" if pretty: diff -r bfc098707375ff25cad706a9929ac29aef84e38c -r 78ebe94f1492e145b298539042a9d9f68e5f0ae1 lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -665,11 +665,7 @@ # <!-- allow installing an R packages --> # <package>https://github.com/bgruening/download_store/raw/master/DESeq2-1_0_18/BiocGenerics_0.6.0.tar.gz</package> # </action> - env_shell_file_paths = td_common_util.get_env_shell_file_paths( app, action_elem.find('repository') ) - - all_env_shell_file_paths.extend( env_shell_file_paths ) - if all_env_shell_file_paths: - action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths + td_common_util.parse_setup_environment_repositories( app, all_env_shell_file_paths, action_elem, action_dict ) r_packages = list() for env_elem in action_elem: if env_elem.tag == 'package': @@ -690,11 +686,7 @@ # <package>protk=1.2.4</package> # <package>http://url-to-some-gem-file.de/protk.gem</package> # </action> - - env_shell_file_paths = td_common_util.get_env_shell_file_paths( app, action_elem.find('repository') ) - all_env_shell_file_paths.extend( env_shell_file_paths ) - if all_env_shell_file_paths: - action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths + td_common_util.parse_setup_environment_repositories( app, all_env_shell_file_paths, action_elem, action_dict ) ruby_packages = list() for env_elem in action_elem: if env_elem.tag == 'package': diff -r bfc098707375ff25cad706a9929ac29aef84e38c -r 78ebe94f1492e145b298539042a9d9f68e5f0ae1 lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py @@ -193,6 +193,7 @@ return os.path.abspath( os.path.join( root, name ) ) return None + def get_env_shell_file_paths( app, elem ): # Currently only the following tag set is supported. # <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056"> @@ -383,7 +384,7 @@ # platform. Append the child element to the list of elements to process. actions_elem_list.append( child_element ) elif child_element.tag == 'action': - # Any <action> tags within an <actions_group> tag set must come after all <actions> tags. + # Any <action> tags within an <actions_group> tag set must come after all <actions> tags. if actions_elems_processed == actions_elem_count: # If all <actions> elements have been processed, then this <action> element can be appended to the list of actions to # execute within this group. @@ -409,6 +410,16 @@ continue return actions_elem_tuples + +def parse_setup_environment_repositories( app, all_env_shell_file_paths, action_elem, action_dict ): + env_shell_file_paths = get_env_shell_file_paths( app, action_elem.find('repository') ) + + all_env_shell_file_paths.extend( env_shell_file_paths ) + if all_env_shell_file_paths: + action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths + action_dict[ 'action_shell_file_paths' ] = env_shell_file_paths + + def url_download( install_dir, downloaded_file_name, download_url, extract=True ): file_path = os.path.join( install_dir, downloaded_file_name ) src = None diff -r bfc098707375ff25cad706a9929ac29aef84e38c -r 78ebe94f1492e145b298539042a9d9f68e5f0ae1 test/unit/tool_shed/test_td_common_util.py --- /dev/null +++ b/test/unit/tool_shed/test_td_common_util.py @@ -0,0 +1,57 @@ +from contextlib import contextmanager +from galaxy.util import parse_xml_string + +from tool_shed.galaxy_install.tool_dependencies import td_common_util + + +class MockApp( object ): + + def __init__( self ): + pass + + +def test_parse_setup_environment_repositories( ): + xml = """<action name="setup_r_environment"> + <repository name="package_r_3_0_1" owner="bgruening" toolshed="toolshed.g2.bx.psu.edu" changeset_revision="1234567"> + <package name="R" version="3.0.1" /> + </repository> + </action> + """ + mock_app = MockApp() + action_elem = parse_xml_string( xml ) + required_for_install_env_sh = '/path/to/existing.sh' + all_env_paths = [ required_for_install_env_sh ] + action_dict = {} + + r_env_sh = '/path/to/go/env.sh' + + def mock_get_env_shell_file_paths( app, elem): + assert app == mock_app + assert elem.get( 'name' ) == "package_r_3_0_1" + return [ r_env_sh ] + + with __mock_common_util_method("get_env_shell_file_paths", mock_get_env_shell_file_paths): + td_common_util.parse_setup_environment_repositories( mock_app, all_env_paths, action_elem, action_dict ) + ## Verify old env files weren't deleted. + assert required_for_install_env_sh in all_env_paths + ## Verify new ones added. + assert r_env_sh in all_env_paths + ## env_shell_file_paths includes everything + assert all( [env in action_dict[ 'env_shell_file_paths' ] for env in all_env_paths] ) + + ## action_shell_file_paths includes only env files defined in + ## inside the setup_ action element. + assert required_for_install_env_sh not in action_dict[ 'action_shell_file_paths' ] + assert r_env_sh in action_dict[ 'action_shell_file_paths' ] + + +## Poor man's mocking. Need to get a real mocking library as real Galaxy development +## dependnecy. +@contextmanager +def __mock_common_util_method(name, mock_method): + real_method = getattr(td_common_util, name) + try: + setattr(td_common_util, name, mock_method) + yield + finally: + setattr(td_common_util, name, real_method) https://bitbucket.org/galaxy/galaxy-central/commits/443628e7b555/ Changeset: 443628e7b555 User: jmchilton Date: 2013-11-14 17:39:49 Summary: Update perl to use parse_setup_environment_repositories. Affected #: 1 file diff -r 78ebe94f1492e145b298539042a9d9f68e5f0ae1 -r 443628e7b555600d33937bebe0ae03f637650025 lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -721,11 +721,7 @@ # <package>XML::Parser</package> # <package>http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz</package> # </action> - - env_shell_file_paths = td_common_util.get_env_shell_file_paths( app, action_elem.find('repository') ) - all_env_shell_file_paths.extend( env_shell_file_paths ) - if all_env_shell_file_paths: - action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths + td_common_util.parse_setup_environment_repositories( app, all_env_shell_file_paths, action_elem, action_dict ) perl_packages = list() for env_elem in action_elem: if env_elem.tag == 'package': https://bitbucket.org/galaxy/galaxy-central/commits/bc18a1706c12/ Changeset: bc18a1706c12 User: jmchilton Date: 2013-11-14 17:39:49 Summary: Add unit test for all existing functionality in td_common_util.create_or_update_env_shell_file Affected #: 1 file diff -r 443628e7b555600d33937bebe0ae03f637650025 -r bc18a1706c12849df23985c0fc5f04feff5bd956 test/unit/tool_shed/test_td_common_util.py --- a/test/unit/tool_shed/test_td_common_util.py +++ b/test/unit/tool_shed/test_td_common_util.py @@ -1,15 +1,35 @@ +from os.path import join from contextlib import contextmanager from galaxy.util import parse_xml_string from tool_shed.galaxy_install.tool_dependencies import td_common_util +TEST_DEPENDENCIES_DIR = "/opt/galaxy/dependencies" +TEST_INSTALL_DIR = "%s/test_install_dir" % TEST_DEPENDENCIES_DIR + + class MockApp( object ): def __init__( self ): pass +def test_create_or_update_env_shell_file( ): + test_path = "/usr/share/R/libs" + line, path = td_common_util.create_or_update_env_shell_file( TEST_INSTALL_DIR, dict(action="append_to", name="R_LIBS", value=test_path)) + assert path == join( TEST_INSTALL_DIR, "env.sh" ) + assert line == "R_LIBS=$R_LIBS:/usr/share/R/libs; export R_LIBS" + + line, path = td_common_util.create_or_update_env_shell_file( TEST_INSTALL_DIR, dict(action="prepend_to", name="R_LIBS", value=test_path)) + assert path == join( TEST_INSTALL_DIR, "env.sh" ) + assert line == "R_LIBS=/usr/share/R/libs:$R_LIBS; export R_LIBS" + + line, path = td_common_util.create_or_update_env_shell_file( TEST_INSTALL_DIR, dict(action="set_to", name="R_LIBS", value=test_path)) + assert path == join( TEST_INSTALL_DIR, "env.sh" ) + assert line == "R_LIBS=/usr/share/R/libs; export R_LIBS" + + def test_parse_setup_environment_repositories( ): xml = """<action name="setup_r_environment"><repository name="package_r_3_0_1" owner="bgruening" toolshed="toolshed.g2.bx.psu.edu" changeset_revision="1234567"> https://bitbucket.org/galaxy/galaxy-central/commits/8e7cface9bf1/ Changeset: 8e7cface9bf1 User: jmchilton Date: 2013-11-14 17:39:49 Summary: Add new action to create_or_update_env_shell_file for sourcing other env.sh files. With unit test. Affected #: 2 files diff -r bc18a1706c12849df23985c0fc5f04feff5bd956 -r 8e7cface9bf1060c41ebbda99e025572abbafe5b lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py @@ -146,16 +146,21 @@ return None def create_or_update_env_shell_file( install_dir, env_var_dict ): - env_var_name = env_var_dict[ 'name' ] env_var_action = env_var_dict[ 'action' ] env_var_value = env_var_dict[ 'value' ] - if env_var_action == 'prepend_to': - changed_value = '%s:$%s' % ( env_var_value, env_var_name ) - elif env_var_action == 'set_to': - changed_value = '%s' % env_var_value - elif env_var_action == 'append_to': - changed_value = '$%s:%s' % ( env_var_name, env_var_value ) - line = "%s=%s; export %s" % ( env_var_name, changed_value, env_var_name ) + if env_var_action in ['prepend_to', 'set_to', 'append_to']: + env_var_name = env_var_dict[ 'name' ] + if env_var_action == 'prepend_to': + changed_value = '%s:$%s' % ( env_var_value, env_var_name ) + elif env_var_action == 'set_to': + changed_value = '%s' % env_var_value + elif env_var_action == 'append_to': + changed_value = '$%s:%s' % ( env_var_name, env_var_value ) + line = "%s=%s; export %s" % ( env_var_name, changed_value, env_var_name ) + elif env_var_action == "source": + line = ". %s" % env_var_value + else: + raise Exception( "Unknown shell file action %s" % env_var_action ) env_shell_file_path = os.path.join( install_dir, 'env.sh' ) return line, env_shell_file_path diff -r bc18a1706c12849df23985c0fc5f04feff5bd956 -r 8e7cface9bf1060c41ebbda99e025572abbafe5b test/unit/tool_shed/test_td_common_util.py --- a/test/unit/tool_shed/test_td_common_util.py +++ b/test/unit/tool_shed/test_td_common_util.py @@ -29,6 +29,10 @@ assert path == join( TEST_INSTALL_DIR, "env.sh" ) assert line == "R_LIBS=/usr/share/R/libs; export R_LIBS" + line, path = td_common_util.create_or_update_env_shell_file( TEST_INSTALL_DIR, dict(action="source", value=test_path)) + assert path == join( TEST_INSTALL_DIR, "env.sh" ) + assert line == ". /usr/share/R/libs" + def test_parse_setup_environment_repositories( ): xml = """<action name="setup_r_environment"> https://bitbucket.org/galaxy/galaxy-central/commits/cf33c63f9a9b/ Changeset: cf33c63f9a9b User: jmchilton Date: 2013-11-14 17:39:49 Summary: Introduce higher level abstraction for building up env.sh files in fabric_util.py. Use it to reduced duplicated code in fabric_util.py. Implement unit tests describing this new class. Affected #: 2 files diff -r 8e7cface9bf1060c41ebbda99e025572abbafe5b -r cf33c63f9a9b14b395d02d459ae6e30236236a67 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -200,6 +200,20 @@ return True +class EnvFileBuilder( object ): + + def __init__( self, install_dir ): + self.install_dir = install_dir + self.return_code = 0 + + def append_line( self, skip_if_contained=True, make_executable=True, **kwds ): + env_var_dict = dict(**kwds) + env_entry, env_file = td_common_util.create_or_update_env_shell_file( self.install_dir, env_var_dict ) + return_code = file_append( env_entry, env_file, skip_if_contained=skip_if_contained, make_executable=make_executable ) + self.return_code = self.return_code or return_code + return self.return_code + + class InstallEnvironment( object ): """ Object describing the environment built up as part of the process of building @@ -373,11 +387,9 @@ return # R libraries are installed to $INSTALL_DIR (install_dir), we now set the R_LIBS path to that directory - # TODO: That code is used a lot for the different environments and should be refactored, once the environments are integrated - modify_env_command_dict = dict( name="R_LIBS", action="prepend_to", value=install_dir ) - env_entry, env_file = td_common_util.create_or_update_env_shell_file( install_dir, modify_env_command_dict ) - return_code = file_append( env_entry, env_file, skip_if_contained=True, make_executable=True ) - + env_file_builder = EnvFileBuilder( install_dir ) + env_file_builder.append_line( name="R_LIBS", action="prepend_to", value=install_dir ) + return_code = env_file_builder.return_code if return_code: return elif action_type == 'setup_ruby_environment': @@ -428,17 +440,10 @@ if return_code: return - # Ruby libraries are installed to $INSTALL_DIR (install_dir), we now set the GEM_PATH path to that directory - # TODO: That code is used a lot for the different environments and should be refactored, once the environments are integrated - modify_env_command_dict = dict( name="GEM_PATH", action="prepend_to", value=install_dir ) - env_entry, env_file = td_common_util.create_or_update_env_shell_file( install_dir, modify_env_command_dict ) - return_code = file_append( env_entry, env_file, skip_if_contained=True, make_executable=True ) - if return_code: - return - - modify_env_command_dict = dict( name="PATH", action="prepend_to", value=os.path.join(install_dir, 'bin') ) - env_entry, env_file = td_common_util.create_or_update_env_shell_file( install_dir, modify_env_command_dict ) - return_code = file_append( env_entry, env_file, skip_if_contained=True, make_executable=True ) + env_file_builder = EnvFileBuilder( install_dir ) + env_file_builder.append_line( name="GEM_PATH", action="prepend_to", value=install_dir ) + env_file_builder.append_line( name="PATH", action="prepend_to", value=os.path.join(install_dir, 'bin') ) + return_code = env_file_builder.return_code if return_code: return elif action_type == 'setup_perl_environment': @@ -573,13 +578,14 @@ # in the set_environment action. cmds = install_environment.environment_commands( 'set_environment' ) env_var_dicts = action_dict[ 'environment_variable' ] + env_file_builder = EnvFileBuilder( install_dir ) for env_var_dict in env_var_dicts: # Check for the presence of the $ENV[] key string and populate it if possible. env_var_dict = handle_environment_variables( app, tool_dependency, install_dir, env_var_dict, cmds ) - env_entry, env_file = td_common_util.create_or_update_env_shell_file( install_dir, env_var_dict ) - return_code = file_append( env_entry, env_file, skip_if_contained=True, make_executable=True ) - if return_code: - return + env_file_builder.append_line( **env_var_dict ) + return_code = env_file_builder.return_code + if return_code: + return elif action_type == 'set_environment_for_install': # Currently the only action supported in this category is a list of paths to one or more tool dependency env.sh files, # the environment setting in each of which will be injected into the environment for all <action type="shell_command"> @@ -618,14 +624,10 @@ if not os.path.exists( output.stdout ): log.error( "virtualenv's site-packages directory '%s' does not exist", output.stdout ) return - modify_env_command_dict = dict( name="PYTHONPATH", action="prepend_to", value=output.stdout ) - env_entry, env_file = td_common_util.create_or_update_env_shell_file( install_dir, modify_env_command_dict ) - return_code = file_append( env_entry, env_file, skip_if_contained=True, make_executable=True ) - if return_code: - return - modify_env_command_dict = dict( name="PATH", action="prepend_to", value=os.path.join( venv_directory, "bin" ) ) - env_entry, env_file = td_common_util.create_or_update_env_shell_file( install_dir, modify_env_command_dict ) - return_code = file_append( env_entry, env_file, skip_if_contained=True, make_executable=True ) + env_file_builder = EnvFileBuilder( install_dir ) + env_file_builder.append_line( name="PYTHONPATH", action="prepend_to", value=output.stdout ) + env_file_builder.append_line( name="PATH", action="prepend_to", value=os.path.join( venv_directory, "bin" ) ) + return_code = env_file_builder.return_code if return_code: return elif action_type == 'shell_command': diff -r 8e7cface9bf1060c41ebbda99e025572abbafe5b -r cf33c63f9a9b14b395d02d459ae6e30236236a67 test/unit/tool_shed/test_fabric_util.py --- /dev/null +++ b/test/unit/tool_shed/test_fabric_util.py @@ -0,0 +1,45 @@ +from contextlib import contextmanager +from tool_shed.galaxy_install.tool_dependencies import fabric_util + + +def test_env_file_builder(): + install_dir = "/opt/galaxy/dependencies/foo/" + env_file_builder = fabric_util.EnvFileBuilder( install_dir ) + added_lines = [] + mock_return = dict(value=0) + + def mock_file_append( text, file_path, **kwds ): + added_lines.append(text) + return mock_return["value"] + + with __mock_fabric_util_method("file_append", mock_file_append): + env_file_builder.append_line( name="PATH", action="prepend_to", value="/usr/bin/local/R" ) + assert added_lines == [ "PATH=/usr/bin/local/R:$PATH; export PATH" ] + assert env_file_builder.return_code == 0 + + # Reset mock lines + del added_lines[:] + # Next time file_append will fail + mock_return["value"] = 1 + + env_file_builder.append_line( action="source", value="/usr/bin/local/R/env.sh" ) + assert added_lines == [ ". /usr/bin/local/R/env.sh" ] + # Check failure + assert env_file_builder.return_code == 1 + + mock_return["value"] = 0 + env_file_builder.append_line( name="LD_LIBRARY_PATH", action="append_to", value="/usr/bin/local/R/lib" ) + # Verify even though last append succeeded, previous failure still recorded. + assert env_file_builder.return_code == 1 + + +## Poor man's mocking. Need to get a real mocking library as real Galaxy development +## dependnecy. +@contextmanager +def __mock_fabric_util_method(name, mock_method): + real_method = getattr(fabric_util, name) + try: + setattr(fabric_util, name, mock_method) + yield + finally: + setattr(fabric_util, name, real_method) https://bitbucket.org/galaxy/galaxy-central/commits/27edf6bd91d1/ Changeset: 27edf6bd91d1 User: jmchilton Date: 2013-11-14 17:39:49 Summary: Pull in dependant R, Ruby, and Perl complete environments at recursively runtime. Affected #: 1 file diff -r cf33c63f9a9b14b395d02d459ae6e30236236a67 -r 27edf6bd91d1b9b2153680adec361674c1f9b113 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -388,6 +388,7 @@ # R libraries are installed to $INSTALL_DIR (install_dir), we now set the R_LIBS path to that directory env_file_builder = EnvFileBuilder( install_dir ) + handle_action_shell_file_paths( env_file_builder, action_dict ) # Pull in R environment (runtime). env_file_builder.append_line( name="R_LIBS", action="prepend_to", value=install_dir ) return_code = env_file_builder.return_code if return_code: @@ -441,6 +442,7 @@ return env_file_builder = EnvFileBuilder( install_dir ) + handle_action_shell_file_paths( env_file_builder, action_dict ) # Pull in ruby dependencies (runtime). env_file_builder.append_line( name="GEM_PATH", action="prepend_to", value=install_dir ) env_file_builder.append_line( name="PATH", action="prepend_to", value=os.path.join(install_dir, 'bin') ) return_code = env_file_builder.return_code @@ -506,40 +508,16 @@ if return_code: return - # Perl libraries are installed to $INSTALL_DIR (install_dir), we now set the PERL5LIB path to that directory - # TODO: That code is used a lot for the different environments and should be refactored, once the environments are integrated - installed_env_dict = install_environment.environment_dict() - perl5lib_path = installed_env_dict.get('PERL5LIB', False) - perlbin_path = installed_env_dict.get('PATH', False) + env_file_builder = EnvFileBuilder( install_dir ) + # Recursively add dependent PERL5LIB and PATH to env.sh & anything else needed. + handle_action_shell_file_paths( env_file_builder, action_dict ) # Pull in ruby dependencies (runtime). - if not perl5lib_path or not perlbin_path: - log.warning( 'Missing PERL5LIB or/and PATH environment variable. Please check if your specified Perl installation is valid.' ) - - modify_env_command_dict = dict( name="PATH", action="set_to", value=perlbin_path ) - env_entry, env_file = td_common_util.create_or_update_env_shell_file( install_dir, modify_env_command_dict ) - return_code = file_append( env_entry, env_file, skip_if_contained=True, make_executable=True ) + env_file_builder.append_line( name="PERL5LIB", action="prepend_to", value=os.path.join(install_dir, 'lib', 'perl5') ) + env_file_builder.append_line( name="PATH", action="prepend_to", value=os.path.join(install_dir, 'bin') ) + return_code = env_file_builder.return_code if return_code: return - modify_env_command_dict = dict( name="PERL5LIB", action="set_to", value=perl5lib_path ) - env_entry, env_file = td_common_util.create_or_update_env_shell_file( install_dir, modify_env_command_dict ) - return_code = file_append( env_entry, env_file, skip_if_contained=True, make_executable=True ) - if return_code: - return - - modify_env_command_dict = dict( name="PERL5LIB", action="prepend_to", value=os.path.join(install_dir, 'lib', 'perl5') ) - env_entry, env_file = td_common_util.create_or_update_env_shell_file( install_dir, modify_env_command_dict ) - return_code = file_append( env_entry, env_file, skip_if_contained=True, make_executable=True ) - if return_code: - return - - modify_env_command_dict = dict( name="PATH", action="prepend_to", value=os.path.join(install_dir, 'bin') ) - env_entry, env_file = td_common_util.create_or_update_env_shell_file( install_dir, modify_env_command_dict ) - return_code = file_append( env_entry, env_file, skip_if_contained=True, make_executable=True ) - if return_code: - return - - else: # We're handling a complex repository dependency where we only have a set_environment tag set. # <action type="set_environment"> @@ -714,6 +692,13 @@ source=downloaded_filename, destination=full_path_to_dir ) + +def handle_action_shell_file_paths( env_file_builder, action_dict ): + shell_file_paths = action_dict.get( 'action_shell_file_paths', []) + for shell_file_path in shell_file_paths: + env_file_builder.append_line( action="source", value=shell_file_path ) + + def log_results( command, fabric_AttributeString, file_path ): """ Write attributes of fabric.operations._AttributeString (which is the output of executing command using fabric's local() method) https://bitbucket.org/galaxy/galaxy-central/commits/a545da034f0a/ Changeset: a545da034f0a User: jmchilton Date: 2013-11-14 17:39:49 Summary: Bring in action fix from @BjoernGruening, not included in previous commit. Affected #: 1 file diff -r 27edf6bd91d1b9b2153680adec361674c1f9b113 -r a545da034f0a4ffe525df3cd5e0346838bffb723 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -404,6 +404,8 @@ # <package>protk=1.2.4</package> # <package>http://url-to-some-gem-file.de/protk.gem</package> # </action> + filtered_actions = actions[ 1: ] + if action_dict.get( 'env_shell_file_paths', False ): install_environment.add_env_shell_file_paths( action_dict[ 'env_shell_file_paths' ] ) else: https://bitbucket.org/galaxy/galaxy-central/commits/ac744f96faa5/ Changeset: ac744f96faa5 User: jmchilton Date: 2013-11-14 17:39:49 Summary: Outline some enhancements to InstallEnvironment, not used yet. Affected #: 1 file diff -r a545da034f0a4ffe525df3cd5e0346838bffb723 -r ac744f96faa5d4a61c73ca2e20655f818af3dddd lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -21,6 +21,7 @@ from fabric.api import lcd from fabric.api import local from fabric.api import settings +from fabric.api import prefix log = logging.getLogger( __name__ ) @@ -245,6 +246,17 @@ log.debug( 'Invalid file %s specified, ignoring %s action.', env_shell_file_path, action_type ) return cmds + def __call__( self, install_dir ): + with settings( warn_only=True, **td_common_util.get_env_var_values( install_dir ) ): + with prefix( self.__setup_environment() ): + yield + + def __setup_environment(self): + return "&&".join( [". %s" % file for file in self.__valid_env_shell_file_paths() ] ) + + def __valid_env_shell_file_paths(self): + return [ file for file in self.env_shell_file_paths if os.path.exists( file ) ] + def environment_dict(self, action_type='template_command'): env_vars = dict() for env_shell_file_path in self.env_shell_file_paths: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.