24 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/be16cffdcb68/ Changeset: be16cffdcb68 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Convert James Taylor's dependency testing file lib/galaxy/tools/deps/tests.py into a formal unit test. Affected #: 2 files diff -r ebd092733e552ef6a846d55b69d22dc779dfcc5f -r be16cffdcb68709e3368bcd599bfef9f8aab72a4 lib/galaxy/tools/deps/tests.py --- a/lib/galaxy/tools/deps/tests.py +++ /dev/null @@ -1,32 +0,0 @@ -import tempfile -import os.path -from os import makedirs, mkdir -import galaxy.tools.deps - -def touch( fname, data=None ): - f = open( fname, 'w' ) - if data: - f.write( data ) - f.close() - -def test(): - - # Setup directories - base_path = tempfile.mkdtemp() - # mkdir( base_path ) - for name, version, sub in [ ( "dep1", "1.0", "env.sh" ), ( "dep1", "2.0", "bin" ), ( "dep2", "1.0", None ) ]: - if sub == "bin": - p = os.path.join( base_path, name, version, "bin" ) - else: - p = os.path.join( base_path, name, version ) - try: - makedirs( p ) - except: - pass - if sub == "env.sh": - touch( os.path.join( p, "env.sh" ) ) - - dm = galaxy.tools.deps.DependencyManager( [ base_path ] ) - - print dm.find_dep( "dep1", "1.0" ) - print dm.find_dep( "dep1", "2.0" ) diff -r ebd092733e552ef6a846d55b69d22dc779dfcc5f -r be16cffdcb68709e3368bcd599bfef9f8aab72a4 test/unit/test_tool_deps.py --- /dev/null +++ b/test/unit/test_tool_deps.py @@ -0,0 +1,40 @@ +import tempfile +import os.path +from os import makedirs +import galaxy.tools.deps + + +def touch( fname, data=None ): + f = open( fname, 'w' ) + if data: + f.write( data ) + f.close() + + +def test_tool_dependencies(): + + # Setup directories + base_path = tempfile.mkdtemp() + # mkdir( base_path ) + for name, version, sub in [ ( "dep1", "1.0", "env.sh" ), ( "dep1", "2.0", "bin" ), ( "dep2", "1.0", None ) ]: + if sub == "bin": + p = os.path.join( base_path, name, version, "bin" ) + else: + p = os.path.join( base_path, name, version ) + try: + makedirs( p ) + except: + pass + if sub == "env.sh": + touch( os.path.join( p, "env.sh" ) ) + + dm = galaxy.tools.deps.DependencyManager( [ base_path ] ) + + d1_script, d1_path, d1_version = dm.find_dep( "dep1", "1.0" ) + assert d1_script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) + assert d1_path == os.path.join( base_path, 'dep1', '1.0' ) + assert d1_version == "1.0" + d2_script, d2_path, d2_version = dm.find_dep( "dep1", "2.0" ) + assert d2_script == None + assert d2_path == os.path.join( base_path, 'dep1', '2.0' ) + assert d2_version == "2.0" https://bitbucket.org/galaxy/galaxy-central/commits/e276f3fc9a74/ Changeset: e276f3fc9a74 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Add tests for addditional existing behavior for tool dependencies. Namely test the fallen: - Test default package will not be fallen back upon when exactly specified version is used. (Not my favorite behavior. -John) - Test tool shed package and set_environment requirement types (standard use). - Test tool shed installed tools will fallback on traditional Galaxy dependencies if a requirement is unmatched in installed dependencies. - Test tool shed dependency takes prescndence over traditional Galaxy dependency when both are available. Other small clean ups of test/unit/test_tool_deps.py. Affected #: 1 file diff -r be16cffdcb68709e3368bcd599bfef9f8aab72a4 -r e276f3fc9a7426923d968e7e56da1e3b26292056 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -1,40 +1,140 @@ import tempfile import os.path -from os import makedirs -import galaxy.tools.deps - - -def touch( fname, data=None ): - f = open( fname, 'w' ) - if data: - f.write( data ) - f.close() +from os import makedirs, symlink +from shutil import rmtree +from galaxy.tools.deps import DependencyManager +from galaxy.util.bunch import Bunch +from contextlib import contextmanager def test_tool_dependencies(): + # Setup directories + with __test_base_path() as base_path: + for name, version, sub in [ ( "dep1", "1.0", "env.sh" ), ( "dep1", "2.0", "bin" ), ( "dep2", "1.0", None ) ]: + if sub == "bin": + p = os.path.join( base_path, name, version, "bin" ) + else: + p = os.path.join( base_path, name, version ) + try: + makedirs( p ) + except: + pass + if sub == "env.sh": + __touch( os.path.join( p, "env.sh" ) ) - # Setup directories + dm = DependencyManager( [base_path] ) + d1_script, d1_path, d1_version = dm.find_dep( "dep1", "1.0" ) + assert d1_script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) + assert d1_path == os.path.join( base_path, 'dep1', '1.0' ) + assert d1_version == "1.0" + d2_script, d2_path, d2_version = dm.find_dep( "dep1", "2.0" ) + assert d2_script == None + assert d2_path == os.path.join( base_path, 'dep1', '2.0' ) + assert d2_version == "2.0" + + ## Test default versions + symlink( os.path.join( base_path, 'dep1', '2.0'), os.path.join( base_path, 'dep1', 'default' ) ) + default_script, default_path, default_version = dm.find_dep( "dep1", None ) + assert default_version == "2.0" + + ## Test default will not be fallen back upon by default + default_script, default_path, default_version = dm.find_dep( "dep1", "2.1" ) + assert default_script == None + assert default_version == None + + +TEST_REPO_USER = "devteam" +TEST_REPO_NAME = "bwa" +TEST_REPO_CHANGESET = "12abcd41223da" +TEST_VERSION = "0.5.9" + + +def test_toolshed_set_enviornment_requiremetns(): + with __test_base_path() as base_path: + test_repo = __build_test_repo('set_environment') + dm = DependencyManager( [base_path] ) + env_settings_dir = os.path.join(base_path, "environment_settings", TEST_REPO_NAME, TEST_REPO_USER, TEST_REPO_NAME, TEST_REPO_CHANGESET) + os.makedirs(env_settings_dir) + d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=None, type='set_environment', installed_tool_dependencies=[test_repo] ) + assert d1_version == None + assert d1_script == os.path.join(env_settings_dir, "env.sh"), d1_script + + +def test_toolshed_package_requirements(): + with __test_base_path() as base_path: + test_repo = __build_test_repo('package', version=TEST_VERSION) + dm = DependencyManager( [base_path] ) + package_dir = __build_ts_test_package(base_path) + d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) + assert d1_version == TEST_VERSION, d1_version + assert d1_script == os.path.join(package_dir, "env.sh"), d1_script + + +def test_toolshed_tools_fallback_on_manual_dependencies(): + with __test_base_path() as base_path: + dm = DependencyManager( [base_path] ) + test_repo = __build_test_repo('package', version=TEST_VERSION) + env_path = __setup_galaxy_package_dep(base_path, "dep1", "1.0") + d1_script, d1_path, d1_version = dm.find_dep( "dep1", version="1.0", type='package', installed_tool_dependencies=[test_repo] ) + assert d1_version == "1.0" + assert d1_script == env_path + + +def test_toolshed_greater_precendence(): + with __test_base_path() as base_path: + dm = DependencyManager( [base_path] ) + test_repo = __build_test_repo('package', version=TEST_VERSION) + ts_package_dir = __build_ts_test_package(base_path) + gx_env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION) + ts_env_path = os.path.join(ts_package_dir, "env.sh") + d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) + assert d1_script != gx_env_path # Not the galaxy path, it should be the tool shed path used. + assert d1_script == ts_env_path + + +def __build_ts_test_package(base_path, script_contents=''): + package_dir = os.path.join(base_path, TEST_REPO_NAME, TEST_VERSION, TEST_REPO_USER, TEST_REPO_NAME, TEST_REPO_CHANGESET) + __touch(os.path.join(package_dir, 'env.sh'), script_contents) + return package_dir + + +def __setup_galaxy_package_dep(base_path, name, version, contents=""): + dep_directory = os.path.join( base_path, name, version ) + env_path = os.path.join( dep_directory, "env.sh" ) + __touch( env_path, contents ) + return env_path + + +def __touch( fname, data=None ): + dirname = os.path.dirname( fname ) + if not os.path.exists( dirname ): + makedirs( dirname ) + f = open( fname, 'w' ) + try: + if data: + f.write( data ) + finally: + f.close() + + +def __build_test_repo(type, version=None): + return Bunch( + owner=TEST_REPO_USER, + name=TEST_REPO_NAME, + type=type, + version=version, + tool_shed_repository=Bunch( + owner=TEST_REPO_USER, + name=TEST_REPO_NAME, + installed_changeset_revision=TEST_REPO_CHANGESET + ) + ) + + +@contextmanager +def __test_base_path(): base_path = tempfile.mkdtemp() - # mkdir( base_path ) - for name, version, sub in [ ( "dep1", "1.0", "env.sh" ), ( "dep1", "2.0", "bin" ), ( "dep2", "1.0", None ) ]: - if sub == "bin": - p = os.path.join( base_path, name, version, "bin" ) - else: - p = os.path.join( base_path, name, version ) - try: - makedirs( p ) - except: - pass - if sub == "env.sh": - touch( os.path.join( p, "env.sh" ) ) - - dm = galaxy.tools.deps.DependencyManager( [ base_path ] ) - - d1_script, d1_path, d1_version = dm.find_dep( "dep1", "1.0" ) - assert d1_script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) - assert d1_path == os.path.join( base_path, 'dep1', '1.0' ) - assert d1_version == "1.0" - d2_script, d2_path, d2_version = dm.find_dep( "dep1", "2.0" ) - assert d2_script == None - assert d2_path == os.path.join( base_path, 'dep1', '2.0' ) - assert d2_version == "2.0" + try: + yield base_path + finally: + rmtree(base_path) https://bitbucket.org/galaxy/galaxy-central/commits/b16ab0d407a0/ Changeset: b16ab0d407a0 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: PEP8 fixes for lib/galaxy/tools/deps/__init__.py Affected #: 1 file diff -r e276f3fc9a7426923d968e7e56da1e3b26292056 -r b16ab0d407a09695aaa0dcbffcd9661b12c4b701 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -7,6 +7,7 @@ import logging log = logging.getLogger( __name__ ) + class DependencyManager( object ): """ A DependencyManager attempts to resolve named and versioned dependencies by searching for them under a list of directories. Directories should be @@ -28,6 +29,7 @@ if not os.path.isdir( base_path ): log.warn( "Path '%s' is not directory, ignoring", base_path ) self.base_paths.append( os.path.abspath( base_path ) ) + def find_dep( self, name, version=None, type='package', installed_tool_dependencies=None ): """ Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a @@ -37,6 +39,7 @@ return self._find_dep_default( name, type=type, installed_tool_dependencies=installed_tool_dependencies ) else: return self._find_dep_versioned( name, version, type=type, installed_tool_dependencies=installed_tool_dependencies ) + def _find_dep_versioned( self, name, version, type='package', installed_tool_dependencies=None ): installed_tool_dependency = self._get_installed_dependency( installed_tool_dependencies, name, type, version=version ) for base_path in self.base_paths: @@ -51,6 +54,7 @@ return None, path, version else: return None, None, None + def _find_dep_default( self, name, type='package', installed_tool_dependencies=None ): if type == 'set_environment' and installed_tool_dependencies: installed_tool_dependency = self._get_installed_dependency( installed_tool_dependencies, name, type, version=None ) @@ -68,20 +72,22 @@ script = os.path.join( real_path, 'env.sh' ) if os.path.exists( script ): return script, real_path, real_version - elif os.path.exists( os.path.join( real_path, 'bin' ) ): + elif os.path.exists( real_bin ): return None, real_path, real_version else: return None, None, None + def _get_installed_dependency( self, installed_tool_dependencies, name, type, version=None ): if installed_tool_dependencies: for installed_tool_dependency in installed_tool_dependencies: if version: - if installed_tool_dependency.name==name and installed_tool_dependency.type==type and installed_tool_dependency.version==version: + if installed_tool_dependency.name == name and installed_tool_dependency.type == type and installed_tool_dependency.version == version: return installed_tool_dependency else: - if installed_tool_dependency.name==name and installed_tool_dependency.type==type: + if installed_tool_dependency.name == name and installed_tool_dependency.type == type: return installed_tool_dependency return None + def _get_package_installed_dependency_path( self, installed_tool_dependency, base_path, name, version ): tool_shed_repository = installed_tool_dependency.tool_shed_repository return os.path.join( base_path, @@ -90,6 +96,7 @@ tool_shed_repository.owner, tool_shed_repository.name, tool_shed_repository.installed_changeset_revision ) + def _get_set_environment_installed_dependency_script_path( self, installed_tool_dependency, name ): tool_shed_repository = installed_tool_dependency.tool_shed_repository for base_path in self.base_paths: https://bitbucket.org/galaxy/galaxy-central/commits/630cfe0030ab/ Changeset: 630cfe0030ab User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Drop multiple tool_dependency_dir implementation for now. Will reimplement once I have added more granular, plugin style tool dependency resolvers. They will benefit from there being just one default_base_path as implemented here, but allowing it to be overridden on a per resolver basis. Affected #: 3 files diff -r b16ab0d407a09695aaa0dcbffcd9661b12c4b701 -r 630cfe0030abf40e46c9839254278531d657e554 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -696,7 +696,7 @@ def init_dependency_manager( self ): if self.app.config.use_tool_dependencies: - self.dependency_manager = DependencyManager( [ self.app.config.tool_dependency_dir ] ) + self.dependency_manager = DependencyManager( self.app.config.tool_dependency_dir ) else: self.dependency_manager = None diff -r b16ab0d407a09695aaa0dcbffcd9661b12c4b701 -r 630cfe0030abf40e46c9839254278531d657e554 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -17,18 +17,16 @@ and should each contain a file 'env.sh' which can be sourced to make the dependency available in the current shell environment. """ - def __init__( self, base_paths=[] ): + def __init__( self, default_base_path ): """ Create a new dependency manager looking for packages under the paths listed in `base_paths`. The default base path is app.config.tool_dependency_dir. """ - self.base_paths = [] - for base_path in base_paths: - if not os.path.exists( base_path ): - log.warn( "Path '%s' does not exist, ignoring", base_path ) - if not os.path.isdir( base_path ): - log.warn( "Path '%s' is not directory, ignoring", base_path ) - self.base_paths.append( os.path.abspath( base_path ) ) + if not os.path.exists( default_base_path ): + log.warn( "Path '%s' does not exist, ignoring", default_base_path ) + if not os.path.isdir( default_base_path ): + log.warn( "Path '%s' is not directory, ignoring", default_base_path ) + self.default_base_path = os.path.abspath( default_base_path ) def find_dep( self, name, version=None, type='package', installed_tool_dependencies=None ): """ @@ -42,18 +40,17 @@ def _find_dep_versioned( self, name, version, type='package', installed_tool_dependencies=None ): installed_tool_dependency = self._get_installed_dependency( installed_tool_dependencies, name, type, version=version ) - for base_path in self.base_paths: - if installed_tool_dependency: - path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) - else: - path = os.path.join( base_path, name, version ) - script = os.path.join( path, 'env.sh' ) - if os.path.exists( script ): - return script, path, version - elif os.path.exists( os.path.join( path, 'bin' ) ): - return None, path, version + base_path = self.default_base_path + if installed_tool_dependency: + path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) else: - return None, None, None + path = os.path.join( base_path, name, version ) + script = os.path.join( path, 'env.sh' ) + if os.path.exists( script ): + return script, path, version + elif os.path.exists( os.path.join( path, 'bin' ) ): + return None, path, version + return None, None, None def _find_dep_default( self, name, type='package', installed_tool_dependencies=None ): if type == 'set_environment' and installed_tool_dependencies: @@ -63,19 +60,18 @@ if script and path: # Environment settings do not use versions. return script, path, None - for base_path in self.base_paths: - path = os.path.join( base_path, name, 'default' ) - if os.path.islink( path ): - real_path = os.path.realpath( path ) - real_bin = os.path.join( real_path, 'bin' ) - real_version = os.path.basename( real_path ) - script = os.path.join( real_path, 'env.sh' ) - if os.path.exists( script ): - return script, real_path, real_version - elif os.path.exists( real_bin ): - return None, real_path, real_version - else: - return None, None, None + base_path = self.default_base_path + path = os.path.join( base_path, name, 'default' ) + if os.path.islink( path ): + real_path = os.path.realpath( path ) + real_bin = os.path.join( real_path, 'bin' ) + real_version = os.path.basename( real_path ) + script = os.path.join( real_path, 'env.sh' ) + if os.path.exists( script ): + return script, real_path, real_version + elif os.path.exists( real_bin ): + return None, real_path, real_version + return None, None, None def _get_installed_dependency( self, installed_tool_dependencies, name, type, version=None ): if installed_tool_dependencies: @@ -99,14 +95,14 @@ def _get_set_environment_installed_dependency_script_path( self, installed_tool_dependency, name ): tool_shed_repository = installed_tool_dependency.tool_shed_repository - for base_path in self.base_paths: - path = os.path.abspath( os.path.join( base_path, - 'environment_settings', - name, - tool_shed_repository.owner, - tool_shed_repository.name, - tool_shed_repository.installed_changeset_revision ) ) - if os.path.exists( path ): - script = os.path.join( path, 'env.sh' ) - return script, path, None + base_path = self.default_base_path + path = os.path.abspath( os.path.join( base_path, + 'environment_settings', + name, + tool_shed_repository.owner, + tool_shed_repository.name, + tool_shed_repository.installed_changeset_revision ) ) + if os.path.exists( path ): + script = os.path.join( path, 'env.sh' ) + return script, path, None return None, None, None diff -r b16ab0d407a09695aaa0dcbffcd9661b12c4b701 -r 630cfe0030abf40e46c9839254278531d657e554 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -9,6 +9,7 @@ def test_tool_dependencies(): # Setup directories + with __test_base_path() as base_path: for name, version, sub in [ ( "dep1", "1.0", "env.sh" ), ( "dep1", "2.0", "bin" ), ( "dep2", "1.0", None ) ]: if sub == "bin": @@ -22,7 +23,7 @@ if sub == "env.sh": __touch( os.path.join( p, "env.sh" ) ) - dm = DependencyManager( [base_path] ) + dm = DependencyManager( default_base_path=base_path ) d1_script, d1_path, d1_version = dm.find_dep( "dep1", "1.0" ) assert d1_script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) assert d1_path == os.path.join( base_path, 'dep1', '1.0' ) @@ -52,7 +53,7 @@ def test_toolshed_set_enviornment_requiremetns(): with __test_base_path() as base_path: test_repo = __build_test_repo('set_environment') - dm = DependencyManager( [base_path] ) + dm = DependencyManager( default_base_path=base_path ) env_settings_dir = os.path.join(base_path, "environment_settings", TEST_REPO_NAME, TEST_REPO_USER, TEST_REPO_NAME, TEST_REPO_CHANGESET) os.makedirs(env_settings_dir) d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=None, type='set_environment', installed_tool_dependencies=[test_repo] ) @@ -63,7 +64,7 @@ def test_toolshed_package_requirements(): with __test_base_path() as base_path: test_repo = __build_test_repo('package', version=TEST_VERSION) - dm = DependencyManager( [base_path] ) + dm = DependencyManager( default_base_path=base_path ) package_dir = __build_ts_test_package(base_path) d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) assert d1_version == TEST_VERSION, d1_version @@ -72,7 +73,7 @@ def test_toolshed_tools_fallback_on_manual_dependencies(): with __test_base_path() as base_path: - dm = DependencyManager( [base_path] ) + dm = DependencyManager( default_base_path=base_path ) test_repo = __build_test_repo('package', version=TEST_VERSION) env_path = __setup_galaxy_package_dep(base_path, "dep1", "1.0") d1_script, d1_path, d1_version = dm.find_dep( "dep1", version="1.0", type='package', installed_tool_dependencies=[test_repo] ) @@ -82,7 +83,7 @@ def test_toolshed_greater_precendence(): with __test_base_path() as base_path: - dm = DependencyManager( [base_path] ) + dm = DependencyManager( default_base_path=base_path ) test_repo = __build_test_repo('package', version=TEST_VERSION) ts_package_dir = __build_ts_test_package(base_path) gx_env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION) https://bitbucket.org/galaxy/galaxy-central/commits/a54a058fa777/ Changeset: a54a058fa777 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Small clean ups. Affected #: 1 file diff -r 630cfe0030abf40e46c9839254278531d657e554 -r a54a058fa77737cd1f9d51393cb46f5bf8dda95f lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -7,15 +7,19 @@ import logging log = logging.getLogger( __name__ ) +INDETERMINATE_DEPENDENCY = (None, None, None) + class DependencyManager( object ): """ - A DependencyManager attempts to resolve named and versioned dependencies by searching for them under a list of directories. Directories should be + A DependencyManager attempts to resolve named and versioned dependencies by + searching for them under a list of directories. Directories should be of the form: $BASE/name/version/... - and should each contain a file 'env.sh' which can be sourced to make the dependency available in the current shell environment. + and should each contain a file 'env.sh' which can be sourced to make the + dependency available in the current shell environment. """ def __init__( self, default_base_path ): """ @@ -50,7 +54,7 @@ return script, path, version elif os.path.exists( os.path.join( path, 'bin' ) ): return None, path, version - return None, None, None + return INDETERMINATE_DEPENDENCY def _find_dep_default( self, name, type='package', installed_tool_dependencies=None ): if type == 'set_environment' and installed_tool_dependencies: @@ -71,16 +75,17 @@ return script, real_path, real_version elif os.path.exists( real_bin ): return None, real_path, real_version - return None, None, None + return INDETERMINATE_DEPENDENCY def _get_installed_dependency( self, installed_tool_dependencies, name, type, version=None ): if installed_tool_dependencies: for installed_tool_dependency in installed_tool_dependencies: + name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type if version: - if installed_tool_dependency.name == name and installed_tool_dependency.type == type and installed_tool_dependency.version == version: + if name_and_type_equal and installed_tool_dependency.version == version: return installed_tool_dependency else: - if installed_tool_dependency.name == name and installed_tool_dependency.type == type: + if name_and_type_equal: return installed_tool_dependency return None @@ -105,4 +110,4 @@ if os.path.exists( path ): script = os.path.join( path, 'env.sh' ) return script, path, None - return None, None, None + return INDETERMINATE_DEPENDENCY https://bitbucket.org/galaxy/galaxy-central/commits/8ca4fdce3052/ Changeset: 8ca4fdce3052 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Begin rearranging arguments so all ones not of interest to every potential dependency resolution plugin can be kwds Affected #: 1 file diff -r a54a058fa77737cd1f9d51393cb46f5bf8dda95f -r 8ca4fdce30526d58737d2b0d21af15cb750ddffc lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -43,7 +43,7 @@ return self._find_dep_versioned( name, version, type=type, installed_tool_dependencies=installed_tool_dependencies ) def _find_dep_versioned( self, name, version, type='package', installed_tool_dependencies=None ): - installed_tool_dependency = self._get_installed_dependency( installed_tool_dependencies, name, type, version=version ) + installed_tool_dependency = self._get_installed_dependency( name, type, version=version, installed_tool_dependencies=installed_tool_dependencies ) base_path = self.default_base_path if installed_tool_dependency: path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) @@ -58,7 +58,7 @@ def _find_dep_default( self, name, type='package', installed_tool_dependencies=None ): if type == 'set_environment' and installed_tool_dependencies: - installed_tool_dependency = self._get_installed_dependency( installed_tool_dependencies, name, type, version=None ) + installed_tool_dependency = self._get_installed_dependency( name, type, version=None, installed_tool_dependencies=installed_tool_dependencies ) if installed_tool_dependency: script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) if script and path: @@ -77,16 +77,15 @@ return None, real_path, real_version return INDETERMINATE_DEPENDENCY - def _get_installed_dependency( self, installed_tool_dependencies, name, type, version=None ): - if installed_tool_dependencies: - for installed_tool_dependency in installed_tool_dependencies: - name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type - if version: - if name_and_type_equal and installed_tool_dependency.version == version: - return installed_tool_dependency - else: - if name_and_type_equal: - return installed_tool_dependency + def _get_installed_dependency( self, name, type, version=None, **kwds ): + for installed_tool_dependency in kwds.get("installed_tool_dependencies", []): + name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type + if version: + if name_and_type_equal and installed_tool_dependency.version == version: + return installed_tool_dependency + else: + if name_and_type_equal: + return installed_tool_dependency return None def _get_package_installed_dependency_path( self, installed_tool_dependency, base_path, name, version ): https://bitbucket.org/galaxy/galaxy-central/commits/22f3aca8ad9c/ Changeset: 22f3aca8ad9c User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Finish refactoring toward generic keyword arguments. Affected #: 1 file diff -r 8ca4fdce30526d58737d2b0d21af15cb750ddffc -r 22f3aca8ad9cb9a0e606a1c69cb4c211fe5cc133 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -32,18 +32,18 @@ log.warn( "Path '%s' is not directory, ignoring", default_base_path ) self.default_base_path = os.path.abspath( default_base_path ) - def find_dep( self, name, version=None, type='package', installed_tool_dependencies=None ): + def find_dep( self, name, version=None, type='package', **kwds ): """ Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a symbolic link (if found). Returns a triple of: env_script, base_path, real_version """ if version is None: - return self._find_dep_default( name, type=type, installed_tool_dependencies=installed_tool_dependencies ) + return self._find_dep_default( name, type=type, **kwds ) else: - return self._find_dep_versioned( name, version, type=type, installed_tool_dependencies=installed_tool_dependencies ) + return self._find_dep_versioned( name, version, type=type, **kwds ) - def _find_dep_versioned( self, name, version, type='package', installed_tool_dependencies=None ): - installed_tool_dependency = self._get_installed_dependency( name, type, version=version, installed_tool_dependencies=installed_tool_dependencies ) + def _find_dep_versioned( self, name, version, type='package', **kwds ): + installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) base_path = self.default_base_path if installed_tool_dependency: path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) @@ -56,9 +56,9 @@ return None, path, version return INDETERMINATE_DEPENDENCY - def _find_dep_default( self, name, type='package', installed_tool_dependencies=None ): - if type == 'set_environment' and installed_tool_dependencies: - installed_tool_dependency = self._get_installed_dependency( name, type, version=None, installed_tool_dependencies=installed_tool_dependencies ) + def _find_dep_default( self, name, type='package', **kwds ): + if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): + installed_tool_dependency = self._get_installed_dependency( name, type, version=None, **kwds ) if installed_tool_dependency: script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) if script and path: https://bitbucket.org/galaxy/galaxy-central/commits/dd2b36c4fc66/ Changeset: dd2b36c4fc66 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Introduce the concept of a dependency resolver, DependencyManager will check each in turn. Affected #: 1 file diff -r 22f3aca8ad9cb9a0e606a1c69cb4c211fe5cc133 -r dd2b36c4fc6682e76cceef10de2ae3154cc107c6 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -31,8 +31,29 @@ if not os.path.isdir( default_base_path ): log.warn( "Path '%s' is not directory, ignoring", default_base_path ) self.default_base_path = os.path.abspath( default_base_path ) + self.dependency_resolvers = [ GalaxyPackageDependencyResolver(self) ] + def find_dep( self, name, version=None, type='package', **kwds ): + for resolver in self.dependency_resolvers: + dependency = resolver.resolve( name, version, type, **kwds ) + if dependency != INDETERMINATE_DEPENDENCY: + return dependency + return INDETERMINATE_DEPENDENCY + + +class DependencyResolver(object): + + def resolve( self, name, version, type, **kwds ): + raise NotImplementedError() + + +class GalaxyPackageDependencyResolver(DependencyResolver): + + def __init__(self, dependency_manager): + self.default_base_path = dependency_manager.default_base_path + + def resolve( self, name, version, type, **kwds ): """ Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a symbolic link (if found). Returns a triple of: env_script, base_path, real_version https://bitbucket.org/galaxy/galaxy-central/commits/ae5dbc930415/ Changeset: ae5dbc930415 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Refactor common code out of _find_dep_versioned and _find_dep_default. Affected #: 1 file diff -r dd2b36c4fc6682e76cceef10de2ae3154cc107c6 -r ae5dbc9304152e034ed5565f5243ab6b5c0cbe77 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -70,12 +70,7 @@ path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) else: path = os.path.join( base_path, name, version ) - script = os.path.join( path, 'env.sh' ) - if os.path.exists( script ): - return script, path, version - elif os.path.exists( os.path.join( path, 'bin' ) ): - return None, path, version - return INDETERMINATE_DEPENDENCY + return self._galaxy_package_dep(path, version) def _find_dep_default( self, name, type='package', **kwds ): if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): @@ -89,13 +84,17 @@ path = os.path.join( base_path, name, 'default' ) if os.path.islink( path ): real_path = os.path.realpath( path ) - real_bin = os.path.join( real_path, 'bin' ) real_version = os.path.basename( real_path ) - script = os.path.join( real_path, 'env.sh' ) - if os.path.exists( script ): - return script, real_path, real_version - elif os.path.exists( real_bin ): - return None, real_path, real_version + return self._galaxy_package_dep(real_path, real_version) + else: + return INDETERMINATE_DEPENDENCY + + def _galaxy_package_dep( self, path, version ): + script = os.path.join( path, 'env.sh' ) + if os.path.exists( script ): + return script, path, version + elif os.path.exists( os.path.join( path, 'bin' ) ): + return None, path, version return INDETERMINATE_DEPENDENCY def _get_installed_dependency( self, name, type, version=None, **kwds ): https://bitbucket.org/galaxy/galaxy-central/commits/8928cf0be809/ Changeset: 8928cf0be809 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Split package resolvers into Tool Shed and traditional Galaxy versions. Affected #: 1 file diff -r ae5dbc9304152e034ed5565f5243ab6b5c0cbe77 -r 8928cf0be80994703689e89c96e2dbfa20a47319 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -31,7 +31,10 @@ if not os.path.isdir( default_base_path ): log.warn( "Path '%s' is not directory, ignoring", default_base_path ) self.default_base_path = os.path.abspath( default_base_path ) - self.dependency_resolvers = [ GalaxyPackageDependencyResolver(self) ] + self.dependency_resolvers = [ + ToolShedPackageDependencyResolver(self), + GalaxyPackageDependencyResolver(self), + ] def find_dep( self, name, version=None, type='package', **kwds ): @@ -64,22 +67,11 @@ return self._find_dep_versioned( name, version, type=type, **kwds ) def _find_dep_versioned( self, name, version, type='package', **kwds ): - installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) base_path = self.default_base_path - if installed_tool_dependency: - path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) - else: - path = os.path.join( base_path, name, version ) + path = os.path.join( base_path, name, version ) return self._galaxy_package_dep(path, version) def _find_dep_default( self, name, type='package', **kwds ): - if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): - installed_tool_dependency = self._get_installed_dependency( name, type, version=None, **kwds ) - if installed_tool_dependency: - script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) - if script and path: - # Environment settings do not use versions. - return script, path, None base_path = self.default_base_path path = os.path.join( base_path, name, 'default' ) if os.path.islink( path ): @@ -97,6 +89,31 @@ return None, path, version return INDETERMINATE_DEPENDENCY + +class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): + + def __init__(self, dependency_manager): + super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager) + + def _find_dep_versioned( self, name, version, type='package', **kwds ): + installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) + base_path = self.default_base_path + if installed_tool_dependency: + path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) + return self._galaxy_package_dep(path, version) + else: + return INDETERMINATE_DEPENDENCY + + def _find_dep_default( self, name, type='package', **kwds ): + if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): + installed_tool_dependency = self._get_installed_dependency( name, type, version=None, **kwds ) + if installed_tool_dependency: + script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) + if script and path: + # Environment settings do not use versions. + return script, path, None + return INDETERMINATE_DEPENDENCY + def _get_installed_dependency( self, name, type, version=None, **kwds ): for installed_tool_dependency in kwds.get("installed_tool_dependencies", []): name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type https://bitbucket.org/galaxy/galaxy-central/commits/0762bd90e85b/ Changeset: 0762bd90e85b User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Implement XML configuration of tool dependency resolver. This is largely modelled off of @natefoo's job_conf.xml. This is a minimally configurable XML layout so far, but will be dding more options. Already can decide between whether to load tool shed installed packages first (default) or manually conifgured ones. The advantages of loading manually configured ones first is outlined here - http://dev.list.galaxyproject.org/Test-Toolshed-Biopython-package-dependency.... Affected #: 4 files diff -r 8928cf0be80994703689e89c96e2dbfa20a47319 -r 0762bd90e85b94e643458c22670de8455dc97f71 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -114,6 +114,7 @@ self.collect_outputs_from = [ x.strip() for x in kwargs.get( 'collect_outputs_from', 'new_file_path,job_working_directory' ).lower().split(',') ] self.template_path = resolve_path( kwargs.get( "template_path", "templates" ), self.root ) self.template_cache = resolve_path( kwargs.get( "template_cache_path", "database/compiled_templates" ), self.root ) + self.dependency_resolvers_config_file = resolve_path( kwargs.get( 'dependency_resolvers_config_file', 'dependency_resolvers_conf.xml' ), self.root ) self.job_config_file = resolve_path( kwargs.get( 'job_config_file', 'job_conf.xml' ), self.root ) self.local_job_queue_workers = int( kwargs.get( "local_job_queue_workers", "5" ) ) self.cluster_job_queue_workers = int( kwargs.get( "cluster_job_queue_workers", "3" ) ) diff -r 8928cf0be80994703689e89c96e2dbfa20a47319 -r 0762bd90e85b94e643458c22670de8455dc97f71 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -696,7 +696,11 @@ def init_dependency_manager( self ): if self.app.config.use_tool_dependencies: - self.dependency_manager = DependencyManager( self.app.config.tool_dependency_dir ) + dependency_manager_kwds = { + 'default_base_path': self.app.config.tool_dependency_dir, + 'conf_file': self.app.config.dependency_resolvers_config_file, + } + self.dependency_manager = DependencyManager( **dependency_manager_kwds ) else: self.dependency_manager = None diff -r 8928cf0be80994703689e89c96e2dbfa20a47319 -r 0762bd90e85b94e643458c22670de8455dc97f71 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -7,6 +7,8 @@ import logging log = logging.getLogger( __name__ ) +from galaxy.util import parse_xml + INDETERMINATE_DEPENDENCY = (None, None, None) @@ -21,7 +23,7 @@ and should each contain a file 'env.sh' which can be sourced to make the dependency available in the current shell environment. """ - def __init__( self, default_base_path ): + def __init__( self, default_base_path, conf_file=None ): """ Create a new dependency manager looking for packages under the paths listed in `base_paths`. The default base path is app.config.tool_dependency_dir. @@ -31,10 +33,7 @@ if not os.path.isdir( default_base_path ): log.warn( "Path '%s' is not directory, ignoring", default_base_path ) self.default_base_path = os.path.abspath( default_base_path ) - self.dependency_resolvers = [ - ToolShedPackageDependencyResolver(self), - GalaxyPackageDependencyResolver(self), - ] + self.dependency_resolvers = self.__build_dependency_resolvers( conf_file ) def find_dep( self, name, version=None, type='package', **kwds ): @@ -44,6 +43,32 @@ return dependency return INDETERMINATE_DEPENDENCY + def __build_dependency_resolvers( self, conf_file ): + if not conf_file or not os.path.exists( conf_file ): + return self.__default_dependency_resolvers() + tree = parse_xml( conf_file ) + return self.__parse_resolver_conf_xml( tree ) + + def __default_dependency_resolvers( self ): + return [ + ToolShedPackageDependencyResolver(self), + GalaxyPackageDependencyResolver(self), + ] + + def __parse_resolver_conf_xml(self, tree): + """ + + :param tree: Object representing the root ``<dependency_resolvers>`` object in the file. + :type tree: ``xml.etree.ElementTree.Element`` + """ + resolvers = [] + resolvers_element = tree.getroot() + for resolver_element in resolvers_element.getchildren(): + resolver_type = resolver_element.tag + resolver = RESOLVER_CLASSES[resolver_type](self) + resolvers.append(resolver) + return resolvers + class DependencyResolver(object): @@ -147,3 +172,9 @@ script = os.path.join( path, 'env.sh' ) return script, path, None return INDETERMINATE_DEPENDENCY + + +RESOLVER_CLASSES = { + 'tool_shed_package': ToolShedPackageDependencyResolver, + 'galaxy_package': GalaxyPackageDependencyResolver, +} diff -r 8928cf0be80994703689e89c96e2dbfa20a47319 -r 0762bd90e85b94e643458c22670de8455dc97f71 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -139,3 +139,32 @@ yield base_path finally: rmtree(base_path) + + +def test_parse(): + with __parse_resolvers('''<dependency_resolvers> + <tool_shed_package /> + <galaxy_package /> +</dependency_resolvers> +''') as dependency_resolvers: + assert 'ToolShed' in dependency_resolvers[0].__class__.__name__ + assert 'Galaxy' in dependency_resolvers[1].__class__.__name__ + + with __parse_resolvers('''<dependency_resolvers> + <galaxy_package /> + <tool_shed_package /> +</dependency_resolvers> +''') as dependency_resolvers: + assert 'Galaxy' in dependency_resolvers[0].__class__.__name__ + assert 'ToolShed' in dependency_resolvers[1].__class__.__name__ + + +@contextmanager +def __parse_resolvers(xml_content): + with __test_base_path() as base_path: + f = tempfile.NamedTemporaryFile() + f.write(xml_content) + f.flush() + dm = DependencyManager( default_base_path=base_path, conf_file=f.name ) + yield dm.dependency_resolvers + https://bitbucket.org/galaxy/galaxy-central/commits/4711ced22745/ Changeset: 4711ced22745 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Add ability to pass arguments to tool resolvers. Use new functionlity to allow a much cleaner, more general solution to the problem outlined in Pull Request 220. Allow creation of a Galaxy package dependency resolver that always falls back to 'default' dependency. I think this should be the default last-ditch behavior if a specified version of a tag is not found, but I understand the desire for it not to be, hopefully making it optional in this fashion is an acceptable middle ground. The original problem is essentially, tool shed installs require an exact version of package requirements to be used, but this very encumbering in rapidly evolving and/or non-toolshed use cases (e.g. my Galaxy-P production and development servers). It is tedius to keep different code bases, environments in such cases. Affected #: 2 files diff -r 0762bd90e85b94e643458c22670de8455dc97f71 -r 4711ced227459680997ffe448ea57abd65349fb2 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -7,7 +7,7 @@ import logging log = logging.getLogger( __name__ ) -from galaxy.util import parse_xml +from galaxy.util import parse_xml, string_as_bool INDETERMINATE_DEPENDENCY = (None, None, None) @@ -65,7 +65,8 @@ resolvers_element = tree.getroot() for resolver_element in resolvers_element.getchildren(): resolver_type = resolver_element.tag - resolver = RESOLVER_CLASSES[resolver_type](self) + resolver_kwds = dict(resolver_element.items()) + resolver = RESOLVER_CLASSES[resolver_type](self, **resolver_kwds) resolvers.append(resolver) return resolvers @@ -78,15 +79,22 @@ class GalaxyPackageDependencyResolver(DependencyResolver): - def __init__(self, dependency_manager): + def __init__(self, dependency_manager, **kwds): self.default_base_path = dependency_manager.default_base_path + ## Galaxy tool shed requires explicit versions on XML elements, + ## this in inconvient for testing or Galaxy instances not utilizing + ## the tool shed so allow a fallback version of the Galaxy package + ## resolver that will just grab 'default' version of exact version + ## unavailable. + self.versionless = string_as_bool(kwds.get('versionless', "false")) + def resolve( self, name, version, type, **kwds ): """ Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a symbolic link (if found). Returns a triple of: env_script, base_path, real_version """ - if version is None: + if version is None or self.versionless: return self._find_dep_default( name, type=type, **kwds ) else: return self._find_dep_versioned( name, version, type=type, **kwds ) @@ -117,8 +125,8 @@ class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): - def __init__(self, dependency_manager): - super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager) + def __init__(self, dependency_manager, **kwds): + super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager, **kwds) def _find_dep_versioned( self, name, version, type='package', **kwds ): installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) diff -r 0762bd90e85b94e643458c22670de8455dc97f71 -r 4711ced227459680997ffe448ea57abd65349fb2 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -158,6 +158,15 @@ assert 'Galaxy' in dependency_resolvers[0].__class__.__name__ assert 'ToolShed' in dependency_resolvers[1].__class__.__name__ + with __parse_resolvers('''<dependency_resolvers> + <galaxy_package /> + <tool_shed_package /> + <galaxy_package versionless="true" /> +</dependency_resolvers> +''') as dependency_resolvers: + assert not dependency_resolvers[0].versionless + assert dependency_resolvers[2].versionless + @contextmanager def __parse_resolvers(xml_content): https://bitbucket.org/galaxy/galaxy-central/commits/73c22fa3ae10/ Changeset: 73c22fa3ae10 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Allow overridding the base_path on 'galaxy_package' tool dependency resolvers. There was some code for doing this earlier but it wasn't configurable in anyway. This version looks like it should work and is cleaner. Affected #: 2 files diff -r 4711ced227459680997ffe448ea57abd65349fb2 -r 73c22fa3ae109e14bd70128c6e00174beae45629 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -80,14 +80,13 @@ class GalaxyPackageDependencyResolver(DependencyResolver): def __init__(self, dependency_manager, **kwds): - self.default_base_path = dependency_manager.default_base_path - ## Galaxy tool shed requires explicit versions on XML elements, ## this in inconvient for testing or Galaxy instances not utilizing ## the tool shed so allow a fallback version of the Galaxy package ## resolver that will just grab 'default' version of exact version ## unavailable. self.versionless = string_as_bool(kwds.get('versionless', "false")) + self.base_path = kwds.get('base_path', dependency_manager.default_base_path) def resolve( self, name, version, type, **kwds ): """ @@ -100,12 +99,12 @@ return self._find_dep_versioned( name, version, type=type, **kwds ) def _find_dep_versioned( self, name, version, type='package', **kwds ): - base_path = self.default_base_path + base_path = self.base_path path = os.path.join( base_path, name, version ) return self._galaxy_package_dep(path, version) def _find_dep_default( self, name, type='package', **kwds ): - base_path = self.default_base_path + base_path = self.base_path path = os.path.join( base_path, name, 'default' ) if os.path.islink( path ): real_path = os.path.realpath( path ) @@ -130,7 +129,7 @@ def _find_dep_versioned( self, name, version, type='package', **kwds ): installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) - base_path = self.default_base_path + base_path = self.base_path if installed_tool_dependency: path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) return self._galaxy_package_dep(path, version) @@ -169,7 +168,7 @@ def _get_set_environment_installed_dependency_script_path( self, installed_tool_dependency, name ): tool_shed_repository = installed_tool_dependency.tool_shed_repository - base_path = self.default_base_path + base_path = self.base_path path = os.path.abspath( os.path.join( base_path, 'environment_settings', name, diff -r 4711ced227459680997ffe448ea57abd65349fb2 -r 73c22fa3ae109e14bd70128c6e00174beae45629 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -167,6 +167,19 @@ assert not dependency_resolvers[0].versionless assert dependency_resolvers[2].versionless + with __parse_resolvers('''<dependency_resolvers> + <galaxy_package /> + <tool_shed_package /> + <galaxy_package base_path="/opt/galaxy/legacy/"/> +</dependency_resolvers> +''') as dependency_resolvers: + # Unspecified base_paths are both default_base_paths + assert dependency_resolvers[0].base_path == dependency_resolvers[1].base_path + # Can specify custom base path... + assert dependency_resolvers[2].base_path == "/opt/galaxy/legacy/" + # ... that is different from the default. + assert dependency_resolvers[0].base_path != dependency_resolvers[2].base_path + @contextmanager def __parse_resolvers(xml_content): https://bitbucket.org/galaxy/galaxy-central/commits/36d6cca02ff2/ Changeset: 36d6cca02ff2 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Modify the tool resolver syntax slightly. <galaxy_packages /> looks better than <galaxy_package /> since it is a whole class of possible dependencies the corresponding resolver could resolve. Affected #: 2 files diff -r 73c22fa3ae109e14bd70128c6e00174beae45629 -r 36d6cca02ff256889ea3a65b256b098431d35192 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -182,6 +182,6 @@ RESOLVER_CLASSES = { - 'tool_shed_package': ToolShedPackageDependencyResolver, - 'galaxy_package': GalaxyPackageDependencyResolver, + 'tool_shed_packages': ToolShedPackageDependencyResolver, + 'galaxy_packages': GalaxyPackageDependencyResolver, } diff -r 73c22fa3ae109e14bd70128c6e00174beae45629 -r 36d6cca02ff256889ea3a65b256b098431d35192 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -143,34 +143,34 @@ def test_parse(): with __parse_resolvers('''<dependency_resolvers> - <tool_shed_package /> - <galaxy_package /> + <tool_shed_packages /> + <galaxy_packages /></dependency_resolvers> ''') as dependency_resolvers: assert 'ToolShed' in dependency_resolvers[0].__class__.__name__ assert 'Galaxy' in dependency_resolvers[1].__class__.__name__ with __parse_resolvers('''<dependency_resolvers> - <galaxy_package /> - <tool_shed_package /> + <galaxy_packages /> + <tool_shed_packages /></dependency_resolvers> ''') as dependency_resolvers: assert 'Galaxy' in dependency_resolvers[0].__class__.__name__ assert 'ToolShed' in dependency_resolvers[1].__class__.__name__ with __parse_resolvers('''<dependency_resolvers> - <galaxy_package /> - <tool_shed_package /> - <galaxy_package versionless="true" /> + <galaxy_packages /> + <tool_shed_packages /> + <galaxy_packages versionless="true" /></dependency_resolvers> ''') as dependency_resolvers: assert not dependency_resolvers[0].versionless assert dependency_resolvers[2].versionless with __parse_resolvers('''<dependency_resolvers> - <galaxy_package /> - <tool_shed_package /> - <galaxy_package base_path="/opt/galaxy/legacy/"/> + <galaxy_packages /> + <tool_shed_packages /> + <galaxy_packages base_path="/opt/galaxy/legacy/"/></dependency_resolvers> ''') as dependency_resolvers: # Unspecified base_paths are both default_base_paths https://bitbucket.org/galaxy/galaxy-central/commits/b7380008b339/ Changeset: b7380008b339 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Refactor actual tool resolvers out into the own module. Cleans up lib/galaxy/tools/deps/__init__.py significantly and makes inter-dependencies more clear. Affected #: 4 files diff -r 36d6cca02ff256889ea3a65b256b098431d35192 -r b7380008b339394d0f2957f778d2b80f00e72727 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -7,9 +7,10 @@ import logging log = logging.getLogger( __name__ ) -from galaxy.util import parse_xml, string_as_bool - -INDETERMINATE_DEPENDENCY = (None, None, None) +from galaxy.util import parse_xml +from .resolvers import INDETERMINATE_DEPENDENCY +from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver +from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver class DependencyManager( object ): @@ -70,117 +71,6 @@ resolvers.append(resolver) return resolvers - -class DependencyResolver(object): - - def resolve( self, name, version, type, **kwds ): - raise NotImplementedError() - - -class GalaxyPackageDependencyResolver(DependencyResolver): - - def __init__(self, dependency_manager, **kwds): - ## Galaxy tool shed requires explicit versions on XML elements, - ## this in inconvient for testing or Galaxy instances not utilizing - ## the tool shed so allow a fallback version of the Galaxy package - ## resolver that will just grab 'default' version of exact version - ## unavailable. - self.versionless = string_as_bool(kwds.get('versionless', "false")) - self.base_path = kwds.get('base_path', dependency_manager.default_base_path) - - def resolve( self, name, version, type, **kwds ): - """ - Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a - symbolic link (if found). Returns a triple of: env_script, base_path, real_version - """ - if version is None or self.versionless: - return self._find_dep_default( name, type=type, **kwds ) - else: - return self._find_dep_versioned( name, version, type=type, **kwds ) - - def _find_dep_versioned( self, name, version, type='package', **kwds ): - base_path = self.base_path - path = os.path.join( base_path, name, version ) - return self._galaxy_package_dep(path, version) - - def _find_dep_default( self, name, type='package', **kwds ): - base_path = self.base_path - path = os.path.join( base_path, name, 'default' ) - if os.path.islink( path ): - real_path = os.path.realpath( path ) - real_version = os.path.basename( real_path ) - return self._galaxy_package_dep(real_path, real_version) - else: - return INDETERMINATE_DEPENDENCY - - def _galaxy_package_dep( self, path, version ): - script = os.path.join( path, 'env.sh' ) - if os.path.exists( script ): - return script, path, version - elif os.path.exists( os.path.join( path, 'bin' ) ): - return None, path, version - return INDETERMINATE_DEPENDENCY - - -class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): - - def __init__(self, dependency_manager, **kwds): - super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager, **kwds) - - def _find_dep_versioned( self, name, version, type='package', **kwds ): - installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) - base_path = self.base_path - if installed_tool_dependency: - path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) - return self._galaxy_package_dep(path, version) - else: - return INDETERMINATE_DEPENDENCY - - def _find_dep_default( self, name, type='package', **kwds ): - if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): - installed_tool_dependency = self._get_installed_dependency( name, type, version=None, **kwds ) - if installed_tool_dependency: - script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) - if script and path: - # Environment settings do not use versions. - return script, path, None - return INDETERMINATE_DEPENDENCY - - def _get_installed_dependency( self, name, type, version=None, **kwds ): - for installed_tool_dependency in kwds.get("installed_tool_dependencies", []): - name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type - if version: - if name_and_type_equal and installed_tool_dependency.version == version: - return installed_tool_dependency - else: - if name_and_type_equal: - return installed_tool_dependency - return None - - def _get_package_installed_dependency_path( self, installed_tool_dependency, base_path, name, version ): - tool_shed_repository = installed_tool_dependency.tool_shed_repository - return os.path.join( base_path, - name, - version, - tool_shed_repository.owner, - tool_shed_repository.name, - tool_shed_repository.installed_changeset_revision ) - - def _get_set_environment_installed_dependency_script_path( self, installed_tool_dependency, name ): - tool_shed_repository = installed_tool_dependency.tool_shed_repository - base_path = self.base_path - path = os.path.abspath( os.path.join( base_path, - 'environment_settings', - name, - tool_shed_repository.owner, - tool_shed_repository.name, - tool_shed_repository.installed_changeset_revision ) ) - if os.path.exists( path ): - script = os.path.join( path, 'env.sh' ) - return script, path, None - return INDETERMINATE_DEPENDENCY - - RESOLVER_CLASSES = { 'tool_shed_packages': ToolShedPackageDependencyResolver, 'galaxy_packages': GalaxyPackageDependencyResolver, diff -r 36d6cca02ff256889ea3a65b256b098431d35192 -r b7380008b339394d0f2957f778d2b80f00e72727 lib/galaxy/tools/deps/resolvers/__init__.py --- /dev/null +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -0,0 +1,18 @@ +from abc import ABCMeta, abstractmethod + +INDETERMINATE_DEPENDENCY = (None, None, None) + + +class DependencyResolver(object): + __metaclass__ = ABCMeta + + @abstractmethod + def resolve( self, name, version, type, **kwds ): + """ + Given inputs describing dependency in the abstract, yield tuple of + (script, bin, version). Here script is the env.sh file to source + before running a job, if that is not found the bin directory will be + appended to the path (if it is not None). Finally, version is the + resolved tool dependency version (which may differ from requested + version for instance if the request version is 'default'.) + """ diff -r 36d6cca02ff256889ea3a65b256b098431d35192 -r b7380008b339394d0f2957f778d2b80f00e72727 lib/galaxy/tools/deps/resolvers/galaxy_packages.py --- /dev/null +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -0,0 +1,51 @@ +from os.path import join, islink, realpath, basename, exists, abspath + +from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY +from galaxy.util import string_as_bool + + +class GalaxyPackageDependencyResolver(DependencyResolver): + + def __init__(self, dependency_manager, **kwds): + ## Galaxy tool shed requires explicit versions on XML elements, + ## this in inconvient for testing or Galaxy instances not utilizing + ## the tool shed so allow a fallback version of the Galaxy package + ## resolver that will just grab 'default' version of exact version + ## unavailable. + self.versionless = string_as_bool(kwds.get('versionless', "false")) + self.base_path = abspath( kwds.get('base_path', dependency_manager.default_base_path) ) + + def resolve( self, name, version, type, **kwds ): + """ + Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a + symbolic link (if found). Returns a triple of: env_script, base_path, real_version + """ + if version is None or self.versionless: + return self._find_dep_default( name, type=type, **kwds ) + else: + return self._find_dep_versioned( name, version, type=type, **kwds ) + + def _find_dep_versioned( self, name, version, type='package', **kwds ): + base_path = self.base_path + path = join( base_path, name, version ) + return self._galaxy_package_dep(path, version) + + def _find_dep_default( self, name, type='package', **kwds ): + base_path = self.base_path + path = join( base_path, name, 'default' ) + if islink( path ): + real_path = realpath( path ) + real_version = basename( real_path ) + return self._galaxy_package_dep(real_path, real_version) + else: + return INDETERMINATE_DEPENDENCY + + def _galaxy_package_dep( self, path, version ): + script = join( path, 'env.sh' ) + if exists( script ): + return script, path, version + elif exists( join( path, 'bin' ) ): + return None, path, version + return INDETERMINATE_DEPENDENCY + +__all__ = [GalaxyPackageDependencyResolver] diff -r 36d6cca02ff256889ea3a65b256b098431d35192 -r b7380008b339394d0f2957f778d2b80f00e72727 lib/galaxy/tools/deps/resolvers/tool_shed_packages.py --- /dev/null +++ b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py @@ -0,0 +1,66 @@ +from os.path import abspath, join, exists + +from .galaxy_packages import GalaxyPackageDependencyResolver +from ..resolvers import INDETERMINATE_DEPENDENCY + + +class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): + + def __init__(self, dependency_manager, **kwds): + super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager, **kwds) + + def _find_dep_versioned( self, name, version, type='package', **kwds ): + installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) + base_path = self.base_path + if installed_tool_dependency: + path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) + return self._galaxy_package_dep(path, version) + else: + return INDETERMINATE_DEPENDENCY + + def _find_dep_default( self, name, type='package', **kwds ): + if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): + installed_tool_dependency = self._get_installed_dependency( name, type, version=None, **kwds ) + if installed_tool_dependency: + script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) + if script and path: + # Environment settings do not use versions. + return script, path, None + return INDETERMINATE_DEPENDENCY + + def _get_installed_dependency( self, name, type, version=None, **kwds ): + installed_tool_dependencies = kwds.get("installed_tool_dependencies", []) + for installed_tool_dependency in (installed_tool_dependencies or []): + name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type + if version: + if name_and_type_equal and installed_tool_dependency.version == version: + return installed_tool_dependency + else: + if name_and_type_equal: + return installed_tool_dependency + return None + + def _get_package_installed_dependency_path( self, installed_tool_dependency, base_path, name, version ): + tool_shed_repository = installed_tool_dependency.tool_shed_repository + return join( base_path, + name, + version, + tool_shed_repository.owner, + tool_shed_repository.name, + tool_shed_repository.installed_changeset_revision ) + + def _get_set_environment_installed_dependency_script_path( self, installed_tool_dependency, name ): + tool_shed_repository = installed_tool_dependency.tool_shed_repository + base_path = self.base_path + path = abspath( join( base_path, + 'environment_settings', + name, + tool_shed_repository.owner, + tool_shed_repository.name, + tool_shed_repository.installed_changeset_revision ) ) + if exists( path ): + script = join( path, 'env.sh' ) + return script, path, None + return INDETERMINATE_DEPENDENCY + +__all__ = [ToolShedPackageDependencyResolver] https://bitbucket.org/galaxy/galaxy-central/commits/21b9e99a73fe/ Changeset: 21b9e99a73fe User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Introduce higher-level abstraction describing dependency. Passing the tuple (script, path, version) around is insufficient to represent other kinds of potential dependency load commands (namely module loads), this generalization allows more expressivity. I believe this abstraction also moves some logic out of the tool class that is better encapsulated in the deps module and submodules. Affected #: 5 files diff -r b7380008b339394d0f2957f778d2b80f00e72727 -r 21b9e99a73fe96721498dda9751680741f78e946 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -40,7 +40,7 @@ from galaxy.tools.actions import DefaultToolAction from galaxy.tools.actions.data_source import DataSourceToolAction from galaxy.tools.actions.data_manager import DataManagerToolAction -from galaxy.tools.deps import DependencyManager +from galaxy.tools.deps import DependencyManager, INDETERMINATE_DEPENDENCY from galaxy.tools.parameters import check_param, params_from_strings, params_to_strings from galaxy.tools.parameters.basic import (BaseURLToolParameter, DataToolParameter, HiddenToolParameter, LibraryDatasetToolParameter, @@ -2681,20 +2681,17 @@ installed_tool_dependencies = None for requirement in self.requirements: log.debug( "Building dependency shell command for dependency '%s'", requirement.name ) - script_file = None - base_path = None - version = None + dependency = INDETERMINATE_DEPENDENCY if requirement.type in [ 'package', 'set_environment' ]: - script_file, base_path, version = self.app.toolbox.dependency_manager.find_dep( name=requirement.name, - version=requirement.version, - type=requirement.type, - installed_tool_dependencies=installed_tool_dependencies ) - if script_file is None and base_path is None: + dependency = self.app.toolbox.dependency_manager.find_dep( name=requirement.name, + version=requirement.version, + type=requirement.type, + installed_tool_dependencies=installed_tool_dependencies ) + dependency_commands = dependency.shell_commands( requirement ) + if not dependency_commands: log.warn( "Failed to resolve dependency on '%s', ignoring", requirement.name ) - elif requirement.type == 'package' and script_file is None: - commands.append( 'PACKAGE_BASE=%s; export PACKAGE_BASE; PATH="%s/bin:$PATH"; export PATH' % ( base_path, base_path ) ) else: - commands.append( 'PACKAGE_BASE=%s; export PACKAGE_BASE; . %s' % ( base_path, script_file ) ) + commands.append(dependency_commands) return commands def build_redirect_url_params( self, param_dict ): """ diff -r b7380008b339394d0f2957f778d2b80f00e72727 -r 21b9e99a73fe96721498dda9751680741f78e946 lib/galaxy/tools/deps/resolvers/__init__.py --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -1,9 +1,7 @@ from abc import ABCMeta, abstractmethod -INDETERMINATE_DEPENDENCY = (None, None, None) - -class DependencyResolver(object): +class DependencyResolver( object ): __metaclass__ = ABCMeta @abstractmethod @@ -16,3 +14,21 @@ resolved tool dependency version (which may differ from requested version for instance if the request version is 'default'.) """ + + +class Dependency( object ): + __metaclass__ = ABCMeta + + @abstractmethod + def shell_commands( self, requirement ): + """ + Return shell commands to enable this dependency. + """ + + +class NullDependency( Dependency ): + + def shell_commands( self, requirement ): + return None + +INDETERMINATE_DEPENDENCY = NullDependency() diff -r b7380008b339394d0f2957f778d2b80f00e72727 -r 21b9e99a73fe96721498dda9751680741f78e946 lib/galaxy/tools/deps/resolvers/galaxy_packages.py --- a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -1,8 +1,11 @@ from os.path import join, islink, realpath, basename, exists, abspath -from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY +from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY, Dependency from galaxy.util import string_as_bool +import logging +log = logging.getLogger( __name__ ) + class GalaxyPackageDependencyResolver(DependencyResolver): @@ -43,9 +46,28 @@ def _galaxy_package_dep( self, path, version ): script = join( path, 'env.sh' ) if exists( script ): - return script, path, version + return GalaxyPackageDependency(script, path, version) elif exists( join( path, 'bin' ) ): - return None, path, version + return GalaxyPackageDependency(None, path, version) return INDETERMINATE_DEPENDENCY -__all__ = [GalaxyPackageDependencyResolver] + +class GalaxyPackageDependency(Dependency): + + def __init__( self, script, path, version ): + self.script = script + self.path = path + self.version = version + + def shell_commands( self, requirement ): + base_path = self.path + if self.script is None and base_path is None: + log.warn( "Failed to resolve dependency on '%s', ignoring", requirement.name ) + commands = None + elif requirement.type == 'package' and self.script is None: + commands = 'PACKAGE_BASE=%s; export PACKAGE_BASE; PATH="%s/bin:$PATH"; export PATH' % ( base_path, base_path ) + else: + commands = 'PACKAGE_BASE=%s; export PACKAGE_BASE; . %s' % ( base_path, self.script ) + return commands + +__all__ = [GalaxyPackageDependencyResolver, GalaxyPackageDependency] diff -r b7380008b339394d0f2957f778d2b80f00e72727 -r 21b9e99a73fe96721498dda9751680741f78e946 lib/galaxy/tools/deps/resolvers/tool_shed_packages.py --- a/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py +++ b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py @@ -1,6 +1,6 @@ from os.path import abspath, join, exists -from .galaxy_packages import GalaxyPackageDependencyResolver +from .galaxy_packages import GalaxyPackageDependencyResolver, GalaxyPackageDependency from ..resolvers import INDETERMINATE_DEPENDENCY @@ -22,10 +22,10 @@ if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): installed_tool_dependency = self._get_installed_dependency( name, type, version=None, **kwds ) if installed_tool_dependency: - script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) - if script and path: + dependency = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) + if dependency.script and dependency.path: # Environment settings do not use versions. - return script, path, None + return GalaxyPackageDependency(dependency.script, dependency.path, None) return INDETERMINATE_DEPENDENCY def _get_installed_dependency( self, name, type, version=None, **kwds ): @@ -60,7 +60,7 @@ tool_shed_repository.installed_changeset_revision ) ) if exists( path ): script = join( path, 'env.sh' ) - return script, path, None + return GalaxyPackageDependency(script, path, None) return INDETERMINATE_DEPENDENCY __all__ = [ToolShedPackageDependencyResolver] diff -r b7380008b339394d0f2957f778d2b80f00e72727 -r 21b9e99a73fe96721498dda9751680741f78e946 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -2,9 +2,11 @@ import os.path from os import makedirs, symlink from shutil import rmtree -from galaxy.tools.deps import DependencyManager +from galaxy.tools.deps import DependencyManager, INDETERMINATE_DEPENDENCY +from galaxy.tools.deps.resolvers.galaxy_packages import GalaxyPackageDependency from galaxy.util.bunch import Bunch from contextlib import contextmanager +from subprocess import Popen, PIPE def test_tool_dependencies(): @@ -24,24 +26,23 @@ __touch( os.path.join( p, "env.sh" ) ) dm = DependencyManager( default_base_path=base_path ) - d1_script, d1_path, d1_version = dm.find_dep( "dep1", "1.0" ) - assert d1_script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) - assert d1_path == os.path.join( base_path, 'dep1', '1.0' ) - assert d1_version == "1.0" - d2_script, d2_path, d2_version = dm.find_dep( "dep1", "2.0" ) - assert d2_script == None - assert d2_path == os.path.join( base_path, 'dep1', '2.0' ) - assert d2_version == "2.0" + dependency = dm.find_dep( "dep1", "1.0" ) + assert dependency.script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) + assert dependency.path == os.path.join( base_path, 'dep1', '1.0' ) + assert dependency.version == "1.0" + dependency = dm.find_dep( "dep1", "2.0" ) + assert dependency.script == None + assert dependency.path == os.path.join( base_path, 'dep1', '2.0' ) + assert dependency.version == "2.0" ## Test default versions symlink( os.path.join( base_path, 'dep1', '2.0'), os.path.join( base_path, 'dep1', 'default' ) ) - default_script, default_path, default_version = dm.find_dep( "dep1", None ) - assert default_version == "2.0" + dependency = dm.find_dep( "dep1", None ) + assert dependency.version == "2.0" ## Test default will not be fallen back upon by default - default_script, default_path, default_version = dm.find_dep( "dep1", "2.1" ) - assert default_script == None - assert default_version == None + dependency = dm.find_dep( "dep1", "2.1" ) + assert dependency == INDETERMINATE_DEPENDENCY TEST_REPO_USER = "devteam" @@ -56,9 +57,9 @@ dm = DependencyManager( default_base_path=base_path ) env_settings_dir = os.path.join(base_path, "environment_settings", TEST_REPO_NAME, TEST_REPO_USER, TEST_REPO_NAME, TEST_REPO_CHANGESET) os.makedirs(env_settings_dir) - d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=None, type='set_environment', installed_tool_dependencies=[test_repo] ) - assert d1_version == None - assert d1_script == os.path.join(env_settings_dir, "env.sh"), d1_script + dependency = dm.find_dep( TEST_REPO_NAME, version=None, type='set_environment', installed_tool_dependencies=[test_repo] ) + assert dependency.version == None + assert dependency.script == os.path.join(env_settings_dir, "env.sh") def test_toolshed_package_requirements(): @@ -66,9 +67,9 @@ test_repo = __build_test_repo('package', version=TEST_VERSION) dm = DependencyManager( default_base_path=base_path ) package_dir = __build_ts_test_package(base_path) - d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) - assert d1_version == TEST_VERSION, d1_version - assert d1_script == os.path.join(package_dir, "env.sh"), d1_script + dependency = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) + assert dependency.version == TEST_VERSION + assert dependency.script == os.path.join(package_dir, "env.sh") def test_toolshed_tools_fallback_on_manual_dependencies(): @@ -76,9 +77,9 @@ dm = DependencyManager( default_base_path=base_path ) test_repo = __build_test_repo('package', version=TEST_VERSION) env_path = __setup_galaxy_package_dep(base_path, "dep1", "1.0") - d1_script, d1_path, d1_version = dm.find_dep( "dep1", version="1.0", type='package', installed_tool_dependencies=[test_repo] ) - assert d1_version == "1.0" - assert d1_script == env_path + dependency = dm.find_dep( "dep1", version="1.0", type='package', installed_tool_dependencies=[test_repo] ) + assert dependency.version == "1.0" + assert dependency.script == env_path def test_toolshed_greater_precendence(): @@ -88,9 +89,9 @@ ts_package_dir = __build_ts_test_package(base_path) gx_env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION) ts_env_path = os.path.join(ts_package_dir, "env.sh") - d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) - assert d1_script != gx_env_path # Not the galaxy path, it should be the tool shed path used. - assert d1_script == ts_env_path + dependency = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) + assert dependency.script != gx_env_path # Not the galaxy path, it should be the tool shed path used. + assert dependency.script == ts_env_path def __build_ts_test_package(base_path, script_contents=''): @@ -99,6 +100,18 @@ return package_dir +def test_galaxy_dependency_object_script(): + with __test_base_path() as base_path: + ## Create env.sh file that just exports variable FOO and verify it + ## shell_commands export it correctly. + env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION, "export FOO=\"bar\"") + dependency = GalaxyPackageDependency(env_path, os.path.dirname(env_path), TEST_VERSION) + command = ["bash", "-c", "%s; echo \"$FOO\"" % dependency.shell_commands(Bunch(type="package"))] + process = Popen(command, stdout=PIPE) + output = process.communicate()[0].strip() + assert output == 'bar' + + def __setup_galaxy_package_dep(base_path, name, version, contents=""): dep_directory = os.path.join( base_path, name, version ) env_path = os.path.join( dep_directory, "env.sh" ) https://bitbucket.org/galaxy/galaxy-central/commits/31f6c565ca43/ Changeset: 31f6c565ca43 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Outline of integrating environment modules. This is completely untested and undoubtedly needs some fixes. The point is largely to outline how it could be done and let someone with interest in using this go in and flush out the implementation. Note: This is a community contributed feature and support from the core Galaxy team for it will be minimal. Affected #: 3 files diff -r 21b9e99a73fe96721498dda9751680741f78e946 -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -11,6 +11,7 @@ from .resolvers import INDETERMINATE_DEPENDENCY from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver +from .resolvers.modules import ModuleDependencyResolver class DependencyManager( object ): @@ -74,4 +75,5 @@ RESOLVER_CLASSES = { 'tool_shed_packages': ToolShedPackageDependencyResolver, 'galaxy_packages': GalaxyPackageDependencyResolver, + 'modules': ModuleDependencyResolver, } diff -r 21b9e99a73fe96721498dda9751680741f78e946 -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea lib/galaxy/tools/deps/resolvers/modules.py --- /dev/null +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -0,0 +1,137 @@ +""" +This file contains the outline of an implementation to load environment modules +(http://modules.sourceforge.net/). + +This is a community contributed feature and the core Galaxy team does utilize +it, hence support for it will be minimal. The Galaxy team eagerly welcomes +community contribution and maintenance however. +""" +from os.path import exists, isdir, join +from StringIO import StringIO +from subprocess import Popen, PIPE + +from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY, Dependency +from galaxy.util import string_as_bool + +import logging +log = logging.getLogger( __name__ ) + + +DEFAULT_MODULE_COMMAND = 'module' +DEFAULT_MODULE_DIRECTORY = '/usr/share/modules/modulefiles' +DEFAULT_INDICATOR = '(default)' +DEFAULT_MODULE_PREFETCH = "true" +UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find modules by [%s], find_by should be one of %s" + + +class ModuleDependencyResolver(DependencyResolver): + + def __init__(self, dependency_manager, **kwds): + self.module_command = kwds.get('command', DEFAULT_MODULE_COMMAND) + self.versionless = string_as_bool(kwds.get('versionless', 'false')) + find_by = kwds.get('find_by', 'avail') + prefetch = string_as_bool(kwds.get('prefetch', DEFAULT_MODULE_PREFETCH)) + if find_by == 'directory': + directory = kwds.get('directory', DEFAULT_MODULE_DIRECTORY) + self.module_checker = DirectoryModuleChecker(self, directory, prefetch) + elif find_by == 'avail': + self.module_checker = AvailModuleChecker(self, prefetch) + else: + raise Exception(UNKNOWN_FIND_BY_MESSAGE % (find_by, ["avail", "directory"])) + + def resolve( self, name, version, type, **kwds ): + if type != "package": + return INDETERMINATE_DEPENDENCY + + if self.versionless: + version = None + + if self.__has_module(name, version): + return ModuleDependency(self, name, version) + + return INDETERMINATE_DEPENDENCY + + def __has_module(self, name, version): + return self.module_checker.has_module(name, version) + + +class DirectoryModuleChecker(object): + + def __init__(self, module_dependency_resolver, directory, prefetch): + self.module_dependency_resolver = module_dependency_resolver + self.directory = directory + if prefetch: + log.warn("Created module dependency resolver with prefetch enabled, but directory module checker does not support this.") + pass + + def has_module(self, module, version): + module_directory = join(self.directory, module) + has_module_directory = isdir( join( self.directory, module ) ) + if not version: + has_module = has_module_directory + else: + modulefile = join( module_directory, version ) + has_modulefile = exists( modulefile ) + has_module = has_module_directory and has_modulefile + return has_module + + +class AvailModuleChecker(object): + + def __init__(self, module_dependency_resolver, prefetch): + self.module_dependency_resolver = module_dependency_resolver + if prefetch: + prefetched_modules = [] + for module in self.__modules(): + prefetched_modules.append(module) + else: + prefetched_modules = None + self.prefetched_modules = prefetched_modules + + def has_module(self, module, version): + module_generator = self.prefetched_modules + if module_generator is None: + module_generator = self.__modules() + + for module_name, module_version in module_generator: + names_match = module == module_name + module_match = names_match and (version == None or module_version == version) + if module_match: + return True + return False + + def __modules(self): + raw_output = self.__module_avail_ouptut() + for line in StringIO(raw_output): + line = line and line.strip() + if not line or line.startswith("-"): + continue + + line_modules = line.split() + for module in line_modules: + if module.endswith(DEFAULT_INDICATOR): + module = module[0:-len(DEFAULT_INDICATOR)].strip() + module_parts = module.split('/') + module_version = None + if len(module_parts) == 2: + module_version = module_parts[1] + module_name = module_parts[0] + yield module_name, module_version + + def __module_avail_ouptut(self): + avail_command = '%s avail' % self.module_dependency_resolver.module_command + return Popen([avail_command], shell=True, stderr=PIPE).communicate()[1] + + +class ModuleDependency(Dependency): + + def __init__(self, module_dependency_resolver, module_name, module_version=None): + self.module_dependency_resolver = module_dependency_resolver + self.module_name = module_name + self.module_version = module_version + + def shell_commands(self, requirement): + command = '%s load %s' % (self.module_dependency_resolver.module_command, self.module_name) + if self.module_version: + command = '%s/%s' % self.module_version + return command diff -r 21b9e99a73fe96721498dda9751680741f78e946 -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -1,9 +1,11 @@ import tempfile import os.path -from os import makedirs, symlink +from stat import S_IXUSR +from os import makedirs, symlink, stat, chmod from shutil import rmtree from galaxy.tools.deps import DependencyManager, INDETERMINATE_DEPENDENCY from galaxy.tools.deps.resolvers.galaxy_packages import GalaxyPackageDependency +from galaxy.tools.deps.resolvers.modules import ModuleDependencyResolver from galaxy.util.bunch import Bunch from contextlib import contextmanager from subprocess import Popen, PIPE @@ -100,6 +102,50 @@ return package_dir +def test_module_dependency_resolver(): + with __test_base_path() as temp_directory: + module_script = os.path.join(temp_directory, "module") + with open(module_script, 'w') as f: + f.write('''#!/bin/sh +cat %s/example_output 1>&2; +''' % temp_directory) + with open(os.path.join(temp_directory, "example_output"), "w") as f: + # Subset of module avail from MSI cluster. + f.write(''' +-------------------------- /soft/modules/modulefiles --------------------------- +JAGS/3.2.0-gcc45 +JAGS/3.3.0-gcc4.7.2 +ProbABEL/0.1-3 +ProbABEL/0.1-9e +R/2.12.2 +R/2.13.1 +R/2.14.1 +R/2.15.0 +R/2.15.1 +R/3.0.1(default) +abokia-blast/2.0.2-130524/ompi_intel +abokia-blast/2.0.2-130630/ompi_intel + +--------------------------- /soft/intel/modulefiles ---------------------------- +advisor/2013/update1 intel/11.1.075 mkl/10.2.1.017 +advisor/2013/update2 intel/11.1.080 mkl/10.2.5.035 +advisor/2013/update3 intel/12.0 mkl/10.2.7.041 +''') + st = os.stat(module_script) + chmod(module_script, st.st_mode | S_IXUSR) + resolver = ModuleDependencyResolver(None, command=module_script) + module = resolver.resolve( name="R", version=None, type="package" ) + assert module.module_name == "R" + assert module.module_version == None + + module = resolver.resolve( name="R", version="3.0.1", type="package" ) + assert module.module_name == "R" + assert module.module_version == "3.0.1" + + module = resolver.resolve( name="R", version="3.0.4", type="package" ) + assert module == INDETERMINATE_DEPENDENCY + + def test_galaxy_dependency_object_script(): with __test_base_path() as base_path: ## Create env.sh file that just exports variable FOO and verify it @@ -189,11 +235,30 @@ # Unspecified base_paths are both default_base_paths assert dependency_resolvers[0].base_path == dependency_resolvers[1].base_path # Can specify custom base path... - assert dependency_resolvers[2].base_path == "/opt/galaxy/legacy/" + assert dependency_resolvers[2].base_path == "/opt/galaxy/legacy" # ... that is different from the default. assert dependency_resolvers[0].base_path != dependency_resolvers[2].base_path +def test_config_module_defaults(): + with __parse_resolvers('''<dependency_resolvers> + <modules /> +</dependency_resolvers> +''') as dependency_resolvers: + module_resolver = dependency_resolvers[0] + assert module_resolver.module_command == "module" + assert module_resolver.module_checker.__class__.__name__ == "AvailModuleChecker" + + +def test_config_module_directory_searcher(): + with __parse_resolvers('''<dependency_resolvers> + <modules find_by="directory" directory="/opt/Modules/modulefiles" /> +</dependency_resolvers> +''') as dependency_resolvers: + module_resolver = dependency_resolvers[0] + assert module_resolver.module_checker.directory == "/opt/Modules/modulefiles" + + @contextmanager def __parse_resolvers(xml_content): with __test_base_path() as base_path: https://bitbucket.org/galaxy/galaxy-central/commits/54b9d1a2b1a9/ Changeset: 54b9d1a2b1a9 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Dynamically load tool dependency resolver plugins. This largely mirrors the code for the dynamic job runner so created common util code meant to be shared between dynamic job destinations and dynamic dependency resolves out into galaxy.util.submodules. TODO: Update job mapper to use this. Affected #: 5 files diff -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -8,10 +8,11 @@ log = logging.getLogger( __name__ ) from galaxy.util import parse_xml + from .resolvers import INDETERMINATE_DEPENDENCY from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver -from .resolvers.modules import ModuleDependencyResolver +from galaxy.util.submodules import submodules class DependencyManager( object ): @@ -35,6 +36,7 @@ if not os.path.isdir( default_base_path ): log.warn( "Path '%s' is not directory, ignoring", default_base_path ) self.default_base_path = os.path.abspath( default_base_path ) + self.resolver_classes = self.__resolvers_dict() self.dependency_resolvers = self.__build_dependency_resolvers( conf_file ) @@ -68,12 +70,19 @@ for resolver_element in resolvers_element.getchildren(): resolver_type = resolver_element.tag resolver_kwds = dict(resolver_element.items()) - resolver = RESOLVER_CLASSES[resolver_type](self, **resolver_kwds) + resolver = self.resolver_classes[resolver_type](self, **resolver_kwds) resolvers.append(resolver) return resolvers -RESOLVER_CLASSES = { - 'tool_shed_packages': ToolShedPackageDependencyResolver, - 'galaxy_packages': GalaxyPackageDependencyResolver, - 'modules': ModuleDependencyResolver, -} + def __resolvers_dict( self ): + resolver_dict = {} + for resolver_module in self.__resolver_modules(): + for clazz in resolver_module.__all__: + resolver_type = getattr(clazz, 'resolver_type', None) + if resolver_type: + resolver_dict[resolver_type] = clazz + return resolver_dict + + def __resolver_modules( self ): + import galaxy.tools.deps.resolvers + return submodules( galaxy.tools.deps.resolvers ) diff -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 lib/galaxy/tools/deps/resolvers/galaxy_packages.py --- a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -8,6 +8,7 @@ class GalaxyPackageDependencyResolver(DependencyResolver): + resolver_type = "galaxy_packages" def __init__(self, dependency_manager, **kwds): ## Galaxy tool shed requires explicit versions on XML elements, diff -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 lib/galaxy/tools/deps/resolvers/modules.py --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -25,6 +25,7 @@ class ModuleDependencyResolver(DependencyResolver): + resolver_type = "modules" def __init__(self, dependency_manager, **kwds): self.module_command = kwds.get('command', DEFAULT_MODULE_COMMAND) @@ -135,3 +136,5 @@ if self.module_version: command = '%s/%s' % self.module_version return command + +__all__ = [ModuleDependencyResolver] diff -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 lib/galaxy/tools/deps/resolvers/tool_shed_packages.py --- a/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py +++ b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py @@ -5,6 +5,7 @@ class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): + resolver_type = "tool_shed_packages" def __init__(self, dependency_manager, **kwds): super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager, **kwds) diff -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 lib/galaxy/util/submodules.py --- /dev/null +++ b/lib/galaxy/util/submodules.py @@ -0,0 +1,30 @@ +from os import listdir +import logging +log = logging.getLogger( __name__ ) + + +def submodules( module ): + unsorted_submodule_names = __submodule_names( module ) + submodule_names = sorted( unsorted_submodule_names, reverse=True ) + submodules = [] + for submodule_name in submodule_names: + full_submodule = "%s.%s" % ( module.__name__, submodule_name ) + try: + __import__( full_submodule ) + submodule = getattr( module, submodule_name ) + submodules.append( submodule ) + except BaseException, exception: + exception_str = str( exception ) + message = "%s dynamic module could not be loaded: %s" % ( full_submodule, exception_str ) + log.debug( message ) + return submodules + + +def __submodule_names( module ): + module_dir = module.__path__[ 0 ] + names = [] + for fname in listdir( module_dir ): + if not( fname.startswith( "_" ) ) and fname.endswith( ".py" ): + submodule_name = fname[ :-len( ".py" ) ] + names.append( submodule_name ) + return names https://bitbucket.org/galaxy/galaxy-central/commits/6f54a6d1fc23/ Changeset: 6f54a6d1fc23 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Move requirement/dependency logic out of galaxy.tools and into galaxy.tools.deps, test. Moved ToolRequirement out into its own module along with parsing. Also moved all requirements handling logic into DependencyManager. Now the Tool class knows nothing about the internals or requirements or how to process dependencies. There is also greater unit test coverage for these concepts. Affected #: 4 files diff -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 -r 6f54a6d1fc23b16e11005f166fe84e73754f6cae lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -41,6 +41,7 @@ from galaxy.tools.actions.data_source import DataSourceToolAction from galaxy.tools.actions.data_manager import DataManagerToolAction from galaxy.tools.deps import DependencyManager, INDETERMINATE_DEPENDENCY +from galaxy.tools.deps.requirements import parse_requirements_from_xml from galaxy.tools.parameters import check_param, params_from_strings, params_to_strings from galaxy.tools.parameters.basic import (BaseURLToolParameter, DataToolParameter, HiddenToolParameter, LibraryDatasetToolParameter, @@ -938,15 +939,6 @@ def __iter__( self ): return iter( ( self.format, self.metadata_source, self.parent ) ) -class ToolRequirement( object ): - """ - Represents an external requirement that must be available for the tool to run (for example, a program, package, or library). - Requirements can optionally assert a specific version. - """ - def __init__( self, name=None, type=None, version=None ): - self.name = name - self.type = type - self.version = version class Tool( object, Dictifiable ): """ @@ -1236,10 +1228,7 @@ else: self.tests = None # Requirements (dependencies) - self.requirements = [] - requirements_elem = root.find( "requirements" ) - if requirements_elem: - self.parse_requirements( requirements_elem ) + self.requirements = parse_requirements_from_xml( root ) # Determine if this tool can be used in workflows self.is_workflow_compatible = self.check_workflow_compatible(root) # Trackster configuration. @@ -1810,17 +1799,6 @@ for name in param.get_dependencies(): context[ name ].refresh_on_change = True return param - def parse_requirements( self, requirements_elem ): - """ - Parse each requirement from the <requirements> element and add to - self.requirements - """ - for requirement_elem in requirements_elem.findall( 'requirement' ): - name = xml_text( requirement_elem ) - type = requirement_elem.get( "type", "package" ) - version = requirement_elem.get( "version", None ) - requirement = ToolRequirement( name=name, type=type, version=version ) - self.requirements.append( requirement ) def populate_tool_shed_info( self ): if self.repository_id is not None and 'ToolShedRepository' in self.app.model: @@ -2672,27 +2650,16 @@ command_line = command_line.replace(executable, abs_executable, 1) command_line = self.interpreter + " " + command_line return command_line + def build_dependency_shell_commands( self ): """Return a list of commands to be run to populate the current environment to include this tools requirements.""" - commands = [] if self.tool_shed_repository: installed_tool_dependencies = self.tool_shed_repository.installed_tool_dependencies else: installed_tool_dependencies = None - for requirement in self.requirements: - log.debug( "Building dependency shell command for dependency '%s'", requirement.name ) - dependency = INDETERMINATE_DEPENDENCY - if requirement.type in [ 'package', 'set_environment' ]: - dependency = self.app.toolbox.dependency_manager.find_dep( name=requirement.name, - version=requirement.version, - type=requirement.type, - installed_tool_dependencies=installed_tool_dependencies ) - dependency_commands = dependency.shell_commands( requirement ) - if not dependency_commands: - log.warn( "Failed to resolve dependency on '%s', ignoring", requirement.name ) - else: - commands.append(dependency_commands) - return commands + return self.app.toolbox.dependency_manager.dependency_shell_commands( self.requirements, + installed_tool_dependencies=installed_tool_dependencies ) + def build_redirect_url_params( self, param_dict ): """ Substitute parameter values into self.redirect_url_params diff -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 -r 6f54a6d1fc23b16e11005f166fe84e73754f6cae lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -39,6 +39,22 @@ self.resolver_classes = self.__resolvers_dict() self.dependency_resolvers = self.__build_dependency_resolvers( conf_file ) + def dependency_shell_commands( self, requirements, **kwds ): + commands = [] + for requirement in requirements: + log.debug( "Building dependency shell command for dependency '%s'", requirement.name ) + dependency = INDETERMINATE_DEPENDENCY + if requirement.type in [ 'package', 'set_environment' ]: + dependency = self.find_dep( name=requirement.name, + version=requirement.version, + type=requirement.type, + **kwds ) + dependency_commands = dependency.shell_commands( requirement ) + if not dependency_commands: + log.warn( "Failed to resolve dependency on '%s', ignoring", requirement.name ) + else: + commands.append( dependency_commands ) + return commands def find_dep( self, name, version=None, type='package', **kwds ): for resolver in self.dependency_resolvers: diff -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 -r 6f54a6d1fc23b16e11005f166fe84e73754f6cae lib/galaxy/tools/deps/requirements.py --- /dev/null +++ b/lib/galaxy/tools/deps/requirements.py @@ -0,0 +1,57 @@ +from galaxy.util import xml_text + +DEFAULT_REQUIREMENT_TYPE = "package" +DEFAULT_REQUIREMENT_VERSION = None + + +class ToolRequirement( object ): + """ + Represents an external requirement that must be available for the tool to + run (for example, a program, package, or library). Requirements can + optionally assert a specific version. + """ + def __init__( self, name=None, type=None, version=None ): + self.name = name + self.type = type + self.version = version + + +def parse_requirements_from_xml( xml_root ): + """ + + >>> from galaxy.util import parse_xml + >>> from elementtree import ElementTree + >>> def load_requirements( contents ): + ... contents_document = '''<tool><requirements>%s</requirements></tool>''' + ... root = ElementTree.fromstring( contents_document % contents ) + ... return parse_requirements_from_xml( root ) + >>> reqs = load_requirements('''<requirement>bwa</requirement>''') + >>> reqs[0].name + 'bwa' + >>> reqs[0].version is None + True + >>> reqs[0].type + 'package' + >>> reqs = load_requirements('''<requirement type="binary" version="1.3.3">cufflinks</requirement>''') + >>> reqs[0].name + 'cufflinks' + >>> reqs[0].version + '1.3.3' + >>> reqs[0].type + 'binary' + """ + requirements_elem = xml_root.find( "requirements" ) + + requirement_elems = [] + if requirements_elem: + requirement_elems = requirements_elem.findall( 'requirement' ) + + requirements = [] + for requirement_elem in requirement_elems: + name = xml_text( requirement_elem ) + type = requirement_elem.get( "type", DEFAULT_REQUIREMENT_TYPE ) + version = requirement_elem.get( "version", DEFAULT_REQUIREMENT_VERSION ) + requirement = ToolRequirement( name=name, type=type, version=version ) + requirements.append( requirement ) + + return requirements diff -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 -r 6f54a6d1fc23b16e11005f166fe84e73754f6cae test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -152,10 +152,25 @@ ## shell_commands export it correctly. env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION, "export FOO=\"bar\"") dependency = GalaxyPackageDependency(env_path, os.path.dirname(env_path), TEST_VERSION) - command = ["bash", "-c", "%s; echo \"$FOO\"" % dependency.shell_commands(Bunch(type="package"))] - process = Popen(command, stdout=PIPE) - output = process.communicate()[0].strip() - assert output == 'bar' + __assert_foo_exported( dependency.shell_commands( Bunch( type="package" ) ) ) + + +def test_shell_commands_built(): + ## Test that dependency manager builds valid shell commands for a list of + ## requirements. + with __test_base_path() as base_path: + dm = DependencyManager( default_base_path=base_path ) + __setup_galaxy_package_dep( base_path, TEST_REPO_NAME, TEST_VERSION, contents="export FOO=\"bar\"" ) + mock_requirements = [ Bunch(type="package", version=TEST_VERSION, name=TEST_REPO_NAME ) ] + commands = dm.dependency_shell_commands( mock_requirements ) + __assert_foo_exported( commands ) + + +def __assert_foo_exported( commands ): + command = ["bash", "-c", "%s; echo \"$FOO\"" % "".join(commands)] + process = Popen(command, stdout=PIPE) + output = process.communicate()[0].strip() + assert output == 'bar' def __setup_galaxy_package_dep(base_path, name, version, contents=""): https://bitbucket.org/galaxy/galaxy-central/commits/dc9c0e065be0/ Changeset: dc9c0e065be0 User: sjguest Date: 2013-10-17 06:15:21 Summary: Fixed ModuleDependencyResolver, versionless fallback, and modulepath Affected #: 1 file diff -r 6f54a6d1fc23b16e11005f166fe84e73754f6cae -r dc9c0e065be01f8d16b35eec7b53098fdfa0fd05 lib/galaxy/tools/deps/resolvers/modules.py --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -6,6 +6,7 @@ it, hence support for it will be minimal. The Galaxy team eagerly welcomes community contribution and maintenance however. """ +from os import environ from os.path import exists, isdir, join from StringIO import StringIO from subprocess import Popen, PIPE @@ -17,8 +18,12 @@ log = logging.getLogger( __name__ ) -DEFAULT_MODULE_COMMAND = 'module' -DEFAULT_MODULE_DIRECTORY = '/usr/share/modules/modulefiles' +if environ.has_key('MODULEPATH'): + DEFAULT_MODULE_PATH = environ['MODULEPATH'] +elif environ.has_key('MODULESHOME'): + DEFAULT_MODULE_PATH = join(environ['MODULESHOME'], 'modulefiles') +else: + DEFAULT_MODULE_PATH = '/usr/share/modules/modulefiles' DEFAULT_INDICATOR = '(default)' DEFAULT_MODULE_PREFETCH = "true" UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find modules by [%s], find_by should be one of %s" @@ -28,13 +33,12 @@ resolver_type = "modules" def __init__(self, dependency_manager, **kwds): - self.module_command = kwds.get('command', DEFAULT_MODULE_COMMAND) self.versionless = string_as_bool(kwds.get('versionless', 'false')) find_by = kwds.get('find_by', 'avail') prefetch = string_as_bool(kwds.get('prefetch', DEFAULT_MODULE_PREFETCH)) if find_by == 'directory': - directory = kwds.get('directory', DEFAULT_MODULE_DIRECTORY) - self.module_checker = DirectoryModuleChecker(self, directory, prefetch) + modulepath = kwds.get('modulepath', DEFAULT_MODULE_PATH) + self.module_checker = DirectoryModuleChecker(self, modulepath, prefetch) elif find_by == 'avail': self.module_checker = AvailModuleChecker(self, prefetch) else: @@ -44,11 +48,10 @@ if type != "package": return INDETERMINATE_DEPENDENCY - if self.versionless: - version = None - if self.__has_module(name, version): return ModuleDependency(self, name, version) + elif self.versionless and self.__has_module(name, None): + return ModuleDependency(self, name, None) return INDETERMINATE_DEPENDENCY @@ -58,23 +61,26 @@ class DirectoryModuleChecker(object): - def __init__(self, module_dependency_resolver, directory, prefetch): + def __init__(self, module_dependency_resolver, modulepath, prefetch): self.module_dependency_resolver = module_dependency_resolver - self.directory = directory + self.directories = modulepath.split(':') if prefetch: log.warn("Created module dependency resolver with prefetch enabled, but directory module checker does not support this.") pass def has_module(self, module, version): - module_directory = join(self.directory, module) - has_module_directory = isdir( join( self.directory, module ) ) - if not version: - has_module = has_module_directory - else: - modulefile = join( module_directory, version ) - has_modulefile = exists( modulefile ) - has_module = has_module_directory and has_modulefile - return has_module + for directory in self.directories: + module_directory = join(directory, module) + has_module_directory = isdir( module_directory ) + if not version: + has_module = has_module_directory or exists(module_directory) # could be a bare modulefile + else: + modulefile = join( module_directory, version ) + has_modulefile = exists( modulefile ) + has_module = has_module_directory and has_modulefile + if has_module: + return True + return False class AvailModuleChecker(object): @@ -102,7 +108,7 @@ return False def __modules(self): - raw_output = self.__module_avail_ouptut() + raw_output = self.__module_avail_output() for line in StringIO(raw_output): line = line and line.strip() if not line or line.startswith("-"): @@ -119,10 +125,9 @@ module_name = module_parts[0] yield module_name, module_version - def __module_avail_ouptut(self): - avail_command = '%s avail' % self.module_dependency_resolver.module_command - return Popen([avail_command], shell=True, stderr=PIPE).communicate()[1] - + def __module_avail_output(self): + avail_command = ['modulecmd', 'sh', 'avail'] + return Popen(avail_command, stderr=PIPE).communicate()[1] class ModuleDependency(Dependency): @@ -132,9 +137,10 @@ self.module_version = module_version def shell_commands(self, requirement): - command = '%s load %s' % (self.module_dependency_resolver.module_command, self.module_name) + module_to_load = self.module_name if self.module_version: - command = '%s/%s' % self.module_version + module_to_load = '%s/%s' % (self.module_name, self.module_version) + command = 'eval `modulecmd sh load %s`' % (module_to_load) return command __all__ = [ModuleDependencyResolver] https://bitbucket.org/galaxy/galaxy-central/commits/93286e74e18b/ Changeset: 93286e74e18b User: jmchilton Date: 2013-10-17 06:15:21 Summary: Small touch ups to module code enhancements. Affected #: 1 file diff -r dc9c0e065be01f8d16b35eec7b53098fdfa0fd05 -r 93286e74e18b7dcb84aa11a326f08d5b9398a90d lib/galaxy/tools/deps/resolvers/modules.py --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -6,7 +6,7 @@ it, hence support for it will be minimal. The Galaxy team eagerly welcomes community contribution and maintenance however. """ -from os import environ +from os import environ, pathsep from os.path import exists, isdir, join from StringIO import StringIO from subprocess import Popen, PIPE @@ -18,12 +18,7 @@ log = logging.getLogger( __name__ ) -if environ.has_key('MODULEPATH'): - DEFAULT_MODULE_PATH = environ['MODULEPATH'] -elif environ.has_key('MODULESHOME'): - DEFAULT_MODULE_PATH = join(environ['MODULESHOME'], 'modulefiles') -else: - DEFAULT_MODULE_PATH = '/usr/share/modules/modulefiles' +DEFAULT_MODULE_PATH = '/usr/share/modules/modulefiles' DEFAULT_INDICATOR = '(default)' DEFAULT_MODULE_PREFETCH = "true" UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find modules by [%s], find_by should be one of %s" @@ -44,6 +39,15 @@ else: raise Exception(UNKNOWN_FIND_BY_MESSAGE % (find_by, ["avail", "directory"])) + def __default_modulespath(self): + if 'MODULEPATH' in environ: + module_path = environ['MODULEPATH'] + elif 'MODULESHOME' in environ: + module_path = join(environ['MODULESHOME'], 'modulefiles') + else: + module_path = DEFAULT_MODULE_PATH + return module_path + def resolve( self, name, version, type, **kwds ): if type != "package": return INDETERMINATE_DEPENDENCY @@ -63,24 +67,25 @@ def __init__(self, module_dependency_resolver, modulepath, prefetch): self.module_dependency_resolver = module_dependency_resolver - self.directories = modulepath.split(':') + self.directories = modulepath.split(pathsep) if prefetch: log.warn("Created module dependency resolver with prefetch enabled, but directory module checker does not support this.") pass def has_module(self, module, version): + has_module = False for directory in self.directories: module_directory = join(directory, module) has_module_directory = isdir( module_directory ) if not version: - has_module = has_module_directory or exists(module_directory) # could be a bare modulefile + has_module = has_module_directory or exists(module_directory) # could be a bare modulefile else: modulefile = join( module_directory, version ) has_modulefile = exists( modulefile ) has_module = has_module_directory and has_modulefile if has_module: - return True - return False + break + return has_module class AvailModuleChecker(object): @@ -129,6 +134,7 @@ avail_command = ['modulecmd', 'sh', 'avail'] return Popen(avail_command, stderr=PIPE).communicate()[1] + class ModuleDependency(Dependency): def __init__(self, module_dependency_resolver, module_name, module_version=None): https://bitbucket.org/galaxy/galaxy-central/commits/66222d30c816/ Changeset: 66222d30c816 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Add unit tests for module behavior contributed by Simon Guest. Affected #: 2 files diff -r 93286e74e18b7dcb84aa11a326f08d5b9398a90d -r 66222d30c81606b57d09e195365707c9f61b2946 lib/galaxy/tools/deps/resolvers/modules.py --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -17,7 +17,7 @@ import logging log = logging.getLogger( __name__ ) - +DEFAULT_MODULECMD_PATH = "modulecmd" # Just check path DEFAULT_MODULE_PATH = '/usr/share/modules/modulefiles' DEFAULT_INDICATOR = '(default)' DEFAULT_MODULE_PREFETCH = "true" @@ -31,8 +31,9 @@ self.versionless = string_as_bool(kwds.get('versionless', 'false')) find_by = kwds.get('find_by', 'avail') prefetch = string_as_bool(kwds.get('prefetch', DEFAULT_MODULE_PREFETCH)) + self.modulecmd = kwds.get('modulecmd', DEFAULT_MODULECMD_PATH) if find_by == 'directory': - modulepath = kwds.get('modulepath', DEFAULT_MODULE_PATH) + modulepath = kwds.get('modulepath', self.__default_modulespath()) self.module_checker = DirectoryModuleChecker(self, modulepath, prefetch) elif find_by == 'avail': self.module_checker = AvailModuleChecker(self, prefetch) @@ -131,7 +132,7 @@ yield module_name, module_version def __module_avail_output(self): - avail_command = ['modulecmd', 'sh', 'avail'] + avail_command = [self.module_dependency_resolver.modulecmd, 'sh', 'avail'] return Popen(avail_command, stderr=PIPE).communicate()[1] @@ -146,7 +147,7 @@ module_to_load = self.module_name if self.module_version: module_to_load = '%s/%s' % (self.module_name, self.module_version) - command = 'eval `modulecmd sh load %s`' % (module_to_load) + command = 'eval `%s sh load %s`' % (self.module_dependency_resolver.modulecmd, module_to_load) return command __all__ = [ModuleDependencyResolver] diff -r 93286e74e18b7dcb84aa11a326f08d5b9398a90d -r 66222d30c81606b57d09e195365707c9f61b2946 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -1,11 +1,11 @@ import tempfile import os.path from stat import S_IXUSR -from os import makedirs, symlink, stat, chmod +from os import makedirs, stat, symlink, chmod, environ from shutil import rmtree from galaxy.tools.deps import DependencyManager, INDETERMINATE_DEPENDENCY from galaxy.tools.deps.resolvers.galaxy_packages import GalaxyPackageDependency -from galaxy.tools.deps.resolvers.modules import ModuleDependencyResolver +from galaxy.tools.deps.resolvers.modules import ModuleDependencyResolver, ModuleDependency from galaxy.util.bunch import Bunch from contextlib import contextmanager from subprocess import Popen, PIPE @@ -92,7 +92,7 @@ gx_env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION) ts_env_path = os.path.join(ts_package_dir, "env.sh") dependency = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) - assert dependency.script != gx_env_path # Not the galaxy path, it should be the tool shed path used. + assert dependency.script != gx_env_path # Not the galaxy path, it should be the tool shed path used. assert dependency.script == ts_env_path @@ -104,9 +104,8 @@ def test_module_dependency_resolver(): with __test_base_path() as temp_directory: - module_script = os.path.join(temp_directory, "module") - with open(module_script, 'w') as f: - f.write('''#!/bin/sh + module_script = os.path.join(temp_directory, "modulecmd") + __write_script(module_script, '''#!/bin/sh cat %s/example_output 1>&2; ''' % temp_directory) with open(os.path.join(temp_directory, "example_output"), "w") as f: @@ -131,9 +130,7 @@ advisor/2013/update2 intel/11.1.080 mkl/10.2.5.035 advisor/2013/update3 intel/12.0 mkl/10.2.7.041 ''') - st = os.stat(module_script) - chmod(module_script, st.st_mode | S_IXUSR) - resolver = ModuleDependencyResolver(None, command=module_script) + resolver = ModuleDependencyResolver(None, modulecmd=module_script) module = resolver.resolve( name="R", version=None, type="package" ) assert module.module_name == "R" assert module.module_version == None @@ -146,6 +143,31 @@ assert module == INDETERMINATE_DEPENDENCY +def test_module_dependency(): + with __test_base_path() as temp_directory: + ## Create mock modulecmd script that just exports a variable + ## the way modulecmd sh load would, but also validate correct + ## module name and version are coming through. + mock_modulecmd = os.path.join(temp_directory, 'modulecmd') + __write_script(mock_modulecmd, '''#!/bin/sh +if [ $3 != "foomodule/1.0" ]; +then + exit 1 +fi +echo 'FOO="bar"' +''') + resolver = Bunch(modulecmd=mock_modulecmd) + dependency = ModuleDependency(resolver, "foomodule", "1.0") + __assert_foo_exported( dependency.shell_commands( Bunch( type="package" ) ) ) + + +def __write_script(path, contents): + with open(path, 'w') as f: + f.write(contents) + st = stat(path) + chmod(path, st.st_mode | S_IXUSR) + + def test_galaxy_dependency_object_script(): with __test_base_path() as base_path: ## Create env.sh file that just exports variable FOO and verify it @@ -170,7 +192,7 @@ command = ["bash", "-c", "%s; echo \"$FOO\"" % "".join(commands)] process = Popen(command, stdout=PIPE) output = process.communicate()[0].strip() - assert output == 'bar' + assert output == 'bar', "Command %s exports FOO as %s, not bar" % (command, output) def __setup_galaxy_package_dep(base_path, name, version, contents=""): @@ -261,17 +283,68 @@ </dependency_resolvers> ''') as dependency_resolvers: module_resolver = dependency_resolvers[0] - assert module_resolver.module_command == "module" assert module_resolver.module_checker.__class__.__name__ == "AvailModuleChecker" +def test_config_modulepath(): + # Test reads and splits MODULEPATH if modulepath is not specified. + with __parse_resolvers('''<dependency_resolvers> + <modules find_by="directory" modulepath="/opt/modules/modulefiles:/usr/local/modules/modulefiles" /> +</dependency_resolvers> +''') as dependency_resolvers: + assert dependency_resolvers[0].module_checker.directories == ["/opt/modules/modulefiles", "/usr/local/modules/modulefiles"] + + +def test_config_MODULEPATH(): + # Test reads and splits MODULEPATH if modulepath is not specified. + with __environ({"MODULEPATH": "/opt/modules/modulefiles:/usr/local/modules/modulefiles"}): + with __parse_resolvers('''<dependency_resolvers> + <modules find_by="directory" /> +</dependency_resolvers> +''') as dependency_resolvers: + assert dependency_resolvers[0].module_checker.directories == ["/opt/modules/modulefiles", "/usr/local/modules/modulefiles"] + + +def test_config_MODULESHOME(): + # Test fallbacks to read MODULESHOME if modulepath is not specified and + # neither is MODULEPATH. + with __environ({"MODULESHOME": "/opt/modules"}, remove="MODULEPATH"): + with __parse_resolvers('''<dependency_resolvers> + <modules find_by="directory" /> +</dependency_resolvers> +''') as dependency_resolvers: + assert dependency_resolvers[0].module_checker.directories == ["/opt/modules/modulefiles"] + + def test_config_module_directory_searcher(): with __parse_resolvers('''<dependency_resolvers> - <modules find_by="directory" directory="/opt/Modules/modulefiles" /> + <modules find_by="directory" modulepath="/opt/Modules/modulefiles" /></dependency_resolvers> ''') as dependency_resolvers: module_resolver = dependency_resolvers[0] - assert module_resolver.module_checker.directory == "/opt/Modules/modulefiles" + assert module_resolver.module_checker.directories == ["/opt/Modules/modulefiles"] + + +@contextmanager +def __environ(values, remove=[]): + """ + Modify the environment for a test, adding/updating values in dict `values` and + removing any environment variables mentioned in list `remove`. + """ + new_keys = set(environ.keys()) - set(values.keys()) + old_environ = environ.copy() + try: + environ.update(values) + for to_remove in remove: + try: + del environ[remove] + except KeyError: + pass + yield + finally: + environ.update(old_environ) + for key in new_keys: + del environ[key] @contextmanager @@ -282,4 +355,3 @@ f.flush() dm = DependencyManager( default_base_path=base_path, conf_file=f.name ) yield dm.dependency_resolvers - https://bitbucket.org/galaxy/galaxy-central/commits/a96f6548bfc3/ Changeset: a96f6548bfc3 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Add method to DependencyManager determining if tool shed dependencies used. Affected #: 2 files diff -r 66222d30c81606b57d09e195365707c9f61b2946 -r a96f6548bfc3f223f93a8b619e0ad2d4634a3f3f lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -56,6 +56,9 @@ commands.append( dependency_commands ) return commands + def uses_tool_shed_dependencies(self): + return any( map( lambda r: isinstance( r, ToolShedPackageDependencyResolver ), self.dependency_resolvers ) ) + def find_dep( self, name, version=None, type='package', **kwds ): for resolver in self.dependency_resolvers: dependency = resolver.resolve( name, version, type, **kwds ) diff -r 66222d30c81606b57d09e195365707c9f61b2946 -r a96f6548bfc3f223f93a8b619e0ad2d4634a3f3f test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -277,6 +277,20 @@ assert dependency_resolvers[0].base_path != dependency_resolvers[2].base_path +def test_uses_tool_shed_dependencies(): + with __dependency_manager('''<dependency_resolvers> + <galaxy_packages /> +</dependency_resolvers> +''') as dm: + assert not dm.uses_tool_shed_dependencies() + + with __dependency_manager('''<dependency_resolvers> + <tool_shed_packages /> +</dependency_resolvers> +''') as dm: + assert dm.uses_tool_shed_dependencies() + + def test_config_module_defaults(): with __parse_resolvers('''<dependency_resolvers><modules /> @@ -349,9 +363,15 @@ @contextmanager def __parse_resolvers(xml_content): + with __dependency_manager(xml_content) as dm: + yield dm.dependency_resolvers + + +@contextmanager +def __dependency_manager(xml_content): with __test_base_path() as base_path: f = tempfile.NamedTemporaryFile() f.write(xml_content) f.flush() dm = DependencyManager( default_base_path=base_path, conf_file=f.name ) - yield dm.dependency_resolvers + yield dm https://bitbucket.org/galaxy/galaxy-central/commits/8e001dc9675c/ Changeset: 8e001dc9675c User: jmchilton Date: 2013-10-17 06:15:21 Summary: Dependencies: Do not attempt to install package dependencies in tool shed if the will be ignored. That is if dependency resolvers are configured to ignore tool shed dependencies as outlined by Simon Guest, though stopping short of marking these dependencies as installed - they will still be marked as errors for now. TODO: Determine what to do with dependencies that can be resolved (just mark as installed? a brand new state? mark as never installed?). Affected #: 1 file diff -r a96f6548bfc3f223f93a8b619e0ad2d4634a3f3f -r 8e001dc9675c105d83d83f6d5a756c11759e8bb2 lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -6,6 +6,7 @@ from galaxy import util from galaxy import web from galaxy.util import json +from galaxy.tools.deps.resolvers import INDETERMINATE_DEPENDENCY import tool_shed.util.shed_util_common as suc from tool_shed.util import common_util from tool_shed.util import container_util @@ -457,7 +458,19 @@ tool_dependency = tool_dependencies[ index ] if tool_dependency.can_install: try: - tool_dependency = install_package( app, elem, tool_shed_repository, tool_dependencies=tool_dependencies ) + dependencies_ignored = app.toolbox.dependency_manager and not app.toolbox.dependency_manager.uses_tool_shed_dependencies() + if dependencies_ignored: + log.info("Skipping package %s, tool shed dependency resolver not enabled." % package_name) + # Tool dependency resolves have been configured and they do not + # include the tool shed. Do not install package. + status = app.model.ToolDependency.installation_status.ERROR + if app.toolbox.dependency_manager.find_dep( package_name, package_version, type='package') != INDETERMINATE_DEPENDENCY: + ## TODO: Do something here such as marking it installed or + ## configured externally. + pass + tool_dependency.status = status + else: + tool_dependency = install_package( app, elem, tool_shed_repository, tool_dependencies=tool_dependencies ) except Exception, e: error_message = "Error installing tool dependency %s version %s: %s" % ( str( package_name ), str( package_version ), str( e ) ) log.exception( error_message ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.