- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: jmchilton: Bug fix related to return code handling in TaskWrapper.
by commits-noreply＠bitbucket.org 17 Oct '13

17 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/329ea7a83af4/ Changeset: 329ea7a83af4 User: jmchilton Date: 2013-10-17 12:30:13 Summary: Bug fix related to return code handling in TaskWrapper. Thanks to Peter Cock for finding this. Affected #: 2 files diff -r 8e001dc9675c105d83d83f6d5a756c11759e8bb2 -r 329ea7a83af4f389a7c95ee4559d88c6fec0211b lib/galaxy/jobs/command_factory.py --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -55,18 +55,20 @@ # Append metadata setting commands, we don't want to overwrite metadata # that was copied over in init_meta(), as per established behavior if include_metadata and job_wrapper.requires_setting_metadata: - if not captured_return_code: - commands += capture_return_code_command - captured_return_code = True - commands += "; cd %s; " % abspath( getcwd() ) - commands += job_wrapper.setup_external_metadata( + metadata_command = job_wrapper.setup_external_metadata( exec_dir=abspath( getcwd() ), tmp_dir=job_wrapper.working_directory, dataset_files_path=job.app.model.Dataset.file_path, output_fnames=job_wrapper.get_output_fnames(), set_extension=False, kwds={ 'overwrite' : False } - ) + ) or '' + metadata_command = metadata_command.strip() + if metadata_command: + if not captured_return_code: + commands += capture_return_code_command + captured_return_code = True + commands += "; cd %s; %s" % (abspath( getcwd() ), metadata_command) if captured_return_code: commands += '; sh -c "exit $return_code"' diff -r 8e001dc9675c105d83d83f6d5a756c11759e8bb2 -r 329ea7a83af4f389a7c95ee4559d88c6fec0211b test/unit/test_command_factory.py --- a/test/unit/test_command_factory.py +++ b/test/unit/test_command_factory.py @@ -45,6 +45,18 @@ expected_command = '%s; return_code=$?; cd %s; %s; sh -c "exit $return_code"' % (MOCK_COMMAND_LINE, getcwd(), metadata_line) self.__assert_command_is( expected_command ) + def test_empty_metadata(self): + """ + As produced by TaskWrapper. + """ + self.include_metadata = True + self.include_work_dir_outputs = False + self.job_wrapper.metadata_line = ' ' + # Empty metadata command do not touch command line. + expected_command = '%s' % (MOCK_COMMAND_LINE) + self.__assert_command_is( expected_command ) + + def __assert_command_is(self, expected_command): command = self.__command() self.assertEqual(command, expected_command) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 24 new changesets
by commits-noreply＠bitbucket.org 17 Oct '13

17 Oct '13

24 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/be16cffdcb68/ Changeset: be16cffdcb68 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Convert James Taylor's dependency testing file lib/galaxy/tools/deps/tests.py into a formal unit test. Affected #: 2 files diff -r ebd092733e552ef6a846d55b69d22dc779dfcc5f -r be16cffdcb68709e3368bcd599bfef9f8aab72a4 lib/galaxy/tools/deps/tests.py --- a/lib/galaxy/tools/deps/tests.py +++ /dev/null @@ -1,32 +0,0 @@ -import tempfile -import os.path -from os import makedirs, mkdir -import galaxy.tools.deps - -def touch( fname, data=None ): - f = open( fname, 'w' ) - if data: - f.write( data ) - f.close() - -def test(): - - # Setup directories - base_path = tempfile.mkdtemp() - # mkdir( base_path ) - for name, version, sub in [ ( "dep1", "1.0", "env.sh" ), ( "dep1", "2.0", "bin" ), ( "dep2", "1.0", None ) ]: - if sub == "bin": - p = os.path.join( base_path, name, version, "bin" ) - else: - p = os.path.join( base_path, name, version ) - try: - makedirs( p ) - except: - pass - if sub == "env.sh": - touch( os.path.join( p, "env.sh" ) ) - - dm = galaxy.tools.deps.DependencyManager( [ base_path ] ) - - print dm.find_dep( "dep1", "1.0" ) - print dm.find_dep( "dep1", "2.0" ) diff -r ebd092733e552ef6a846d55b69d22dc779dfcc5f -r be16cffdcb68709e3368bcd599bfef9f8aab72a4 test/unit/test_tool_deps.py --- /dev/null +++ b/test/unit/test_tool_deps.py @@ -0,0 +1,40 @@ +import tempfile +import os.path +from os import makedirs +import galaxy.tools.deps + + +def touch( fname, data=None ): + f = open( fname, 'w' ) + if data: + f.write( data ) + f.close() + + +def test_tool_dependencies(): + + # Setup directories + base_path = tempfile.mkdtemp() + # mkdir( base_path ) + for name, version, sub in [ ( "dep1", "1.0", "env.sh" ), ( "dep1", "2.0", "bin" ), ( "dep2", "1.0", None ) ]: + if sub == "bin": + p = os.path.join( base_path, name, version, "bin" ) + else: + p = os.path.join( base_path, name, version ) + try: + makedirs( p ) + except: + pass + if sub == "env.sh": + touch( os.path.join( p, "env.sh" ) ) + + dm = galaxy.tools.deps.DependencyManager( [ base_path ] ) + + d1_script, d1_path, d1_version = dm.find_dep( "dep1", "1.0" ) + assert d1_script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) + assert d1_path == os.path.join( base_path, 'dep1', '1.0' ) + assert d1_version == "1.0" + d2_script, d2_path, d2_version = dm.find_dep( "dep1", "2.0" ) + assert d2_script == None + assert d2_path == os.path.join( base_path, 'dep1', '2.0' ) + assert d2_version == "2.0" https://bitbucket.org/galaxy/galaxy-central/commits/e276f3fc9a74/ Changeset: e276f3fc9a74 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Add tests for addditional existing behavior for tool dependencies. Namely test the fallen: - Test default package will not be fallen back upon when exactly specified version is used. (Not my favorite behavior. -John) - Test tool shed package and set_environment requirement types (standard use). - Test tool shed installed tools will fallback on traditional Galaxy dependencies if a requirement is unmatched in installed dependencies. - Test tool shed dependency takes prescndence over traditional Galaxy dependency when both are available. Other small clean ups of test/unit/test_tool_deps.py. Affected #: 1 file diff -r be16cffdcb68709e3368bcd599bfef9f8aab72a4 -r e276f3fc9a7426923d968e7e56da1e3b26292056 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -1,40 +1,140 @@ import tempfile import os.path -from os import makedirs -import galaxy.tools.deps - - -def touch( fname, data=None ): - f = open( fname, 'w' ) - if data: - f.write( data ) - f.close() +from os import makedirs, symlink +from shutil import rmtree +from galaxy.tools.deps import DependencyManager +from galaxy.util.bunch import Bunch +from contextlib import contextmanager def test_tool_dependencies(): + # Setup directories + with __test_base_path() as base_path: + for name, version, sub in [ ( "dep1", "1.0", "env.sh" ), ( "dep1", "2.0", "bin" ), ( "dep2", "1.0", None ) ]: + if sub == "bin": + p = os.path.join( base_path, name, version, "bin" ) + else: + p = os.path.join( base_path, name, version ) + try: + makedirs( p ) + except: + pass + if sub == "env.sh": + __touch( os.path.join( p, "env.sh" ) ) - # Setup directories + dm = DependencyManager( [base_path] ) + d1_script, d1_path, d1_version = dm.find_dep( "dep1", "1.0" ) + assert d1_script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) + assert d1_path == os.path.join( base_path, 'dep1', '1.0' ) + assert d1_version == "1.0" + d2_script, d2_path, d2_version = dm.find_dep( "dep1", "2.0" ) + assert d2_script == None + assert d2_path == os.path.join( base_path, 'dep1', '2.0' ) + assert d2_version == "2.0" + + ## Test default versions + symlink( os.path.join( base_path, 'dep1', '2.0'), os.path.join( base_path, 'dep1', 'default' ) ) + default_script, default_path, default_version = dm.find_dep( "dep1", None ) + assert default_version == "2.0" + + ## Test default will not be fallen back upon by default + default_script, default_path, default_version = dm.find_dep( "dep1", "2.1" ) + assert default_script == None + assert default_version == None + + +TEST_REPO_USER = "devteam" +TEST_REPO_NAME = "bwa" +TEST_REPO_CHANGESET = "12abcd41223da" +TEST_VERSION = "0.5.9" + + +def test_toolshed_set_enviornment_requiremetns(): + with __test_base_path() as base_path: + test_repo = __build_test_repo('set_environment') + dm = DependencyManager( [base_path] ) + env_settings_dir = os.path.join(base_path, "environment_settings", TEST_REPO_NAME, TEST_REPO_USER, TEST_REPO_NAME, TEST_REPO_CHANGESET) + os.makedirs(env_settings_dir) + d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=None, type='set_environment', installed_tool_dependencies=[test_repo] ) + assert d1_version == None + assert d1_script == os.path.join(env_settings_dir, "env.sh"), d1_script + + +def test_toolshed_package_requirements(): + with __test_base_path() as base_path: + test_repo = __build_test_repo('package', version=TEST_VERSION) + dm = DependencyManager( [base_path] ) + package_dir = __build_ts_test_package(base_path) + d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) + assert d1_version == TEST_VERSION, d1_version + assert d1_script == os.path.join(package_dir, "env.sh"), d1_script + + +def test_toolshed_tools_fallback_on_manual_dependencies(): + with __test_base_path() as base_path: + dm = DependencyManager( [base_path] ) + test_repo = __build_test_repo('package', version=TEST_VERSION) + env_path = __setup_galaxy_package_dep(base_path, "dep1", "1.0") + d1_script, d1_path, d1_version = dm.find_dep( "dep1", version="1.0", type='package', installed_tool_dependencies=[test_repo] ) + assert d1_version == "1.0" + assert d1_script == env_path + + +def test_toolshed_greater_precendence(): + with __test_base_path() as base_path: + dm = DependencyManager( [base_path] ) + test_repo = __build_test_repo('package', version=TEST_VERSION) + ts_package_dir = __build_ts_test_package(base_path) + gx_env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION) + ts_env_path = os.path.join(ts_package_dir, "env.sh") + d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) + assert d1_script != gx_env_path # Not the galaxy path, it should be the tool shed path used. + assert d1_script == ts_env_path + + +def __build_ts_test_package(base_path, script_contents=''): + package_dir = os.path.join(base_path, TEST_REPO_NAME, TEST_VERSION, TEST_REPO_USER, TEST_REPO_NAME, TEST_REPO_CHANGESET) + __touch(os.path.join(package_dir, 'env.sh'), script_contents) + return package_dir + + +def __setup_galaxy_package_dep(base_path, name, version, contents=""): + dep_directory = os.path.join( base_path, name, version ) + env_path = os.path.join( dep_directory, "env.sh" ) + __touch( env_path, contents ) + return env_path + + +def __touch( fname, data=None ): + dirname = os.path.dirname( fname ) + if not os.path.exists( dirname ): + makedirs( dirname ) + f = open( fname, 'w' ) + try: + if data: + f.write( data ) + finally: + f.close() + + +def __build_test_repo(type, version=None): + return Bunch( + owner=TEST_REPO_USER, + name=TEST_REPO_NAME, + type=type, + version=version, + tool_shed_repository=Bunch( + owner=TEST_REPO_USER, + name=TEST_REPO_NAME, + installed_changeset_revision=TEST_REPO_CHANGESET + ) + ) + + +@contextmanager +def __test_base_path(): base_path = tempfile.mkdtemp() - # mkdir( base_path ) - for name, version, sub in [ ( "dep1", "1.0", "env.sh" ), ( "dep1", "2.0", "bin" ), ( "dep2", "1.0", None ) ]: - if sub == "bin": - p = os.path.join( base_path, name, version, "bin" ) - else: - p = os.path.join( base_path, name, version ) - try: - makedirs( p ) - except: - pass - if sub == "env.sh": - touch( os.path.join( p, "env.sh" ) ) - - dm = galaxy.tools.deps.DependencyManager( [ base_path ] ) - - d1_script, d1_path, d1_version = dm.find_dep( "dep1", "1.0" ) - assert d1_script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) - assert d1_path == os.path.join( base_path, 'dep1', '1.0' ) - assert d1_version == "1.0" - d2_script, d2_path, d2_version = dm.find_dep( "dep1", "2.0" ) - assert d2_script == None - assert d2_path == os.path.join( base_path, 'dep1', '2.0' ) - assert d2_version == "2.0" + try: + yield base_path + finally: + rmtree(base_path) https://bitbucket.org/galaxy/galaxy-central/commits/b16ab0d407a0/ Changeset: b16ab0d407a0 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: PEP8 fixes for lib/galaxy/tools/deps/__init__.py Affected #: 1 file diff -r e276f3fc9a7426923d968e7e56da1e3b26292056 -r b16ab0d407a09695aaa0dcbffcd9661b12c4b701 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -7,6 +7,7 @@ import logging log = logging.getLogger( __name__ ) + class DependencyManager( object ): """ A DependencyManager attempts to resolve named and versioned dependencies by searching for them under a list of directories. Directories should be @@ -28,6 +29,7 @@ if not os.path.isdir( base_path ): log.warn( "Path '%s' is not directory, ignoring", base_path ) self.base_paths.append( os.path.abspath( base_path ) ) + def find_dep( self, name, version=None, type='package', installed_tool_dependencies=None ): """ Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a @@ -37,6 +39,7 @@ return self._find_dep_default( name, type=type, installed_tool_dependencies=installed_tool_dependencies ) else: return self._find_dep_versioned( name, version, type=type, installed_tool_dependencies=installed_tool_dependencies ) + def _find_dep_versioned( self, name, version, type='package', installed_tool_dependencies=None ): installed_tool_dependency = self._get_installed_dependency( installed_tool_dependencies, name, type, version=version ) for base_path in self.base_paths: @@ -51,6 +54,7 @@ return None, path, version else: return None, None, None + def _find_dep_default( self, name, type='package', installed_tool_dependencies=None ): if type == 'set_environment' and installed_tool_dependencies: installed_tool_dependency = self._get_installed_dependency( installed_tool_dependencies, name, type, version=None ) @@ -68,20 +72,22 @@ script = os.path.join( real_path, 'env.sh' ) if os.path.exists( script ): return script, real_path, real_version - elif os.path.exists( os.path.join( real_path, 'bin' ) ): + elif os.path.exists( real_bin ): return None, real_path, real_version else: return None, None, None + def _get_installed_dependency( self, installed_tool_dependencies, name, type, version=None ): if installed_tool_dependencies: for installed_tool_dependency in installed_tool_dependencies: if version: - if installed_tool_dependency.name==name and installed_tool_dependency.type==type and installed_tool_dependency.version==version: + if installed_tool_dependency.name == name and installed_tool_dependency.type == type and installed_tool_dependency.version == version: return installed_tool_dependency else: - if installed_tool_dependency.name==name and installed_tool_dependency.type==type: + if installed_tool_dependency.name == name and installed_tool_dependency.type == type: return installed_tool_dependency return None + def _get_package_installed_dependency_path( self, installed_tool_dependency, base_path, name, version ): tool_shed_repository = installed_tool_dependency.tool_shed_repository return os.path.join( base_path, @@ -90,6 +96,7 @@ tool_shed_repository.owner, tool_shed_repository.name, tool_shed_repository.installed_changeset_revision ) + def _get_set_environment_installed_dependency_script_path( self, installed_tool_dependency, name ): tool_shed_repository = installed_tool_dependency.tool_shed_repository for base_path in self.base_paths: https://bitbucket.org/galaxy/galaxy-central/commits/630cfe0030ab/ Changeset: 630cfe0030ab User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Drop multiple tool_dependency_dir implementation for now. Will reimplement once I have added more granular, plugin style tool dependency resolvers. They will benefit from there being just one default_base_path as implemented here, but allowing it to be overridden on a per resolver basis. Affected #: 3 files diff -r b16ab0d407a09695aaa0dcbffcd9661b12c4b701 -r 630cfe0030abf40e46c9839254278531d657e554 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -696,7 +696,7 @@ def init_dependency_manager( self ): if self.app.config.use_tool_dependencies: - self.dependency_manager = DependencyManager( [ self.app.config.tool_dependency_dir ] ) + self.dependency_manager = DependencyManager( self.app.config.tool_dependency_dir ) else: self.dependency_manager = None diff -r b16ab0d407a09695aaa0dcbffcd9661b12c4b701 -r 630cfe0030abf40e46c9839254278531d657e554 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -17,18 +17,16 @@ and should each contain a file 'env.sh' which can be sourced to make the dependency available in the current shell environment. """ - def __init__( self, base_paths=[] ): + def __init__( self, default_base_path ): """ Create a new dependency manager looking for packages under the paths listed in `base_paths`. The default base path is app.config.tool_dependency_dir. """ - self.base_paths = [] - for base_path in base_paths: - if not os.path.exists( base_path ): - log.warn( "Path '%s' does not exist, ignoring", base_path ) - if not os.path.isdir( base_path ): - log.warn( "Path '%s' is not directory, ignoring", base_path ) - self.base_paths.append( os.path.abspath( base_path ) ) + if not os.path.exists( default_base_path ): + log.warn( "Path '%s' does not exist, ignoring", default_base_path ) + if not os.path.isdir( default_base_path ): + log.warn( "Path '%s' is not directory, ignoring", default_base_path ) + self.default_base_path = os.path.abspath( default_base_path ) def find_dep( self, name, version=None, type='package', installed_tool_dependencies=None ): """ @@ -42,18 +40,17 @@ def _find_dep_versioned( self, name, version, type='package', installed_tool_dependencies=None ): installed_tool_dependency = self._get_installed_dependency( installed_tool_dependencies, name, type, version=version ) - for base_path in self.base_paths: - if installed_tool_dependency: - path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) - else: - path = os.path.join( base_path, name, version ) - script = os.path.join( path, 'env.sh' ) - if os.path.exists( script ): - return script, path, version - elif os.path.exists( os.path.join( path, 'bin' ) ): - return None, path, version + base_path = self.default_base_path + if installed_tool_dependency: + path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) else: - return None, None, None + path = os.path.join( base_path, name, version ) + script = os.path.join( path, 'env.sh' ) + if os.path.exists( script ): + return script, path, version + elif os.path.exists( os.path.join( path, 'bin' ) ): + return None, path, version + return None, None, None def _find_dep_default( self, name, type='package', installed_tool_dependencies=None ): if type == 'set_environment' and installed_tool_dependencies: @@ -63,19 +60,18 @@ if script and path: # Environment settings do not use versions. return script, path, None - for base_path in self.base_paths: - path = os.path.join( base_path, name, 'default' ) - if os.path.islink( path ): - real_path = os.path.realpath( path ) - real_bin = os.path.join( real_path, 'bin' ) - real_version = os.path.basename( real_path ) - script = os.path.join( real_path, 'env.sh' ) - if os.path.exists( script ): - return script, real_path, real_version - elif os.path.exists( real_bin ): - return None, real_path, real_version - else: - return None, None, None + base_path = self.default_base_path + path = os.path.join( base_path, name, 'default' ) + if os.path.islink( path ): + real_path = os.path.realpath( path ) + real_bin = os.path.join( real_path, 'bin' ) + real_version = os.path.basename( real_path ) + script = os.path.join( real_path, 'env.sh' ) + if os.path.exists( script ): + return script, real_path, real_version + elif os.path.exists( real_bin ): + return None, real_path, real_version + return None, None, None def _get_installed_dependency( self, installed_tool_dependencies, name, type, version=None ): if installed_tool_dependencies: @@ -99,14 +95,14 @@ def _get_set_environment_installed_dependency_script_path( self, installed_tool_dependency, name ): tool_shed_repository = installed_tool_dependency.tool_shed_repository - for base_path in self.base_paths: - path = os.path.abspath( os.path.join( base_path, - 'environment_settings', - name, - tool_shed_repository.owner, - tool_shed_repository.name, - tool_shed_repository.installed_changeset_revision ) ) - if os.path.exists( path ): - script = os.path.join( path, 'env.sh' ) - return script, path, None + base_path = self.default_base_path + path = os.path.abspath( os.path.join( base_path, + 'environment_settings', + name, + tool_shed_repository.owner, + tool_shed_repository.name, + tool_shed_repository.installed_changeset_revision ) ) + if os.path.exists( path ): + script = os.path.join( path, 'env.sh' ) + return script, path, None return None, None, None diff -r b16ab0d407a09695aaa0dcbffcd9661b12c4b701 -r 630cfe0030abf40e46c9839254278531d657e554 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -9,6 +9,7 @@ def test_tool_dependencies(): # Setup directories + with __test_base_path() as base_path: for name, version, sub in [ ( "dep1", "1.0", "env.sh" ), ( "dep1", "2.0", "bin" ), ( "dep2", "1.0", None ) ]: if sub == "bin": @@ -22,7 +23,7 @@ if sub == "env.sh": __touch( os.path.join( p, "env.sh" ) ) - dm = DependencyManager( [base_path] ) + dm = DependencyManager( default_base_path=base_path ) d1_script, d1_path, d1_version = dm.find_dep( "dep1", "1.0" ) assert d1_script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) assert d1_path == os.path.join( base_path, 'dep1', '1.0' ) @@ -52,7 +53,7 @@ def test_toolshed_set_enviornment_requiremetns(): with __test_base_path() as base_path: test_repo = __build_test_repo('set_environment') - dm = DependencyManager( [base_path] ) + dm = DependencyManager( default_base_path=base_path ) env_settings_dir = os.path.join(base_path, "environment_settings", TEST_REPO_NAME, TEST_REPO_USER, TEST_REPO_NAME, TEST_REPO_CHANGESET) os.makedirs(env_settings_dir) d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=None, type='set_environment', installed_tool_dependencies=[test_repo] ) @@ -63,7 +64,7 @@ def test_toolshed_package_requirements(): with __test_base_path() as base_path: test_repo = __build_test_repo('package', version=TEST_VERSION) - dm = DependencyManager( [base_path] ) + dm = DependencyManager( default_base_path=base_path ) package_dir = __build_ts_test_package(base_path) d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) assert d1_version == TEST_VERSION, d1_version @@ -72,7 +73,7 @@ def test_toolshed_tools_fallback_on_manual_dependencies(): with __test_base_path() as base_path: - dm = DependencyManager( [base_path] ) + dm = DependencyManager( default_base_path=base_path ) test_repo = __build_test_repo('package', version=TEST_VERSION) env_path = __setup_galaxy_package_dep(base_path, "dep1", "1.0") d1_script, d1_path, d1_version = dm.find_dep( "dep1", version="1.0", type='package', installed_tool_dependencies=[test_repo] ) @@ -82,7 +83,7 @@ def test_toolshed_greater_precendence(): with __test_base_path() as base_path: - dm = DependencyManager( [base_path] ) + dm = DependencyManager( default_base_path=base_path ) test_repo = __build_test_repo('package', version=TEST_VERSION) ts_package_dir = __build_ts_test_package(base_path) gx_env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION) https://bitbucket.org/galaxy/galaxy-central/commits/a54a058fa777/ Changeset: a54a058fa777 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Small clean ups. Affected #: 1 file diff -r 630cfe0030abf40e46c9839254278531d657e554 -r a54a058fa77737cd1f9d51393cb46f5bf8dda95f lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -7,15 +7,19 @@ import logging log = logging.getLogger( __name__ ) +INDETERMINATE_DEPENDENCY = (None, None, None) + class DependencyManager( object ): """ - A DependencyManager attempts to resolve named and versioned dependencies by searching for them under a list of directories. Directories should be + A DependencyManager attempts to resolve named and versioned dependencies by + searching for them under a list of directories. Directories should be of the form: $BASE/name/version/... - and should each contain a file 'env.sh' which can be sourced to make the dependency available in the current shell environment. + and should each contain a file 'env.sh' which can be sourced to make the + dependency available in the current shell environment. """ def __init__( self, default_base_path ): """ @@ -50,7 +54,7 @@ return script, path, version elif os.path.exists( os.path.join( path, 'bin' ) ): return None, path, version - return None, None, None + return INDETERMINATE_DEPENDENCY def _find_dep_default( self, name, type='package', installed_tool_dependencies=None ): if type == 'set_environment' and installed_tool_dependencies: @@ -71,16 +75,17 @@ return script, real_path, real_version elif os.path.exists( real_bin ): return None, real_path, real_version - return None, None, None + return INDETERMINATE_DEPENDENCY def _get_installed_dependency( self, installed_tool_dependencies, name, type, version=None ): if installed_tool_dependencies: for installed_tool_dependency in installed_tool_dependencies: + name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type if version: - if installed_tool_dependency.name == name and installed_tool_dependency.type == type and installed_tool_dependency.version == version: + if name_and_type_equal and installed_tool_dependency.version == version: return installed_tool_dependency else: - if installed_tool_dependency.name == name and installed_tool_dependency.type == type: + if name_and_type_equal: return installed_tool_dependency return None @@ -105,4 +110,4 @@ if os.path.exists( path ): script = os.path.join( path, 'env.sh' ) return script, path, None - return None, None, None + return INDETERMINATE_DEPENDENCY https://bitbucket.org/galaxy/galaxy-central/commits/8ca4fdce3052/ Changeset: 8ca4fdce3052 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Begin rearranging arguments so all ones not of interest to every potential dependency resolution plugin can be kwds Affected #: 1 file diff -r a54a058fa77737cd1f9d51393cb46f5bf8dda95f -r 8ca4fdce30526d58737d2b0d21af15cb750ddffc lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -43,7 +43,7 @@ return self._find_dep_versioned( name, version, type=type, installed_tool_dependencies=installed_tool_dependencies ) def _find_dep_versioned( self, name, version, type='package', installed_tool_dependencies=None ): - installed_tool_dependency = self._get_installed_dependency( installed_tool_dependencies, name, type, version=version ) + installed_tool_dependency = self._get_installed_dependency( name, type, version=version, installed_tool_dependencies=installed_tool_dependencies ) base_path = self.default_base_path if installed_tool_dependency: path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) @@ -58,7 +58,7 @@ def _find_dep_default( self, name, type='package', installed_tool_dependencies=None ): if type == 'set_environment' and installed_tool_dependencies: - installed_tool_dependency = self._get_installed_dependency( installed_tool_dependencies, name, type, version=None ) + installed_tool_dependency = self._get_installed_dependency( name, type, version=None, installed_tool_dependencies=installed_tool_dependencies ) if installed_tool_dependency: script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) if script and path: @@ -77,16 +77,15 @@ return None, real_path, real_version return INDETERMINATE_DEPENDENCY - def _get_installed_dependency( self, installed_tool_dependencies, name, type, version=None ): - if installed_tool_dependencies: - for installed_tool_dependency in installed_tool_dependencies: - name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type - if version: - if name_and_type_equal and installed_tool_dependency.version == version: - return installed_tool_dependency - else: - if name_and_type_equal: - return installed_tool_dependency + def _get_installed_dependency( self, name, type, version=None, **kwds ): + for installed_tool_dependency in kwds.get("installed_tool_dependencies", []): + name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type + if version: + if name_and_type_equal and installed_tool_dependency.version == version: + return installed_tool_dependency + else: + if name_and_type_equal: + return installed_tool_dependency return None def _get_package_installed_dependency_path( self, installed_tool_dependency, base_path, name, version ): https://bitbucket.org/galaxy/galaxy-central/commits/22f3aca8ad9c/ Changeset: 22f3aca8ad9c User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Finish refactoring toward generic keyword arguments. Affected #: 1 file diff -r 8ca4fdce30526d58737d2b0d21af15cb750ddffc -r 22f3aca8ad9cb9a0e606a1c69cb4c211fe5cc133 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -32,18 +32,18 @@ log.warn( "Path '%s' is not directory, ignoring", default_base_path ) self.default_base_path = os.path.abspath( default_base_path ) - def find_dep( self, name, version=None, type='package', installed_tool_dependencies=None ): + def find_dep( self, name, version=None, type='package', **kwds ): """ Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a symbolic link (if found). Returns a triple of: env_script, base_path, real_version """ if version is None: - return self._find_dep_default( name, type=type, installed_tool_dependencies=installed_tool_dependencies ) + return self._find_dep_default( name, type=type, **kwds ) else: - return self._find_dep_versioned( name, version, type=type, installed_tool_dependencies=installed_tool_dependencies ) + return self._find_dep_versioned( name, version, type=type, **kwds ) - def _find_dep_versioned( self, name, version, type='package', installed_tool_dependencies=None ): - installed_tool_dependency = self._get_installed_dependency( name, type, version=version, installed_tool_dependencies=installed_tool_dependencies ) + def _find_dep_versioned( self, name, version, type='package', **kwds ): + installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) base_path = self.default_base_path if installed_tool_dependency: path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) @@ -56,9 +56,9 @@ return None, path, version return INDETERMINATE_DEPENDENCY - def _find_dep_default( self, name, type='package', installed_tool_dependencies=None ): - if type == 'set_environment' and installed_tool_dependencies: - installed_tool_dependency = self._get_installed_dependency( name, type, version=None, installed_tool_dependencies=installed_tool_dependencies ) + def _find_dep_default( self, name, type='package', **kwds ): + if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): + installed_tool_dependency = self._get_installed_dependency( name, type, version=None, **kwds ) if installed_tool_dependency: script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) if script and path: https://bitbucket.org/galaxy/galaxy-central/commits/dd2b36c4fc66/ Changeset: dd2b36c4fc66 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Introduce the concept of a dependency resolver, DependencyManager will check each in turn. Affected #: 1 file diff -r 22f3aca8ad9cb9a0e606a1c69cb4c211fe5cc133 -r dd2b36c4fc6682e76cceef10de2ae3154cc107c6 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -31,8 +31,29 @@ if not os.path.isdir( default_base_path ): log.warn( "Path '%s' is not directory, ignoring", default_base_path ) self.default_base_path = os.path.abspath( default_base_path ) + self.dependency_resolvers = [ GalaxyPackageDependencyResolver(self) ] + def find_dep( self, name, version=None, type='package', **kwds ): + for resolver in self.dependency_resolvers: + dependency = resolver.resolve( name, version, type, **kwds ) + if dependency != INDETERMINATE_DEPENDENCY: + return dependency + return INDETERMINATE_DEPENDENCY + + +class DependencyResolver(object): + + def resolve( self, name, version, type, **kwds ): + raise NotImplementedError() + + +class GalaxyPackageDependencyResolver(DependencyResolver): + + def __init__(self, dependency_manager): + self.default_base_path = dependency_manager.default_base_path + + def resolve( self, name, version, type, **kwds ): """ Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a symbolic link (if found). Returns a triple of: env_script, base_path, real_version https://bitbucket.org/galaxy/galaxy-central/commits/ae5dbc930415/ Changeset: ae5dbc930415 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Refactor common code out of _find_dep_versioned and _find_dep_default. Affected #: 1 file diff -r dd2b36c4fc6682e76cceef10de2ae3154cc107c6 -r ae5dbc9304152e034ed5565f5243ab6b5c0cbe77 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -70,12 +70,7 @@ path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) else: path = os.path.join( base_path, name, version ) - script = os.path.join( path, 'env.sh' ) - if os.path.exists( script ): - return script, path, version - elif os.path.exists( os.path.join( path, 'bin' ) ): - return None, path, version - return INDETERMINATE_DEPENDENCY + return self._galaxy_package_dep(path, version) def _find_dep_default( self, name, type='package', **kwds ): if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): @@ -89,13 +84,17 @@ path = os.path.join( base_path, name, 'default' ) if os.path.islink( path ): real_path = os.path.realpath( path ) - real_bin = os.path.join( real_path, 'bin' ) real_version = os.path.basename( real_path ) - script = os.path.join( real_path, 'env.sh' ) - if os.path.exists( script ): - return script, real_path, real_version - elif os.path.exists( real_bin ): - return None, real_path, real_version + return self._galaxy_package_dep(real_path, real_version) + else: + return INDETERMINATE_DEPENDENCY + + def _galaxy_package_dep( self, path, version ): + script = os.path.join( path, 'env.sh' ) + if os.path.exists( script ): + return script, path, version + elif os.path.exists( os.path.join( path, 'bin' ) ): + return None, path, version return INDETERMINATE_DEPENDENCY def _get_installed_dependency( self, name, type, version=None, **kwds ): https://bitbucket.org/galaxy/galaxy-central/commits/8928cf0be809/ Changeset: 8928cf0be809 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Split package resolvers into Tool Shed and traditional Galaxy versions. Affected #: 1 file diff -r ae5dbc9304152e034ed5565f5243ab6b5c0cbe77 -r 8928cf0be80994703689e89c96e2dbfa20a47319 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -31,7 +31,10 @@ if not os.path.isdir( default_base_path ): log.warn( "Path '%s' is not directory, ignoring", default_base_path ) self.default_base_path = os.path.abspath( default_base_path ) - self.dependency_resolvers = [ GalaxyPackageDependencyResolver(self) ] + self.dependency_resolvers = [ + ToolShedPackageDependencyResolver(self), + GalaxyPackageDependencyResolver(self), + ] def find_dep( self, name, version=None, type='package', **kwds ): @@ -64,22 +67,11 @@ return self._find_dep_versioned( name, version, type=type, **kwds ) def _find_dep_versioned( self, name, version, type='package', **kwds ): - installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) base_path = self.default_base_path - if installed_tool_dependency: - path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) - else: - path = os.path.join( base_path, name, version ) + path = os.path.join( base_path, name, version ) return self._galaxy_package_dep(path, version) def _find_dep_default( self, name, type='package', **kwds ): - if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): - installed_tool_dependency = self._get_installed_dependency( name, type, version=None, **kwds ) - if installed_tool_dependency: - script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) - if script and path: - # Environment settings do not use versions. - return script, path, None base_path = self.default_base_path path = os.path.join( base_path, name, 'default' ) if os.path.islink( path ): @@ -97,6 +89,31 @@ return None, path, version return INDETERMINATE_DEPENDENCY + +class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): + + def __init__(self, dependency_manager): + super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager) + + def _find_dep_versioned( self, name, version, type='package', **kwds ): + installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) + base_path = self.default_base_path + if installed_tool_dependency: + path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) + return self._galaxy_package_dep(path, version) + else: + return INDETERMINATE_DEPENDENCY + + def _find_dep_default( self, name, type='package', **kwds ): + if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): + installed_tool_dependency = self._get_installed_dependency( name, type, version=None, **kwds ) + if installed_tool_dependency: + script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) + if script and path: + # Environment settings do not use versions. + return script, path, None + return INDETERMINATE_DEPENDENCY + def _get_installed_dependency( self, name, type, version=None, **kwds ): for installed_tool_dependency in kwds.get("installed_tool_dependencies", []): name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type https://bitbucket.org/galaxy/galaxy-central/commits/0762bd90e85b/ Changeset: 0762bd90e85b User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Implement XML configuration of tool dependency resolver. This is largely modelled off of @natefoo's job_conf.xml. This is a minimally configurable XML layout so far, but will be dding more options. Already can decide between whether to load tool shed installed packages first (default) or manually conifgured ones. The advantages of loading manually configured ones first is outlined here - http://dev.list.galaxyproject.org/Test-Toolshed-Biopython-package-dependenc…. Affected #: 4 files diff -r 8928cf0be80994703689e89c96e2dbfa20a47319 -r 0762bd90e85b94e643458c22670de8455dc97f71 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -114,6 +114,7 @@ self.collect_outputs_from = [ x.strip() for x in kwargs.get( 'collect_outputs_from', 'new_file_path,job_working_directory' ).lower().split(',') ] self.template_path = resolve_path( kwargs.get( "template_path", "templates" ), self.root ) self.template_cache = resolve_path( kwargs.get( "template_cache_path", "database/compiled_templates" ), self.root ) + self.dependency_resolvers_config_file = resolve_path( kwargs.get( 'dependency_resolvers_config_file', 'dependency_resolvers_conf.xml' ), self.root ) self.job_config_file = resolve_path( kwargs.get( 'job_config_file', 'job_conf.xml' ), self.root ) self.local_job_queue_workers = int( kwargs.get( "local_job_queue_workers", "5" ) ) self.cluster_job_queue_workers = int( kwargs.get( "cluster_job_queue_workers", "3" ) ) diff -r 8928cf0be80994703689e89c96e2dbfa20a47319 -r 0762bd90e85b94e643458c22670de8455dc97f71 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -696,7 +696,11 @@ def init_dependency_manager( self ): if self.app.config.use_tool_dependencies: - self.dependency_manager = DependencyManager( self.app.config.tool_dependency_dir ) + dependency_manager_kwds = { + 'default_base_path': self.app.config.tool_dependency_dir, + 'conf_file': self.app.config.dependency_resolvers_config_file, + } + self.dependency_manager = DependencyManager( **dependency_manager_kwds ) else: self.dependency_manager = None diff -r 8928cf0be80994703689e89c96e2dbfa20a47319 -r 0762bd90e85b94e643458c22670de8455dc97f71 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -7,6 +7,8 @@ import logging log = logging.getLogger( __name__ ) +from galaxy.util import parse_xml + INDETERMINATE_DEPENDENCY = (None, None, None) @@ -21,7 +23,7 @@ and should each contain a file 'env.sh' which can be sourced to make the dependency available in the current shell environment. """ - def __init__( self, default_base_path ): + def __init__( self, default_base_path, conf_file=None ): """ Create a new dependency manager looking for packages under the paths listed in `base_paths`. The default base path is app.config.tool_dependency_dir. @@ -31,10 +33,7 @@ if not os.path.isdir( default_base_path ): log.warn( "Path '%s' is not directory, ignoring", default_base_path ) self.default_base_path = os.path.abspath( default_base_path ) - self.dependency_resolvers = [ - ToolShedPackageDependencyResolver(self), - GalaxyPackageDependencyResolver(self), - ] + self.dependency_resolvers = self.__build_dependency_resolvers( conf_file ) def find_dep( self, name, version=None, type='package', **kwds ): @@ -44,6 +43,32 @@ return dependency return INDETERMINATE_DEPENDENCY + def __build_dependency_resolvers( self, conf_file ): + if not conf_file or not os.path.exists( conf_file ): + return self.__default_dependency_resolvers() + tree = parse_xml( conf_file ) + return self.__parse_resolver_conf_xml( tree ) + + def __default_dependency_resolvers( self ): + return [ + ToolShedPackageDependencyResolver(self), + GalaxyPackageDependencyResolver(self), + ] + + def __parse_resolver_conf_xml(self, tree): + """ + + :param tree: Object representing the root ``<dependency_resolvers>`` object in the file. + :type tree: ``xml.etree.ElementTree.Element`` + """ + resolvers = [] + resolvers_element = tree.getroot() + for resolver_element in resolvers_element.getchildren(): + resolver_type = resolver_element.tag + resolver = RESOLVER_CLASSES[resolver_type](self) + resolvers.append(resolver) + return resolvers + class DependencyResolver(object): @@ -147,3 +172,9 @@ script = os.path.join( path, 'env.sh' ) return script, path, None return INDETERMINATE_DEPENDENCY + + +RESOLVER_CLASSES = { + 'tool_shed_package': ToolShedPackageDependencyResolver, + 'galaxy_package': GalaxyPackageDependencyResolver, +} diff -r 8928cf0be80994703689e89c96e2dbfa20a47319 -r 0762bd90e85b94e643458c22670de8455dc97f71 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -139,3 +139,32 @@ yield base_path finally: rmtree(base_path) + + +def test_parse(): + with __parse_resolvers('''<dependency_resolvers> + <tool_shed_package /> + <galaxy_package /> +</dependency_resolvers> +''') as dependency_resolvers: + assert 'ToolShed' in dependency_resolvers[0].__class__.__name__ + assert 'Galaxy' in dependency_resolvers[1].__class__.__name__ + + with __parse_resolvers('''<dependency_resolvers> + <galaxy_package /> + <tool_shed_package /> +</dependency_resolvers> +''') as dependency_resolvers: + assert 'Galaxy' in dependency_resolvers[0].__class__.__name__ + assert 'ToolShed' in dependency_resolvers[1].__class__.__name__ + + +@contextmanager +def __parse_resolvers(xml_content): + with __test_base_path() as base_path: + f = tempfile.NamedTemporaryFile() + f.write(xml_content) + f.flush() + dm = DependencyManager( default_base_path=base_path, conf_file=f.name ) + yield dm.dependency_resolvers + https://bitbucket.org/galaxy/galaxy-central/commits/4711ced22745/ Changeset: 4711ced22745 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Add ability to pass arguments to tool resolvers. Use new functionlity to allow a much cleaner, more general solution to the problem outlined in Pull Request 220. Allow creation of a Galaxy package dependency resolver that always falls back to 'default' dependency. I think this should be the default last-ditch behavior if a specified version of a tag is not found, but I understand the desire for it not to be, hopefully making it optional in this fashion is an acceptable middle ground. The original problem is essentially, tool shed installs require an exact version of package requirements to be used, but this very encumbering in rapidly evolving and/or non-toolshed use cases (e.g. my Galaxy-P production and development servers). It is tedius to keep different code bases, environments in such cases. Affected #: 2 files diff -r 0762bd90e85b94e643458c22670de8455dc97f71 -r 4711ced227459680997ffe448ea57abd65349fb2 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -7,7 +7,7 @@ import logging log = logging.getLogger( __name__ ) -from galaxy.util import parse_xml +from galaxy.util import parse_xml, string_as_bool INDETERMINATE_DEPENDENCY = (None, None, None) @@ -65,7 +65,8 @@ resolvers_element = tree.getroot() for resolver_element in resolvers_element.getchildren(): resolver_type = resolver_element.tag - resolver = RESOLVER_CLASSES[resolver_type](self) + resolver_kwds = dict(resolver_element.items()) + resolver = RESOLVER_CLASSES[resolver_type](self, **resolver_kwds) resolvers.append(resolver) return resolvers @@ -78,15 +79,22 @@ class GalaxyPackageDependencyResolver(DependencyResolver): - def __init__(self, dependency_manager): + def __init__(self, dependency_manager, **kwds): self.default_base_path = dependency_manager.default_base_path + ## Galaxy tool shed requires explicit versions on XML elements, + ## this in inconvient for testing or Galaxy instances not utilizing + ## the tool shed so allow a fallback version of the Galaxy package + ## resolver that will just grab 'default' version of exact version + ## unavailable. + self.versionless = string_as_bool(kwds.get('versionless', "false")) + def resolve( self, name, version, type, **kwds ): """ Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a symbolic link (if found). Returns a triple of: env_script, base_path, real_version """ - if version is None: + if version is None or self.versionless: return self._find_dep_default( name, type=type, **kwds ) else: return self._find_dep_versioned( name, version, type=type, **kwds ) @@ -117,8 +125,8 @@ class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): - def __init__(self, dependency_manager): - super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager) + def __init__(self, dependency_manager, **kwds): + super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager, **kwds) def _find_dep_versioned( self, name, version, type='package', **kwds ): installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) diff -r 0762bd90e85b94e643458c22670de8455dc97f71 -r 4711ced227459680997ffe448ea57abd65349fb2 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -158,6 +158,15 @@ assert 'Galaxy' in dependency_resolvers[0].__class__.__name__ assert 'ToolShed' in dependency_resolvers[1].__class__.__name__ + with __parse_resolvers('''<dependency_resolvers> + <galaxy_package /> + <tool_shed_package /> + <galaxy_package versionless="true" /> +</dependency_resolvers> +''') as dependency_resolvers: + assert not dependency_resolvers[0].versionless + assert dependency_resolvers[2].versionless + @contextmanager def __parse_resolvers(xml_content): https://bitbucket.org/galaxy/galaxy-central/commits/73c22fa3ae10/ Changeset: 73c22fa3ae10 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Allow overridding the base_path on 'galaxy_package' tool dependency resolvers. There was some code for doing this earlier but it wasn't configurable in anyway. This version looks like it should work and is cleaner. Affected #: 2 files diff -r 4711ced227459680997ffe448ea57abd65349fb2 -r 73c22fa3ae109e14bd70128c6e00174beae45629 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -80,14 +80,13 @@ class GalaxyPackageDependencyResolver(DependencyResolver): def __init__(self, dependency_manager, **kwds): - self.default_base_path = dependency_manager.default_base_path - ## Galaxy tool shed requires explicit versions on XML elements, ## this in inconvient for testing or Galaxy instances not utilizing ## the tool shed so allow a fallback version of the Galaxy package ## resolver that will just grab 'default' version of exact version ## unavailable. self.versionless = string_as_bool(kwds.get('versionless', "false")) + self.base_path = kwds.get('base_path', dependency_manager.default_base_path) def resolve( self, name, version, type, **kwds ): """ @@ -100,12 +99,12 @@ return self._find_dep_versioned( name, version, type=type, **kwds ) def _find_dep_versioned( self, name, version, type='package', **kwds ): - base_path = self.default_base_path + base_path = self.base_path path = os.path.join( base_path, name, version ) return self._galaxy_package_dep(path, version) def _find_dep_default( self, name, type='package', **kwds ): - base_path = self.default_base_path + base_path = self.base_path path = os.path.join( base_path, name, 'default' ) if os.path.islink( path ): real_path = os.path.realpath( path ) @@ -130,7 +129,7 @@ def _find_dep_versioned( self, name, version, type='package', **kwds ): installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) - base_path = self.default_base_path + base_path = self.base_path if installed_tool_dependency: path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) return self._galaxy_package_dep(path, version) @@ -169,7 +168,7 @@ def _get_set_environment_installed_dependency_script_path( self, installed_tool_dependency, name ): tool_shed_repository = installed_tool_dependency.tool_shed_repository - base_path = self.default_base_path + base_path = self.base_path path = os.path.abspath( os.path.join( base_path, 'environment_settings', name, diff -r 4711ced227459680997ffe448ea57abd65349fb2 -r 73c22fa3ae109e14bd70128c6e00174beae45629 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -167,6 +167,19 @@ assert not dependency_resolvers[0].versionless assert dependency_resolvers[2].versionless + with __parse_resolvers('''<dependency_resolvers> + <galaxy_package /> + <tool_shed_package /> + <galaxy_package base_path="/opt/galaxy/legacy/"/> +</dependency_resolvers> +''') as dependency_resolvers: + # Unspecified base_paths are both default_base_paths + assert dependency_resolvers[0].base_path == dependency_resolvers[1].base_path + # Can specify custom base path... + assert dependency_resolvers[2].base_path == "/opt/galaxy/legacy/" + # ... that is different from the default. + assert dependency_resolvers[0].base_path != dependency_resolvers[2].base_path + @contextmanager def __parse_resolvers(xml_content): https://bitbucket.org/galaxy/galaxy-central/commits/36d6cca02ff2/ Changeset: 36d6cca02ff2 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Modify the tool resolver syntax slightly. <galaxy_packages /> looks better than <galaxy_package /> since it is a whole class of possible dependencies the corresponding resolver could resolve. Affected #: 2 files diff -r 73c22fa3ae109e14bd70128c6e00174beae45629 -r 36d6cca02ff256889ea3a65b256b098431d35192 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -182,6 +182,6 @@ RESOLVER_CLASSES = { - 'tool_shed_package': ToolShedPackageDependencyResolver, - 'galaxy_package': GalaxyPackageDependencyResolver, + 'tool_shed_packages': ToolShedPackageDependencyResolver, + 'galaxy_packages': GalaxyPackageDependencyResolver, } diff -r 73c22fa3ae109e14bd70128c6e00174beae45629 -r 36d6cca02ff256889ea3a65b256b098431d35192 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -143,34 +143,34 @@ def test_parse(): with __parse_resolvers('''<dependency_resolvers> - <tool_shed_package /> - <galaxy_package /> + <tool_shed_packages /> + <galaxy_packages /></dependency_resolvers> ''') as dependency_resolvers: assert 'ToolShed' in dependency_resolvers[0].__class__.__name__ assert 'Galaxy' in dependency_resolvers[1].__class__.__name__ with __parse_resolvers('''<dependency_resolvers> - <galaxy_package /> - <tool_shed_package /> + <galaxy_packages /> + <tool_shed_packages /></dependency_resolvers> ''') as dependency_resolvers: assert 'Galaxy' in dependency_resolvers[0].__class__.__name__ assert 'ToolShed' in dependency_resolvers[1].__class__.__name__ with __parse_resolvers('''<dependency_resolvers> - <galaxy_package /> - <tool_shed_package /> - <galaxy_package versionless="true" /> + <galaxy_packages /> + <tool_shed_packages /> + <galaxy_packages versionless="true" /></dependency_resolvers> ''') as dependency_resolvers: assert not dependency_resolvers[0].versionless assert dependency_resolvers[2].versionless with __parse_resolvers('''<dependency_resolvers> - <galaxy_package /> - <tool_shed_package /> - <galaxy_package base_path="/opt/galaxy/legacy/"/> + <galaxy_packages /> + <tool_shed_packages /> + <galaxy_packages base_path="/opt/galaxy/legacy/"/></dependency_resolvers> ''') as dependency_resolvers: # Unspecified base_paths are both default_base_paths https://bitbucket.org/galaxy/galaxy-central/commits/b7380008b339/ Changeset: b7380008b339 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Refactor actual tool resolvers out into the own module. Cleans up lib/galaxy/tools/deps/__init__.py significantly and makes inter-dependencies more clear. Affected #: 4 files diff -r 36d6cca02ff256889ea3a65b256b098431d35192 -r b7380008b339394d0f2957f778d2b80f00e72727 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -7,9 +7,10 @@ import logging log = logging.getLogger( __name__ ) -from galaxy.util import parse_xml, string_as_bool - -INDETERMINATE_DEPENDENCY = (None, None, None) +from galaxy.util import parse_xml +from .resolvers import INDETERMINATE_DEPENDENCY +from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver +from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver class DependencyManager( object ): @@ -70,117 +71,6 @@ resolvers.append(resolver) return resolvers - -class DependencyResolver(object): - - def resolve( self, name, version, type, **kwds ): - raise NotImplementedError() - - -class GalaxyPackageDependencyResolver(DependencyResolver): - - def __init__(self, dependency_manager, **kwds): - ## Galaxy tool shed requires explicit versions on XML elements, - ## this in inconvient for testing or Galaxy instances not utilizing - ## the tool shed so allow a fallback version of the Galaxy package - ## resolver that will just grab 'default' version of exact version - ## unavailable. - self.versionless = string_as_bool(kwds.get('versionless', "false")) - self.base_path = kwds.get('base_path', dependency_manager.default_base_path) - - def resolve( self, name, version, type, **kwds ): - """ - Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a - symbolic link (if found). Returns a triple of: env_script, base_path, real_version - """ - if version is None or self.versionless: - return self._find_dep_default( name, type=type, **kwds ) - else: - return self._find_dep_versioned( name, version, type=type, **kwds ) - - def _find_dep_versioned( self, name, version, type='package', **kwds ): - base_path = self.base_path - path = os.path.join( base_path, name, version ) - return self._galaxy_package_dep(path, version) - - def _find_dep_default( self, name, type='package', **kwds ): - base_path = self.base_path - path = os.path.join( base_path, name, 'default' ) - if os.path.islink( path ): - real_path = os.path.realpath( path ) - real_version = os.path.basename( real_path ) - return self._galaxy_package_dep(real_path, real_version) - else: - return INDETERMINATE_DEPENDENCY - - def _galaxy_package_dep( self, path, version ): - script = os.path.join( path, 'env.sh' ) - if os.path.exists( script ): - return script, path, version - elif os.path.exists( os.path.join( path, 'bin' ) ): - return None, path, version - return INDETERMINATE_DEPENDENCY - - -class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): - - def __init__(self, dependency_manager, **kwds): - super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager, **kwds) - - def _find_dep_versioned( self, name, version, type='package', **kwds ): - installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) - base_path = self.base_path - if installed_tool_dependency: - path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) - return self._galaxy_package_dep(path, version) - else: - return INDETERMINATE_DEPENDENCY - - def _find_dep_default( self, name, type='package', **kwds ): - if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): - installed_tool_dependency = self._get_installed_dependency( name, type, version=None, **kwds ) - if installed_tool_dependency: - script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) - if script and path: - # Environment settings do not use versions. - return script, path, None - return INDETERMINATE_DEPENDENCY - - def _get_installed_dependency( self, name, type, version=None, **kwds ): - for installed_tool_dependency in kwds.get("installed_tool_dependencies", []): - name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type - if version: - if name_and_type_equal and installed_tool_dependency.version == version: - return installed_tool_dependency - else: - if name_and_type_equal: - return installed_tool_dependency - return None - - def _get_package_installed_dependency_path( self, installed_tool_dependency, base_path, name, version ): - tool_shed_repository = installed_tool_dependency.tool_shed_repository - return os.path.join( base_path, - name, - version, - tool_shed_repository.owner, - tool_shed_repository.name, - tool_shed_repository.installed_changeset_revision ) - - def _get_set_environment_installed_dependency_script_path( self, installed_tool_dependency, name ): - tool_shed_repository = installed_tool_dependency.tool_shed_repository - base_path = self.base_path - path = os.path.abspath( os.path.join( base_path, - 'environment_settings', - name, - tool_shed_repository.owner, - tool_shed_repository.name, - tool_shed_repository.installed_changeset_revision ) ) - if os.path.exists( path ): - script = os.path.join( path, 'env.sh' ) - return script, path, None - return INDETERMINATE_DEPENDENCY - - RESOLVER_CLASSES = { 'tool_shed_packages': ToolShedPackageDependencyResolver, 'galaxy_packages': GalaxyPackageDependencyResolver, diff -r 36d6cca02ff256889ea3a65b256b098431d35192 -r b7380008b339394d0f2957f778d2b80f00e72727 lib/galaxy/tools/deps/resolvers/__init__.py --- /dev/null +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -0,0 +1,18 @@ +from abc import ABCMeta, abstractmethod + +INDETERMINATE_DEPENDENCY = (None, None, None) + + +class DependencyResolver(object): + __metaclass__ = ABCMeta + + @abstractmethod + def resolve( self, name, version, type, **kwds ): + """ + Given inputs describing dependency in the abstract, yield tuple of + (script, bin, version). Here script is the env.sh file to source + before running a job, if that is not found the bin directory will be + appended to the path (if it is not None). Finally, version is the + resolved tool dependency version (which may differ from requested + version for instance if the request version is 'default'.) + """ diff -r 36d6cca02ff256889ea3a65b256b098431d35192 -r b7380008b339394d0f2957f778d2b80f00e72727 lib/galaxy/tools/deps/resolvers/galaxy_packages.py --- /dev/null +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -0,0 +1,51 @@ +from os.path import join, islink, realpath, basename, exists, abspath + +from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY +from galaxy.util import string_as_bool + + +class GalaxyPackageDependencyResolver(DependencyResolver): + + def __init__(self, dependency_manager, **kwds): + ## Galaxy tool shed requires explicit versions on XML elements, + ## this in inconvient for testing or Galaxy instances not utilizing + ## the tool shed so allow a fallback version of the Galaxy package + ## resolver that will just grab 'default' version of exact version + ## unavailable. + self.versionless = string_as_bool(kwds.get('versionless', "false")) + self.base_path = abspath( kwds.get('base_path', dependency_manager.default_base_path) ) + + def resolve( self, name, version, type, **kwds ): + """ + Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a + symbolic link (if found). Returns a triple of: env_script, base_path, real_version + """ + if version is None or self.versionless: + return self._find_dep_default( name, type=type, **kwds ) + else: + return self._find_dep_versioned( name, version, type=type, **kwds ) + + def _find_dep_versioned( self, name, version, type='package', **kwds ): + base_path = self.base_path + path = join( base_path, name, version ) + return self._galaxy_package_dep(path, version) + + def _find_dep_default( self, name, type='package', **kwds ): + base_path = self.base_path + path = join( base_path, name, 'default' ) + if islink( path ): + real_path = realpath( path ) + real_version = basename( real_path ) + return self._galaxy_package_dep(real_path, real_version) + else: + return INDETERMINATE_DEPENDENCY + + def _galaxy_package_dep( self, path, version ): + script = join( path, 'env.sh' ) + if exists( script ): + return script, path, version + elif exists( join( path, 'bin' ) ): + return None, path, version + return INDETERMINATE_DEPENDENCY + +__all__ = [GalaxyPackageDependencyResolver] diff -r 36d6cca02ff256889ea3a65b256b098431d35192 -r b7380008b339394d0f2957f778d2b80f00e72727 lib/galaxy/tools/deps/resolvers/tool_shed_packages.py --- /dev/null +++ b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py @@ -0,0 +1,66 @@ +from os.path import abspath, join, exists + +from .galaxy_packages import GalaxyPackageDependencyResolver +from ..resolvers import INDETERMINATE_DEPENDENCY + + +class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): + + def __init__(self, dependency_manager, **kwds): + super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager, **kwds) + + def _find_dep_versioned( self, name, version, type='package', **kwds ): + installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) + base_path = self.base_path + if installed_tool_dependency: + path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) + return self._galaxy_package_dep(path, version) + else: + return INDETERMINATE_DEPENDENCY + + def _find_dep_default( self, name, type='package', **kwds ): + if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): + installed_tool_dependency = self._get_installed_dependency( name, type, version=None, **kwds ) + if installed_tool_dependency: + script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) + if script and path: + # Environment settings do not use versions. + return script, path, None + return INDETERMINATE_DEPENDENCY + + def _get_installed_dependency( self, name, type, version=None, **kwds ): + installed_tool_dependencies = kwds.get("installed_tool_dependencies", []) + for installed_tool_dependency in (installed_tool_dependencies or []): + name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type + if version: + if name_and_type_equal and installed_tool_dependency.version == version: + return installed_tool_dependency + else: + if name_and_type_equal: + return installed_tool_dependency + return None + + def _get_package_installed_dependency_path( self, installed_tool_dependency, base_path, name, version ): + tool_shed_repository = installed_tool_dependency.tool_shed_repository + return join( base_path, + name, + version, + tool_shed_repository.owner, + tool_shed_repository.name, + tool_shed_repository.installed_changeset_revision ) + + def _get_set_environment_installed_dependency_script_path( self, installed_tool_dependency, name ): + tool_shed_repository = installed_tool_dependency.tool_shed_repository + base_path = self.base_path + path = abspath( join( base_path, + 'environment_settings', + name, + tool_shed_repository.owner, + tool_shed_repository.name, + tool_shed_repository.installed_changeset_revision ) ) + if exists( path ): + script = join( path, 'env.sh' ) + return script, path, None + return INDETERMINATE_DEPENDENCY + +__all__ = [ToolShedPackageDependencyResolver] https://bitbucket.org/galaxy/galaxy-central/commits/21b9e99a73fe/ Changeset: 21b9e99a73fe User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Introduce higher-level abstraction describing dependency. Passing the tuple (script, path, version) around is insufficient to represent other kinds of potential dependency load commands (namely module loads), this generalization allows more expressivity. I believe this abstraction also moves some logic out of the tool class that is better encapsulated in the deps module and submodules. Affected #: 5 files diff -r b7380008b339394d0f2957f778d2b80f00e72727 -r 21b9e99a73fe96721498dda9751680741f78e946 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -40,7 +40,7 @@ from galaxy.tools.actions import DefaultToolAction from galaxy.tools.actions.data_source import DataSourceToolAction from galaxy.tools.actions.data_manager import DataManagerToolAction -from galaxy.tools.deps import DependencyManager +from galaxy.tools.deps import DependencyManager, INDETERMINATE_DEPENDENCY from galaxy.tools.parameters import check_param, params_from_strings, params_to_strings from galaxy.tools.parameters.basic import (BaseURLToolParameter, DataToolParameter, HiddenToolParameter, LibraryDatasetToolParameter, @@ -2681,20 +2681,17 @@ installed_tool_dependencies = None for requirement in self.requirements: log.debug( "Building dependency shell command for dependency '%s'", requirement.name ) - script_file = None - base_path = None - version = None + dependency = INDETERMINATE_DEPENDENCY if requirement.type in [ 'package', 'set_environment' ]: - script_file, base_path, version = self.app.toolbox.dependency_manager.find_dep( name=requirement.name, - version=requirement.version, - type=requirement.type, - installed_tool_dependencies=installed_tool_dependencies ) - if script_file is None and base_path is None: + dependency = self.app.toolbox.dependency_manager.find_dep( name=requirement.name, + version=requirement.version, + type=requirement.type, + installed_tool_dependencies=installed_tool_dependencies ) + dependency_commands = dependency.shell_commands( requirement ) + if not dependency_commands: log.warn( "Failed to resolve dependency on '%s', ignoring", requirement.name ) - elif requirement.type == 'package' and script_file is None: - commands.append( 'PACKAGE_BASE=%s; export PACKAGE_BASE; PATH="%s/bin:$PATH"; export PATH' % ( base_path, base_path ) ) else: - commands.append( 'PACKAGE_BASE=%s; export PACKAGE_BASE; . %s' % ( base_path, script_file ) ) + commands.append(dependency_commands) return commands def build_redirect_url_params( self, param_dict ): """ diff -r b7380008b339394d0f2957f778d2b80f00e72727 -r 21b9e99a73fe96721498dda9751680741f78e946 lib/galaxy/tools/deps/resolvers/__init__.py --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -1,9 +1,7 @@ from abc import ABCMeta, abstractmethod -INDETERMINATE_DEPENDENCY = (None, None, None) - -class DependencyResolver(object): +class DependencyResolver( object ): __metaclass__ = ABCMeta @abstractmethod @@ -16,3 +14,21 @@ resolved tool dependency version (which may differ from requested version for instance if the request version is 'default'.) """ + + +class Dependency( object ): + __metaclass__ = ABCMeta + + @abstractmethod + def shell_commands( self, requirement ): + """ + Return shell commands to enable this dependency. + """ + + +class NullDependency( Dependency ): + + def shell_commands( self, requirement ): + return None + +INDETERMINATE_DEPENDENCY = NullDependency() diff -r b7380008b339394d0f2957f778d2b80f00e72727 -r 21b9e99a73fe96721498dda9751680741f78e946 lib/galaxy/tools/deps/resolvers/galaxy_packages.py --- a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -1,8 +1,11 @@ from os.path import join, islink, realpath, basename, exists, abspath -from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY +from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY, Dependency from galaxy.util import string_as_bool +import logging +log = logging.getLogger( __name__ ) + class GalaxyPackageDependencyResolver(DependencyResolver): @@ -43,9 +46,28 @@ def _galaxy_package_dep( self, path, version ): script = join( path, 'env.sh' ) if exists( script ): - return script, path, version + return GalaxyPackageDependency(script, path, version) elif exists( join( path, 'bin' ) ): - return None, path, version + return GalaxyPackageDependency(None, path, version) return INDETERMINATE_DEPENDENCY -__all__ = [GalaxyPackageDependencyResolver] + +class GalaxyPackageDependency(Dependency): + + def __init__( self, script, path, version ): + self.script = script + self.path = path + self.version = version + + def shell_commands( self, requirement ): + base_path = self.path + if self.script is None and base_path is None: + log.warn( "Failed to resolve dependency on '%s', ignoring", requirement.name ) + commands = None + elif requirement.type == 'package' and self.script is None: + commands = 'PACKAGE_BASE=%s; export PACKAGE_BASE; PATH="%s/bin:$PATH"; export PATH' % ( base_path, base_path ) + else: + commands = 'PACKAGE_BASE=%s; export PACKAGE_BASE; . %s' % ( base_path, self.script ) + return commands + +__all__ = [GalaxyPackageDependencyResolver, GalaxyPackageDependency] diff -r b7380008b339394d0f2957f778d2b80f00e72727 -r 21b9e99a73fe96721498dda9751680741f78e946 lib/galaxy/tools/deps/resolvers/tool_shed_packages.py --- a/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py +++ b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py @@ -1,6 +1,6 @@ from os.path import abspath, join, exists -from .galaxy_packages import GalaxyPackageDependencyResolver +from .galaxy_packages import GalaxyPackageDependencyResolver, GalaxyPackageDependency from ..resolvers import INDETERMINATE_DEPENDENCY @@ -22,10 +22,10 @@ if type == 'set_environment' and kwds.get('installed_tool_dependencies', None): installed_tool_dependency = self._get_installed_dependency( name, type, version=None, **kwds ) if installed_tool_dependency: - script, path, version = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) - if script and path: + dependency = self._get_set_environment_installed_dependency_script_path( installed_tool_dependency, name ) + if dependency.script and dependency.path: # Environment settings do not use versions. - return script, path, None + return GalaxyPackageDependency(dependency.script, dependency.path, None) return INDETERMINATE_DEPENDENCY def _get_installed_dependency( self, name, type, version=None, **kwds ): @@ -60,7 +60,7 @@ tool_shed_repository.installed_changeset_revision ) ) if exists( path ): script = join( path, 'env.sh' ) - return script, path, None + return GalaxyPackageDependency(script, path, None) return INDETERMINATE_DEPENDENCY __all__ = [ToolShedPackageDependencyResolver] diff -r b7380008b339394d0f2957f778d2b80f00e72727 -r 21b9e99a73fe96721498dda9751680741f78e946 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -2,9 +2,11 @@ import os.path from os import makedirs, symlink from shutil import rmtree -from galaxy.tools.deps import DependencyManager +from galaxy.tools.deps import DependencyManager, INDETERMINATE_DEPENDENCY +from galaxy.tools.deps.resolvers.galaxy_packages import GalaxyPackageDependency from galaxy.util.bunch import Bunch from contextlib import contextmanager +from subprocess import Popen, PIPE def test_tool_dependencies(): @@ -24,24 +26,23 @@ __touch( os.path.join( p, "env.sh" ) ) dm = DependencyManager( default_base_path=base_path ) - d1_script, d1_path, d1_version = dm.find_dep( "dep1", "1.0" ) - assert d1_script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) - assert d1_path == os.path.join( base_path, 'dep1', '1.0' ) - assert d1_version == "1.0" - d2_script, d2_path, d2_version = dm.find_dep( "dep1", "2.0" ) - assert d2_script == None - assert d2_path == os.path.join( base_path, 'dep1', '2.0' ) - assert d2_version == "2.0" + dependency = dm.find_dep( "dep1", "1.0" ) + assert dependency.script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) + assert dependency.path == os.path.join( base_path, 'dep1', '1.0' ) + assert dependency.version == "1.0" + dependency = dm.find_dep( "dep1", "2.0" ) + assert dependency.script == None + assert dependency.path == os.path.join( base_path, 'dep1', '2.0' ) + assert dependency.version == "2.0" ## Test default versions symlink( os.path.join( base_path, 'dep1', '2.0'), os.path.join( base_path, 'dep1', 'default' ) ) - default_script, default_path, default_version = dm.find_dep( "dep1", None ) - assert default_version == "2.0" + dependency = dm.find_dep( "dep1", None ) + assert dependency.version == "2.0" ## Test default will not be fallen back upon by default - default_script, default_path, default_version = dm.find_dep( "dep1", "2.1" ) - assert default_script == None - assert default_version == None + dependency = dm.find_dep( "dep1", "2.1" ) + assert dependency == INDETERMINATE_DEPENDENCY TEST_REPO_USER = "devteam" @@ -56,9 +57,9 @@ dm = DependencyManager( default_base_path=base_path ) env_settings_dir = os.path.join(base_path, "environment_settings", TEST_REPO_NAME, TEST_REPO_USER, TEST_REPO_NAME, TEST_REPO_CHANGESET) os.makedirs(env_settings_dir) - d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=None, type='set_environment', installed_tool_dependencies=[test_repo] ) - assert d1_version == None - assert d1_script == os.path.join(env_settings_dir, "env.sh"), d1_script + dependency = dm.find_dep( TEST_REPO_NAME, version=None, type='set_environment', installed_tool_dependencies=[test_repo] ) + assert dependency.version == None + assert dependency.script == os.path.join(env_settings_dir, "env.sh") def test_toolshed_package_requirements(): @@ -66,9 +67,9 @@ test_repo = __build_test_repo('package', version=TEST_VERSION) dm = DependencyManager( default_base_path=base_path ) package_dir = __build_ts_test_package(base_path) - d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) - assert d1_version == TEST_VERSION, d1_version - assert d1_script == os.path.join(package_dir, "env.sh"), d1_script + dependency = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) + assert dependency.version == TEST_VERSION + assert dependency.script == os.path.join(package_dir, "env.sh") def test_toolshed_tools_fallback_on_manual_dependencies(): @@ -76,9 +77,9 @@ dm = DependencyManager( default_base_path=base_path ) test_repo = __build_test_repo('package', version=TEST_VERSION) env_path = __setup_galaxy_package_dep(base_path, "dep1", "1.0") - d1_script, d1_path, d1_version = dm.find_dep( "dep1", version="1.0", type='package', installed_tool_dependencies=[test_repo] ) - assert d1_version == "1.0" - assert d1_script == env_path + dependency = dm.find_dep( "dep1", version="1.0", type='package', installed_tool_dependencies=[test_repo] ) + assert dependency.version == "1.0" + assert dependency.script == env_path def test_toolshed_greater_precendence(): @@ -88,9 +89,9 @@ ts_package_dir = __build_ts_test_package(base_path) gx_env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION) ts_env_path = os.path.join(ts_package_dir, "env.sh") - d1_script, d1_path, d1_version = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) - assert d1_script != gx_env_path # Not the galaxy path, it should be the tool shed path used. - assert d1_script == ts_env_path + dependency = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) + assert dependency.script != gx_env_path # Not the galaxy path, it should be the tool shed path used. + assert dependency.script == ts_env_path def __build_ts_test_package(base_path, script_contents=''): @@ -99,6 +100,18 @@ return package_dir +def test_galaxy_dependency_object_script(): + with __test_base_path() as base_path: + ## Create env.sh file that just exports variable FOO and verify it + ## shell_commands export it correctly. + env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION, "export FOO=\"bar\"") + dependency = GalaxyPackageDependency(env_path, os.path.dirname(env_path), TEST_VERSION) + command = ["bash", "-c", "%s; echo \"$FOO\"" % dependency.shell_commands(Bunch(type="package"))] + process = Popen(command, stdout=PIPE) + output = process.communicate()[0].strip() + assert output == 'bar' + + def __setup_galaxy_package_dep(base_path, name, version, contents=""): dep_directory = os.path.join( base_path, name, version ) env_path = os.path.join( dep_directory, "env.sh" ) https://bitbucket.org/galaxy/galaxy-central/commits/31f6c565ca43/ Changeset: 31f6c565ca43 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Outline of integrating environment modules. This is completely untested and undoubtedly needs some fixes. The point is largely to outline how it could be done and let someone with interest in using this go in and flush out the implementation. Note: This is a community contributed feature and support from the core Galaxy team for it will be minimal. Affected #: 3 files diff -r 21b9e99a73fe96721498dda9751680741f78e946 -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -11,6 +11,7 @@ from .resolvers import INDETERMINATE_DEPENDENCY from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver +from .resolvers.modules import ModuleDependencyResolver class DependencyManager( object ): @@ -74,4 +75,5 @@ RESOLVER_CLASSES = { 'tool_shed_packages': ToolShedPackageDependencyResolver, 'galaxy_packages': GalaxyPackageDependencyResolver, + 'modules': ModuleDependencyResolver, } diff -r 21b9e99a73fe96721498dda9751680741f78e946 -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea lib/galaxy/tools/deps/resolvers/modules.py --- /dev/null +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -0,0 +1,137 @@ +""" +This file contains the outline of an implementation to load environment modules +(http://modules.sourceforge.net/). + +This is a community contributed feature and the core Galaxy team does utilize +it, hence support for it will be minimal. The Galaxy team eagerly welcomes +community contribution and maintenance however. +""" +from os.path import exists, isdir, join +from StringIO import StringIO +from subprocess import Popen, PIPE + +from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY, Dependency +from galaxy.util import string_as_bool + +import logging +log = logging.getLogger( __name__ ) + + +DEFAULT_MODULE_COMMAND = 'module' +DEFAULT_MODULE_DIRECTORY = '/usr/share/modules/modulefiles' +DEFAULT_INDICATOR = '(default)' +DEFAULT_MODULE_PREFETCH = "true" +UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find modules by [%s], find_by should be one of %s" + + +class ModuleDependencyResolver(DependencyResolver): + + def __init__(self, dependency_manager, **kwds): + self.module_command = kwds.get('command', DEFAULT_MODULE_COMMAND) + self.versionless = string_as_bool(kwds.get('versionless', 'false')) + find_by = kwds.get('find_by', 'avail') + prefetch = string_as_bool(kwds.get('prefetch', DEFAULT_MODULE_PREFETCH)) + if find_by == 'directory': + directory = kwds.get('directory', DEFAULT_MODULE_DIRECTORY) + self.module_checker = DirectoryModuleChecker(self, directory, prefetch) + elif find_by == 'avail': + self.module_checker = AvailModuleChecker(self, prefetch) + else: + raise Exception(UNKNOWN_FIND_BY_MESSAGE % (find_by, ["avail", "directory"])) + + def resolve( self, name, version, type, **kwds ): + if type != "package": + return INDETERMINATE_DEPENDENCY + + if self.versionless: + version = None + + if self.__has_module(name, version): + return ModuleDependency(self, name, version) + + return INDETERMINATE_DEPENDENCY + + def __has_module(self, name, version): + return self.module_checker.has_module(name, version) + + +class DirectoryModuleChecker(object): + + def __init__(self, module_dependency_resolver, directory, prefetch): + self.module_dependency_resolver = module_dependency_resolver + self.directory = directory + if prefetch: + log.warn("Created module dependency resolver with prefetch enabled, but directory module checker does not support this.") + pass + + def has_module(self, module, version): + module_directory = join(self.directory, module) + has_module_directory = isdir( join( self.directory, module ) ) + if not version: + has_module = has_module_directory + else: + modulefile = join( module_directory, version ) + has_modulefile = exists( modulefile ) + has_module = has_module_directory and has_modulefile + return has_module + + +class AvailModuleChecker(object): + + def __init__(self, module_dependency_resolver, prefetch): + self.module_dependency_resolver = module_dependency_resolver + if prefetch: + prefetched_modules = [] + for module in self.__modules(): + prefetched_modules.append(module) + else: + prefetched_modules = None + self.prefetched_modules = prefetched_modules + + def has_module(self, module, version): + module_generator = self.prefetched_modules + if module_generator is None: + module_generator = self.__modules() + + for module_name, module_version in module_generator: + names_match = module == module_name + module_match = names_match and (version == None or module_version == version) + if module_match: + return True + return False + + def __modules(self): + raw_output = self.__module_avail_ouptut() + for line in StringIO(raw_output): + line = line and line.strip() + if not line or line.startswith("-"): + continue + + line_modules = line.split() + for module in line_modules: + if module.endswith(DEFAULT_INDICATOR): + module = module[0:-len(DEFAULT_INDICATOR)].strip() + module_parts = module.split('/') + module_version = None + if len(module_parts) == 2: + module_version = module_parts[1] + module_name = module_parts[0] + yield module_name, module_version + + def __module_avail_ouptut(self): + avail_command = '%s avail' % self.module_dependency_resolver.module_command + return Popen([avail_command], shell=True, stderr=PIPE).communicate()[1] + + +class ModuleDependency(Dependency): + + def __init__(self, module_dependency_resolver, module_name, module_version=None): + self.module_dependency_resolver = module_dependency_resolver + self.module_name = module_name + self.module_version = module_version + + def shell_commands(self, requirement): + command = '%s load %s' % (self.module_dependency_resolver.module_command, self.module_name) + if self.module_version: + command = '%s/%s' % self.module_version + return command diff -r 21b9e99a73fe96721498dda9751680741f78e946 -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -1,9 +1,11 @@ import tempfile import os.path -from os import makedirs, symlink +from stat import S_IXUSR +from os import makedirs, symlink, stat, chmod from shutil import rmtree from galaxy.tools.deps import DependencyManager, INDETERMINATE_DEPENDENCY from galaxy.tools.deps.resolvers.galaxy_packages import GalaxyPackageDependency +from galaxy.tools.deps.resolvers.modules import ModuleDependencyResolver from galaxy.util.bunch import Bunch from contextlib import contextmanager from subprocess import Popen, PIPE @@ -100,6 +102,50 @@ return package_dir +def test_module_dependency_resolver(): + with __test_base_path() as temp_directory: + module_script = os.path.join(temp_directory, "module") + with open(module_script, 'w') as f: + f.write('''#!/bin/sh +cat %s/example_output 1>&2; +''' % temp_directory) + with open(os.path.join(temp_directory, "example_output"), "w") as f: + # Subset of module avail from MSI cluster. + f.write(''' +-------------------------- /soft/modules/modulefiles --------------------------- +JAGS/3.2.0-gcc45 +JAGS/3.3.0-gcc4.7.2 +ProbABEL/0.1-3 +ProbABEL/0.1-9e +R/2.12.2 +R/2.13.1 +R/2.14.1 +R/2.15.0 +R/2.15.1 +R/3.0.1(default) +abokia-blast/2.0.2-130524/ompi_intel +abokia-blast/2.0.2-130630/ompi_intel + +--------------------------- /soft/intel/modulefiles ---------------------------- +advisor/2013/update1 intel/11.1.075 mkl/10.2.1.017 +advisor/2013/update2 intel/11.1.080 mkl/10.2.5.035 +advisor/2013/update3 intel/12.0 mkl/10.2.7.041 +''') + st = os.stat(module_script) + chmod(module_script, st.st_mode | S_IXUSR) + resolver = ModuleDependencyResolver(None, command=module_script) + module = resolver.resolve( name="R", version=None, type="package" ) + assert module.module_name == "R" + assert module.module_version == None + + module = resolver.resolve( name="R", version="3.0.1", type="package" ) + assert module.module_name == "R" + assert module.module_version == "3.0.1" + + module = resolver.resolve( name="R", version="3.0.4", type="package" ) + assert module == INDETERMINATE_DEPENDENCY + + def test_galaxy_dependency_object_script(): with __test_base_path() as base_path: ## Create env.sh file that just exports variable FOO and verify it @@ -189,11 +235,30 @@ # Unspecified base_paths are both default_base_paths assert dependency_resolvers[0].base_path == dependency_resolvers[1].base_path # Can specify custom base path... - assert dependency_resolvers[2].base_path == "/opt/galaxy/legacy/" + assert dependency_resolvers[2].base_path == "/opt/galaxy/legacy" # ... that is different from the default. assert dependency_resolvers[0].base_path != dependency_resolvers[2].base_path +def test_config_module_defaults(): + with __parse_resolvers('''<dependency_resolvers> + <modules /> +</dependency_resolvers> +''') as dependency_resolvers: + module_resolver = dependency_resolvers[0] + assert module_resolver.module_command == "module" + assert module_resolver.module_checker.__class__.__name__ == "AvailModuleChecker" + + +def test_config_module_directory_searcher(): + with __parse_resolvers('''<dependency_resolvers> + <modules find_by="directory" directory="/opt/Modules/modulefiles" /> +</dependency_resolvers> +''') as dependency_resolvers: + module_resolver = dependency_resolvers[0] + assert module_resolver.module_checker.directory == "/opt/Modules/modulefiles" + + @contextmanager def __parse_resolvers(xml_content): with __test_base_path() as base_path: https://bitbucket.org/galaxy/galaxy-central/commits/54b9d1a2b1a9/ Changeset: 54b9d1a2b1a9 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Dynamically load tool dependency resolver plugins. This largely mirrors the code for the dynamic job runner so created common util code meant to be shared between dynamic job destinations and dynamic dependency resolves out into galaxy.util.submodules. TODO: Update job mapper to use this. Affected #: 5 files diff -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -8,10 +8,11 @@ log = logging.getLogger( __name__ ) from galaxy.util import parse_xml + from .resolvers import INDETERMINATE_DEPENDENCY from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver -from .resolvers.modules import ModuleDependencyResolver +from galaxy.util.submodules import submodules class DependencyManager( object ): @@ -35,6 +36,7 @@ if not os.path.isdir( default_base_path ): log.warn( "Path '%s' is not directory, ignoring", default_base_path ) self.default_base_path = os.path.abspath( default_base_path ) + self.resolver_classes = self.__resolvers_dict() self.dependency_resolvers = self.__build_dependency_resolvers( conf_file ) @@ -68,12 +70,19 @@ for resolver_element in resolvers_element.getchildren(): resolver_type = resolver_element.tag resolver_kwds = dict(resolver_element.items()) - resolver = RESOLVER_CLASSES[resolver_type](self, **resolver_kwds) + resolver = self.resolver_classes[resolver_type](self, **resolver_kwds) resolvers.append(resolver) return resolvers -RESOLVER_CLASSES = { - 'tool_shed_packages': ToolShedPackageDependencyResolver, - 'galaxy_packages': GalaxyPackageDependencyResolver, - 'modules': ModuleDependencyResolver, -} + def __resolvers_dict( self ): + resolver_dict = {} + for resolver_module in self.__resolver_modules(): + for clazz in resolver_module.__all__: + resolver_type = getattr(clazz, 'resolver_type', None) + if resolver_type: + resolver_dict[resolver_type] = clazz + return resolver_dict + + def __resolver_modules( self ): + import galaxy.tools.deps.resolvers + return submodules( galaxy.tools.deps.resolvers ) diff -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 lib/galaxy/tools/deps/resolvers/galaxy_packages.py --- a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -8,6 +8,7 @@ class GalaxyPackageDependencyResolver(DependencyResolver): + resolver_type = "galaxy_packages" def __init__(self, dependency_manager, **kwds): ## Galaxy tool shed requires explicit versions on XML elements, diff -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 lib/galaxy/tools/deps/resolvers/modules.py --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -25,6 +25,7 @@ class ModuleDependencyResolver(DependencyResolver): + resolver_type = "modules" def __init__(self, dependency_manager, **kwds): self.module_command = kwds.get('command', DEFAULT_MODULE_COMMAND) @@ -135,3 +136,5 @@ if self.module_version: command = '%s/%s' % self.module_version return command + +__all__ = [ModuleDependencyResolver] diff -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 lib/galaxy/tools/deps/resolvers/tool_shed_packages.py --- a/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py +++ b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py @@ -5,6 +5,7 @@ class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): + resolver_type = "tool_shed_packages" def __init__(self, dependency_manager, **kwds): super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager, **kwds) diff -r 31f6c565ca43d21b12cd78ffe72509517cd9cfea -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 lib/galaxy/util/submodules.py --- /dev/null +++ b/lib/galaxy/util/submodules.py @@ -0,0 +1,30 @@ +from os import listdir +import logging +log = logging.getLogger( __name__ ) + + +def submodules( module ): + unsorted_submodule_names = __submodule_names( module ) + submodule_names = sorted( unsorted_submodule_names, reverse=True ) + submodules = [] + for submodule_name in submodule_names: + full_submodule = "%s.%s" % ( module.__name__, submodule_name ) + try: + __import__( full_submodule ) + submodule = getattr( module, submodule_name ) + submodules.append( submodule ) + except BaseException, exception: + exception_str = str( exception ) + message = "%s dynamic module could not be loaded: %s" % ( full_submodule, exception_str ) + log.debug( message ) + return submodules + + +def __submodule_names( module ): + module_dir = module.__path__[ 0 ] + names = [] + for fname in listdir( module_dir ): + if not( fname.startswith( "_" ) ) and fname.endswith( ".py" ): + submodule_name = fname[ :-len( ".py" ) ] + names.append( submodule_name ) + return names https://bitbucket.org/galaxy/galaxy-central/commits/6f54a6d1fc23/ Changeset: 6f54a6d1fc23 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Tool Dependencies: Move requirement/dependency logic out of galaxy.tools and into galaxy.tools.deps, test. Moved ToolRequirement out into its own module along with parsing. Also moved all requirements handling logic into DependencyManager. Now the Tool class knows nothing about the internals or requirements or how to process dependencies. There is also greater unit test coverage for these concepts. Affected #: 4 files diff -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 -r 6f54a6d1fc23b16e11005f166fe84e73754f6cae lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -41,6 +41,7 @@ from galaxy.tools.actions.data_source import DataSourceToolAction from galaxy.tools.actions.data_manager import DataManagerToolAction from galaxy.tools.deps import DependencyManager, INDETERMINATE_DEPENDENCY +from galaxy.tools.deps.requirements import parse_requirements_from_xml from galaxy.tools.parameters import check_param, params_from_strings, params_to_strings from galaxy.tools.parameters.basic import (BaseURLToolParameter, DataToolParameter, HiddenToolParameter, LibraryDatasetToolParameter, @@ -938,15 +939,6 @@ def __iter__( self ): return iter( ( self.format, self.metadata_source, self.parent ) ) -class ToolRequirement( object ): - """ - Represents an external requirement that must be available for the tool to run (for example, a program, package, or library). - Requirements can optionally assert a specific version. - """ - def __init__( self, name=None, type=None, version=None ): - self.name = name - self.type = type - self.version = version class Tool( object, Dictifiable ): """ @@ -1236,10 +1228,7 @@ else: self.tests = None # Requirements (dependencies) - self.requirements = [] - requirements_elem = root.find( "requirements" ) - if requirements_elem: - self.parse_requirements( requirements_elem ) + self.requirements = parse_requirements_from_xml( root ) # Determine if this tool can be used in workflows self.is_workflow_compatible = self.check_workflow_compatible(root) # Trackster configuration. @@ -1810,17 +1799,6 @@ for name in param.get_dependencies(): context[ name ].refresh_on_change = True return param - def parse_requirements( self, requirements_elem ): - """ - Parse each requirement from the <requirements> element and add to - self.requirements - """ - for requirement_elem in requirements_elem.findall( 'requirement' ): - name = xml_text( requirement_elem ) - type = requirement_elem.get( "type", "package" ) - version = requirement_elem.get( "version", None ) - requirement = ToolRequirement( name=name, type=type, version=version ) - self.requirements.append( requirement ) def populate_tool_shed_info( self ): if self.repository_id is not None and 'ToolShedRepository' in self.app.model: @@ -2672,27 +2650,16 @@ command_line = command_line.replace(executable, abs_executable, 1) command_line = self.interpreter + " " + command_line return command_line + def build_dependency_shell_commands( self ): """Return a list of commands to be run to populate the current environment to include this tools requirements.""" - commands = [] if self.tool_shed_repository: installed_tool_dependencies = self.tool_shed_repository.installed_tool_dependencies else: installed_tool_dependencies = None - for requirement in self.requirements: - log.debug( "Building dependency shell command for dependency '%s'", requirement.name ) - dependency = INDETERMINATE_DEPENDENCY - if requirement.type in [ 'package', 'set_environment' ]: - dependency = self.app.toolbox.dependency_manager.find_dep( name=requirement.name, - version=requirement.version, - type=requirement.type, - installed_tool_dependencies=installed_tool_dependencies ) - dependency_commands = dependency.shell_commands( requirement ) - if not dependency_commands: - log.warn( "Failed to resolve dependency on '%s', ignoring", requirement.name ) - else: - commands.append(dependency_commands) - return commands + return self.app.toolbox.dependency_manager.dependency_shell_commands( self.requirements, + installed_tool_dependencies=installed_tool_dependencies ) + def build_redirect_url_params( self, param_dict ): """ Substitute parameter values into self.redirect_url_params diff -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 -r 6f54a6d1fc23b16e11005f166fe84e73754f6cae lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -39,6 +39,22 @@ self.resolver_classes = self.__resolvers_dict() self.dependency_resolvers = self.__build_dependency_resolvers( conf_file ) + def dependency_shell_commands( self, requirements, **kwds ): + commands = [] + for requirement in requirements: + log.debug( "Building dependency shell command for dependency '%s'", requirement.name ) + dependency = INDETERMINATE_DEPENDENCY + if requirement.type in [ 'package', 'set_environment' ]: + dependency = self.find_dep( name=requirement.name, + version=requirement.version, + type=requirement.type, + **kwds ) + dependency_commands = dependency.shell_commands( requirement ) + if not dependency_commands: + log.warn( "Failed to resolve dependency on '%s', ignoring", requirement.name ) + else: + commands.append( dependency_commands ) + return commands def find_dep( self, name, version=None, type='package', **kwds ): for resolver in self.dependency_resolvers: diff -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 -r 6f54a6d1fc23b16e11005f166fe84e73754f6cae lib/galaxy/tools/deps/requirements.py --- /dev/null +++ b/lib/galaxy/tools/deps/requirements.py @@ -0,0 +1,57 @@ +from galaxy.util import xml_text + +DEFAULT_REQUIREMENT_TYPE = "package" +DEFAULT_REQUIREMENT_VERSION = None + + +class ToolRequirement( object ): + """ + Represents an external requirement that must be available for the tool to + run (for example, a program, package, or library). Requirements can + optionally assert a specific version. + """ + def __init__( self, name=None, type=None, version=None ): + self.name = name + self.type = type + self.version = version + + +def parse_requirements_from_xml( xml_root ): + """ + + >>> from galaxy.util import parse_xml + >>> from elementtree import ElementTree + >>> def load_requirements( contents ): + ... contents_document = '''<tool><requirements>%s</requirements></tool>''' + ... root = ElementTree.fromstring( contents_document % contents ) + ... return parse_requirements_from_xml( root ) + >>> reqs = load_requirements('''<requirement>bwa</requirement>''') + >>> reqs[0].name + 'bwa' + >>> reqs[0].version is None + True + >>> reqs[0].type + 'package' + >>> reqs = load_requirements('''<requirement type="binary" version="1.3.3">cufflinks</requirement>''') + >>> reqs[0].name + 'cufflinks' + >>> reqs[0].version + '1.3.3' + >>> reqs[0].type + 'binary' + """ + requirements_elem = xml_root.find( "requirements" ) + + requirement_elems = [] + if requirements_elem: + requirement_elems = requirements_elem.findall( 'requirement' ) + + requirements = [] + for requirement_elem in requirement_elems: + name = xml_text( requirement_elem ) + type = requirement_elem.get( "type", DEFAULT_REQUIREMENT_TYPE ) + version = requirement_elem.get( "version", DEFAULT_REQUIREMENT_VERSION ) + requirement = ToolRequirement( name=name, type=type, version=version ) + requirements.append( requirement ) + + return requirements diff -r 54b9d1a2b1a971274c2d02f6856c4cd08b9edf88 -r 6f54a6d1fc23b16e11005f166fe84e73754f6cae test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -152,10 +152,25 @@ ## shell_commands export it correctly. env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION, "export FOO=\"bar\"") dependency = GalaxyPackageDependency(env_path, os.path.dirname(env_path), TEST_VERSION) - command = ["bash", "-c", "%s; echo \"$FOO\"" % dependency.shell_commands(Bunch(type="package"))] - process = Popen(command, stdout=PIPE) - output = process.communicate()[0].strip() - assert output == 'bar' + __assert_foo_exported( dependency.shell_commands( Bunch( type="package" ) ) ) + + +def test_shell_commands_built(): + ## Test that dependency manager builds valid shell commands for a list of + ## requirements. + with __test_base_path() as base_path: + dm = DependencyManager( default_base_path=base_path ) + __setup_galaxy_package_dep( base_path, TEST_REPO_NAME, TEST_VERSION, contents="export FOO=\"bar\"" ) + mock_requirements = [ Bunch(type="package", version=TEST_VERSION, name=TEST_REPO_NAME ) ] + commands = dm.dependency_shell_commands( mock_requirements ) + __assert_foo_exported( commands ) + + +def __assert_foo_exported( commands ): + command = ["bash", "-c", "%s; echo \"$FOO\"" % "".join(commands)] + process = Popen(command, stdout=PIPE) + output = process.communicate()[0].strip() + assert output == 'bar' def __setup_galaxy_package_dep(base_path, name, version, contents=""): https://bitbucket.org/galaxy/galaxy-central/commits/dc9c0e065be0/ Changeset: dc9c0e065be0 User: sjguest Date: 2013-10-17 06:15:21 Summary: Fixed ModuleDependencyResolver, versionless fallback, and modulepath Affected #: 1 file diff -r 6f54a6d1fc23b16e11005f166fe84e73754f6cae -r dc9c0e065be01f8d16b35eec7b53098fdfa0fd05 lib/galaxy/tools/deps/resolvers/modules.py --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -6,6 +6,7 @@ it, hence support for it will be minimal. The Galaxy team eagerly welcomes community contribution and maintenance however. """ +from os import environ from os.path import exists, isdir, join from StringIO import StringIO from subprocess import Popen, PIPE @@ -17,8 +18,12 @@ log = logging.getLogger( __name__ ) -DEFAULT_MODULE_COMMAND = 'module' -DEFAULT_MODULE_DIRECTORY = '/usr/share/modules/modulefiles' +if environ.has_key('MODULEPATH'): + DEFAULT_MODULE_PATH = environ['MODULEPATH'] +elif environ.has_key('MODULESHOME'): + DEFAULT_MODULE_PATH = join(environ['MODULESHOME'], 'modulefiles') +else: + DEFAULT_MODULE_PATH = '/usr/share/modules/modulefiles' DEFAULT_INDICATOR = '(default)' DEFAULT_MODULE_PREFETCH = "true" UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find modules by [%s], find_by should be one of %s" @@ -28,13 +33,12 @@ resolver_type = "modules" def __init__(self, dependency_manager, **kwds): - self.module_command = kwds.get('command', DEFAULT_MODULE_COMMAND) self.versionless = string_as_bool(kwds.get('versionless', 'false')) find_by = kwds.get('find_by', 'avail') prefetch = string_as_bool(kwds.get('prefetch', DEFAULT_MODULE_PREFETCH)) if find_by == 'directory': - directory = kwds.get('directory', DEFAULT_MODULE_DIRECTORY) - self.module_checker = DirectoryModuleChecker(self, directory, prefetch) + modulepath = kwds.get('modulepath', DEFAULT_MODULE_PATH) + self.module_checker = DirectoryModuleChecker(self, modulepath, prefetch) elif find_by == 'avail': self.module_checker = AvailModuleChecker(self, prefetch) else: @@ -44,11 +48,10 @@ if type != "package": return INDETERMINATE_DEPENDENCY - if self.versionless: - version = None - if self.__has_module(name, version): return ModuleDependency(self, name, version) + elif self.versionless and self.__has_module(name, None): + return ModuleDependency(self, name, None) return INDETERMINATE_DEPENDENCY @@ -58,23 +61,26 @@ class DirectoryModuleChecker(object): - def __init__(self, module_dependency_resolver, directory, prefetch): + def __init__(self, module_dependency_resolver, modulepath, prefetch): self.module_dependency_resolver = module_dependency_resolver - self.directory = directory + self.directories = modulepath.split(':') if prefetch: log.warn("Created module dependency resolver with prefetch enabled, but directory module checker does not support this.") pass def has_module(self, module, version): - module_directory = join(self.directory, module) - has_module_directory = isdir( join( self.directory, module ) ) - if not version: - has_module = has_module_directory - else: - modulefile = join( module_directory, version ) - has_modulefile = exists( modulefile ) - has_module = has_module_directory and has_modulefile - return has_module + for directory in self.directories: + module_directory = join(directory, module) + has_module_directory = isdir( module_directory ) + if not version: + has_module = has_module_directory or exists(module_directory) # could be a bare modulefile + else: + modulefile = join( module_directory, version ) + has_modulefile = exists( modulefile ) + has_module = has_module_directory and has_modulefile + if has_module: + return True + return False class AvailModuleChecker(object): @@ -102,7 +108,7 @@ return False def __modules(self): - raw_output = self.__module_avail_ouptut() + raw_output = self.__module_avail_output() for line in StringIO(raw_output): line = line and line.strip() if not line or line.startswith("-"): @@ -119,10 +125,9 @@ module_name = module_parts[0] yield module_name, module_version - def __module_avail_ouptut(self): - avail_command = '%s avail' % self.module_dependency_resolver.module_command - return Popen([avail_command], shell=True, stderr=PIPE).communicate()[1] - + def __module_avail_output(self): + avail_command = ['modulecmd', 'sh', 'avail'] + return Popen(avail_command, stderr=PIPE).communicate()[1] class ModuleDependency(Dependency): @@ -132,9 +137,10 @@ self.module_version = module_version def shell_commands(self, requirement): - command = '%s load %s' % (self.module_dependency_resolver.module_command, self.module_name) + module_to_load = self.module_name if self.module_version: - command = '%s/%s' % self.module_version + module_to_load = '%s/%s' % (self.module_name, self.module_version) + command = 'eval `modulecmd sh load %s`' % (module_to_load) return command __all__ = [ModuleDependencyResolver] https://bitbucket.org/galaxy/galaxy-central/commits/93286e74e18b/ Changeset: 93286e74e18b User: jmchilton Date: 2013-10-17 06:15:21 Summary: Small touch ups to module code enhancements. Affected #: 1 file diff -r dc9c0e065be01f8d16b35eec7b53098fdfa0fd05 -r 93286e74e18b7dcb84aa11a326f08d5b9398a90d lib/galaxy/tools/deps/resolvers/modules.py --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -6,7 +6,7 @@ it, hence support for it will be minimal. The Galaxy team eagerly welcomes community contribution and maintenance however. """ -from os import environ +from os import environ, pathsep from os.path import exists, isdir, join from StringIO import StringIO from subprocess import Popen, PIPE @@ -18,12 +18,7 @@ log = logging.getLogger( __name__ ) -if environ.has_key('MODULEPATH'): - DEFAULT_MODULE_PATH = environ['MODULEPATH'] -elif environ.has_key('MODULESHOME'): - DEFAULT_MODULE_PATH = join(environ['MODULESHOME'], 'modulefiles') -else: - DEFAULT_MODULE_PATH = '/usr/share/modules/modulefiles' +DEFAULT_MODULE_PATH = '/usr/share/modules/modulefiles' DEFAULT_INDICATOR = '(default)' DEFAULT_MODULE_PREFETCH = "true" UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find modules by [%s], find_by should be one of %s" @@ -44,6 +39,15 @@ else: raise Exception(UNKNOWN_FIND_BY_MESSAGE % (find_by, ["avail", "directory"])) + def __default_modulespath(self): + if 'MODULEPATH' in environ: + module_path = environ['MODULEPATH'] + elif 'MODULESHOME' in environ: + module_path = join(environ['MODULESHOME'], 'modulefiles') + else: + module_path = DEFAULT_MODULE_PATH + return module_path + def resolve( self, name, version, type, **kwds ): if type != "package": return INDETERMINATE_DEPENDENCY @@ -63,24 +67,25 @@ def __init__(self, module_dependency_resolver, modulepath, prefetch): self.module_dependency_resolver = module_dependency_resolver - self.directories = modulepath.split(':') + self.directories = modulepath.split(pathsep) if prefetch: log.warn("Created module dependency resolver with prefetch enabled, but directory module checker does not support this.") pass def has_module(self, module, version): + has_module = False for directory in self.directories: module_directory = join(directory, module) has_module_directory = isdir( module_directory ) if not version: - has_module = has_module_directory or exists(module_directory) # could be a bare modulefile + has_module = has_module_directory or exists(module_directory) # could be a bare modulefile else: modulefile = join( module_directory, version ) has_modulefile = exists( modulefile ) has_module = has_module_directory and has_modulefile if has_module: - return True - return False + break + return has_module class AvailModuleChecker(object): @@ -129,6 +134,7 @@ avail_command = ['modulecmd', 'sh', 'avail'] return Popen(avail_command, stderr=PIPE).communicate()[1] + class ModuleDependency(Dependency): def __init__(self, module_dependency_resolver, module_name, module_version=None): https://bitbucket.org/galaxy/galaxy-central/commits/66222d30c816/ Changeset: 66222d30c816 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Add unit tests for module behavior contributed by Simon Guest. Affected #: 2 files diff -r 93286e74e18b7dcb84aa11a326f08d5b9398a90d -r 66222d30c81606b57d09e195365707c9f61b2946 lib/galaxy/tools/deps/resolvers/modules.py --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -17,7 +17,7 @@ import logging log = logging.getLogger( __name__ ) - +DEFAULT_MODULECMD_PATH = "modulecmd" # Just check path DEFAULT_MODULE_PATH = '/usr/share/modules/modulefiles' DEFAULT_INDICATOR = '(default)' DEFAULT_MODULE_PREFETCH = "true" @@ -31,8 +31,9 @@ self.versionless = string_as_bool(kwds.get('versionless', 'false')) find_by = kwds.get('find_by', 'avail') prefetch = string_as_bool(kwds.get('prefetch', DEFAULT_MODULE_PREFETCH)) + self.modulecmd = kwds.get('modulecmd', DEFAULT_MODULECMD_PATH) if find_by == 'directory': - modulepath = kwds.get('modulepath', DEFAULT_MODULE_PATH) + modulepath = kwds.get('modulepath', self.__default_modulespath()) self.module_checker = DirectoryModuleChecker(self, modulepath, prefetch) elif find_by == 'avail': self.module_checker = AvailModuleChecker(self, prefetch) @@ -131,7 +132,7 @@ yield module_name, module_version def __module_avail_output(self): - avail_command = ['modulecmd', 'sh', 'avail'] + avail_command = [self.module_dependency_resolver.modulecmd, 'sh', 'avail'] return Popen(avail_command, stderr=PIPE).communicate()[1] @@ -146,7 +147,7 @@ module_to_load = self.module_name if self.module_version: module_to_load = '%s/%s' % (self.module_name, self.module_version) - command = 'eval `modulecmd sh load %s`' % (module_to_load) + command = 'eval `%s sh load %s`' % (self.module_dependency_resolver.modulecmd, module_to_load) return command __all__ = [ModuleDependencyResolver] diff -r 93286e74e18b7dcb84aa11a326f08d5b9398a90d -r 66222d30c81606b57d09e195365707c9f61b2946 test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -1,11 +1,11 @@ import tempfile import os.path from stat import S_IXUSR -from os import makedirs, symlink, stat, chmod +from os import makedirs, stat, symlink, chmod, environ from shutil import rmtree from galaxy.tools.deps import DependencyManager, INDETERMINATE_DEPENDENCY from galaxy.tools.deps.resolvers.galaxy_packages import GalaxyPackageDependency -from galaxy.tools.deps.resolvers.modules import ModuleDependencyResolver +from galaxy.tools.deps.resolvers.modules import ModuleDependencyResolver, ModuleDependency from galaxy.util.bunch import Bunch from contextlib import contextmanager from subprocess import Popen, PIPE @@ -92,7 +92,7 @@ gx_env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION) ts_env_path = os.path.join(ts_package_dir, "env.sh") dependency = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) - assert dependency.script != gx_env_path # Not the galaxy path, it should be the tool shed path used. + assert dependency.script != gx_env_path # Not the galaxy path, it should be the tool shed path used. assert dependency.script == ts_env_path @@ -104,9 +104,8 @@ def test_module_dependency_resolver(): with __test_base_path() as temp_directory: - module_script = os.path.join(temp_directory, "module") - with open(module_script, 'w') as f: - f.write('''#!/bin/sh + module_script = os.path.join(temp_directory, "modulecmd") + __write_script(module_script, '''#!/bin/sh cat %s/example_output 1>&2; ''' % temp_directory) with open(os.path.join(temp_directory, "example_output"), "w") as f: @@ -131,9 +130,7 @@ advisor/2013/update2 intel/11.1.080 mkl/10.2.5.035 advisor/2013/update3 intel/12.0 mkl/10.2.7.041 ''') - st = os.stat(module_script) - chmod(module_script, st.st_mode | S_IXUSR) - resolver = ModuleDependencyResolver(None, command=module_script) + resolver = ModuleDependencyResolver(None, modulecmd=module_script) module = resolver.resolve( name="R", version=None, type="package" ) assert module.module_name == "R" assert module.module_version == None @@ -146,6 +143,31 @@ assert module == INDETERMINATE_DEPENDENCY +def test_module_dependency(): + with __test_base_path() as temp_directory: + ## Create mock modulecmd script that just exports a variable + ## the way modulecmd sh load would, but also validate correct + ## module name and version are coming through. + mock_modulecmd = os.path.join(temp_directory, 'modulecmd') + __write_script(mock_modulecmd, '''#!/bin/sh +if [ $3 != "foomodule/1.0" ]; +then + exit 1 +fi +echo 'FOO="bar"' +''') + resolver = Bunch(modulecmd=mock_modulecmd) + dependency = ModuleDependency(resolver, "foomodule", "1.0") + __assert_foo_exported( dependency.shell_commands( Bunch( type="package" ) ) ) + + +def __write_script(path, contents): + with open(path, 'w') as f: + f.write(contents) + st = stat(path) + chmod(path, st.st_mode | S_IXUSR) + + def test_galaxy_dependency_object_script(): with __test_base_path() as base_path: ## Create env.sh file that just exports variable FOO and verify it @@ -170,7 +192,7 @@ command = ["bash", "-c", "%s; echo \"$FOO\"" % "".join(commands)] process = Popen(command, stdout=PIPE) output = process.communicate()[0].strip() - assert output == 'bar' + assert output == 'bar', "Command %s exports FOO as %s, not bar" % (command, output) def __setup_galaxy_package_dep(base_path, name, version, contents=""): @@ -261,17 +283,68 @@ </dependency_resolvers> ''') as dependency_resolvers: module_resolver = dependency_resolvers[0] - assert module_resolver.module_command == "module" assert module_resolver.module_checker.__class__.__name__ == "AvailModuleChecker" +def test_config_modulepath(): + # Test reads and splits MODULEPATH if modulepath is not specified. + with __parse_resolvers('''<dependency_resolvers> + <modules find_by="directory" modulepath="/opt/modules/modulefiles:/usr/local/modules/modulefiles" /> +</dependency_resolvers> +''') as dependency_resolvers: + assert dependency_resolvers[0].module_checker.directories == ["/opt/modules/modulefiles", "/usr/local/modules/modulefiles"] + + +def test_config_MODULEPATH(): + # Test reads and splits MODULEPATH if modulepath is not specified. + with __environ({"MODULEPATH": "/opt/modules/modulefiles:/usr/local/modules/modulefiles"}): + with __parse_resolvers('''<dependency_resolvers> + <modules find_by="directory" /> +</dependency_resolvers> +''') as dependency_resolvers: + assert dependency_resolvers[0].module_checker.directories == ["/opt/modules/modulefiles", "/usr/local/modules/modulefiles"] + + +def test_config_MODULESHOME(): + # Test fallbacks to read MODULESHOME if modulepath is not specified and + # neither is MODULEPATH. + with __environ({"MODULESHOME": "/opt/modules"}, remove="MODULEPATH"): + with __parse_resolvers('''<dependency_resolvers> + <modules find_by="directory" /> +</dependency_resolvers> +''') as dependency_resolvers: + assert dependency_resolvers[0].module_checker.directories == ["/opt/modules/modulefiles"] + + def test_config_module_directory_searcher(): with __parse_resolvers('''<dependency_resolvers> - <modules find_by="directory" directory="/opt/Modules/modulefiles" /> + <modules find_by="directory" modulepath="/opt/Modules/modulefiles" /></dependency_resolvers> ''') as dependency_resolvers: module_resolver = dependency_resolvers[0] - assert module_resolver.module_checker.directory == "/opt/Modules/modulefiles" + assert module_resolver.module_checker.directories == ["/opt/Modules/modulefiles"] + + +@contextmanager +def __environ(values, remove=[]): + """ + Modify the environment for a test, adding/updating values in dict `values` and + removing any environment variables mentioned in list `remove`. + """ + new_keys = set(environ.keys()) - set(values.keys()) + old_environ = environ.copy() + try: + environ.update(values) + for to_remove in remove: + try: + del environ[remove] + except KeyError: + pass + yield + finally: + environ.update(old_environ) + for key in new_keys: + del environ[key] @contextmanager @@ -282,4 +355,3 @@ f.flush() dm = DependencyManager( default_base_path=base_path, conf_file=f.name ) yield dm.dependency_resolvers - https://bitbucket.org/galaxy/galaxy-central/commits/a96f6548bfc3/ Changeset: a96f6548bfc3 User: jmchilton Date: 2013-10-17 06:15:21 Summary: Add method to DependencyManager determining if tool shed dependencies used. Affected #: 2 files diff -r 66222d30c81606b57d09e195365707c9f61b2946 -r a96f6548bfc3f223f93a8b619e0ad2d4634a3f3f lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -56,6 +56,9 @@ commands.append( dependency_commands ) return commands + def uses_tool_shed_dependencies(self): + return any( map( lambda r: isinstance( r, ToolShedPackageDependencyResolver ), self.dependency_resolvers ) ) + def find_dep( self, name, version=None, type='package', **kwds ): for resolver in self.dependency_resolvers: dependency = resolver.resolve( name, version, type, **kwds ) diff -r 66222d30c81606b57d09e195365707c9f61b2946 -r a96f6548bfc3f223f93a8b619e0ad2d4634a3f3f test/unit/test_tool_deps.py --- a/test/unit/test_tool_deps.py +++ b/test/unit/test_tool_deps.py @@ -277,6 +277,20 @@ assert dependency_resolvers[0].base_path != dependency_resolvers[2].base_path +def test_uses_tool_shed_dependencies(): + with __dependency_manager('''<dependency_resolvers> + <galaxy_packages /> +</dependency_resolvers> +''') as dm: + assert not dm.uses_tool_shed_dependencies() + + with __dependency_manager('''<dependency_resolvers> + <tool_shed_packages /> +</dependency_resolvers> +''') as dm: + assert dm.uses_tool_shed_dependencies() + + def test_config_module_defaults(): with __parse_resolvers('''<dependency_resolvers><modules /> @@ -349,9 +363,15 @@ @contextmanager def __parse_resolvers(xml_content): + with __dependency_manager(xml_content) as dm: + yield dm.dependency_resolvers + + +@contextmanager +def __dependency_manager(xml_content): with __test_base_path() as base_path: f = tempfile.NamedTemporaryFile() f.write(xml_content) f.flush() dm = DependencyManager( default_base_path=base_path, conf_file=f.name ) - yield dm.dependency_resolvers + yield dm https://bitbucket.org/galaxy/galaxy-central/commits/8e001dc9675c/ Changeset: 8e001dc9675c User: jmchilton Date: 2013-10-17 06:15:21 Summary: Dependencies: Do not attempt to install package dependencies in tool shed if the will be ignored. That is if dependency resolvers are configured to ignore tool shed dependencies as outlined by Simon Guest, though stopping short of marking these dependencies as installed - they will still be marked as errors for now. TODO: Determine what to do with dependencies that can be resolved (just mark as installed? a brand new state? mark as never installed?). Affected #: 1 file diff -r a96f6548bfc3f223f93a8b619e0ad2d4634a3f3f -r 8e001dc9675c105d83d83f6d5a756c11759e8bb2 lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -6,6 +6,7 @@ from galaxy import util from galaxy import web from galaxy.util import json +from galaxy.tools.deps.resolvers import INDETERMINATE_DEPENDENCY import tool_shed.util.shed_util_common as suc from tool_shed.util import common_util from tool_shed.util import container_util @@ -457,7 +458,19 @@ tool_dependency = tool_dependencies[ index ] if tool_dependency.can_install: try: - tool_dependency = install_package( app, elem, tool_shed_repository, tool_dependencies=tool_dependencies ) + dependencies_ignored = app.toolbox.dependency_manager and not app.toolbox.dependency_manager.uses_tool_shed_dependencies() + if dependencies_ignored: + log.info("Skipping package %s, tool shed dependency resolver not enabled." % package_name) + # Tool dependency resolves have been configured and they do not + # include the tool shed. Do not install package. + status = app.model.ToolDependency.installation_status.ERROR + if app.toolbox.dependency_manager.find_dep( package_name, package_version, type='package') != INDETERMINATE_DEPENDENCY: + ## TODO: Do something here such as marking it installed or + ## configured externally. + pass + tool_dependency.status = status + else: + tool_dependency = install_package( app, elem, tool_shed_repository, tool_dependencies=tool_dependencies ) except Exception, e: error_message = "Error installing tool dependency %s version %s: %s" % ( str( package_name ), str( package_version ), str( e ) ) log.exception( error_message ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Tool shed install move_file bug fix (divergence between 67be847 and 8f2b113).
by commits-noreply＠bitbucket.org 17 Oct '13

17 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ebd092733e55/ Changeset: ebd092733e55 User: jmchilton Date: 2013-10-17 06:00:55 Summary: Tool shed install move_file bug fix (divergence between 67be847 and 8f2b113). Affected #: 1 file diff -r 7c05567cdc1b9d1c08e59c2927fb9e85009262e4 -r ebd092733e552ef6a846d55b69d22dc779dfcc5f lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -560,8 +560,8 @@ # <source>misc/some_file</source> # <destination>$INSTALL_DIR/bin</destination> # </action> - action_dict[ 'source' ] = evaluate_template( action_elem.find( 'source' ).text ) - action_dict[ 'destination' ] = evaluate_template( action_elem.find( 'destination' ).text ) + action_dict[ 'source' ] = td_common_util.evaluate_template( action_elem.find( 'source' ).text, install_dir ) + action_dict[ 'destination' ] = td_common_util.evaluate_template( action_elem.find( 'destination' ).text, install_dir ) action_dict[ 'rename_to' ] = action_elem.get( 'rename_to' ) elif action_type == 'set_environment': # <action type="set_environment"> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 17 Oct '13

17 Oct '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/913f3ab84629/ Changeset: 913f3ab84629 Branch: abrenner/histogram2xml-python-module-rpy-requires-1379010965302 User: abrenner Date: 2013-09-12 20:36:20 Summary: histogram2.xml python module rpy requires R package. Affected #: 1 file diff -r b9505d30f85f6e5e49640c6428185ab427899ec4 -r 913f3ab84629b1c2dff39a8788e1e0201f82ecaa tools/plotting/histogram2.xml --- a/tools/plotting/histogram2.xml +++ b/tools/plotting/histogram2.xml @@ -27,6 +27,7 @@ </tests><requirements><requirement type="python-module">rpy</requirement> + <requirement type="package">R</requirement></requirements><help> https://bitbucket.org/galaxy/galaxy-central/commits/f3e736fe03df/ Changeset: f3e736fe03df User: abrenner Date: 2013-09-12 20:39:39 Summary: Merged in abrenner/histogram2xml-python-module-rpy-requires-1379010965302 (pull request #1) histogram2.xml python module rpy requires R package. Affected #: 1 file diff -r b9505d30f85f6e5e49640c6428185ab427899ec4 -r f3e736fe03df3a6dd5438c12ba35ea791a1eaca9 tools/plotting/histogram2.xml --- a/tools/plotting/histogram2.xml +++ b/tools/plotting/histogram2.xml @@ -27,6 +27,7 @@ </tests><requirements><requirement type="python-module">rpy</requirement> + <requirement type="package">R</requirement></requirements><help> https://bitbucket.org/galaxy/galaxy-central/commits/7c05567cdc1b/ Changeset: 7c05567cdc1b User: jmchilton Date: 2013-10-17 05:12:40 Summary: Merged in abrenner/galaxy-central (pull request #215) histogram2 - RPY requires R - missing from requirements tag Thanks for the contribution! Affected #: 1 file diff -r 3825c8d493649a72c6fc65bdd06b2362f0bf5930 -r 7c05567cdc1b9d1c08e59c2927fb9e85009262e4 tools/plotting/histogram2.xml --- a/tools/plotting/histogram2.xml +++ b/tools/plotting/histogram2.xml @@ -27,6 +27,7 @@ </tests><requirements><requirement type="python-module">rpy</requirement> + <requirement type="package">R</requirement></requirements><help> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 6 new changesets
by commits-noreply＠bitbucket.org 16 Oct '13

16 Oct '13

6 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/432999eabbaa/ Changeset: 432999eabbaa User: jmchilton Date: 2013-10-17 00:37:37 Summary: Add job_script module from LWR including GALAXY_SLOTS logic. Use new job_script module in the DRMAA runner. This implements the long discussed GALAXY_SLOTS logic for PBS/TORQUE, SLURM, and grid engine when coming through the DRMAA runner. Affected #: 6 files diff -r b681b2aeddbc52c3d953abcb431688df6fb8a005 -r 432999eabbaa97528a2bd37f1b99de86b39a924f lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -15,6 +15,7 @@ from galaxy.jobs.command_factory import build_command from galaxy import model from galaxy.util import DATABASE_MAX_STRING_SIZE, shrink_stream_by_size +from galaxy.jobs.runners.util.job_script import job_script log = logging.getLogger( __name__ ) @@ -218,6 +219,16 @@ external_metadata_proc.wait() log.debug( 'execution of external set_meta for job %d finished' % job_wrapper.job_id ) + def get_job_file(self, job_wrapper, **kwds): + options = dict( + galaxy_lib=job_wrapper.galaxy_lib_dir, + env_setup_commands=job_wrapper.get_env_setup_clause(), + working_directory=os.path.abspath( job_wrapper.working_directory ), + command=job_wrapper.runner_command_line, + ) + options.update(**kwds) + return job_script(**options) + class AsynchronousJobState( object ): """ diff -r b681b2aeddbc52c3d953abcb431688df6fb8a005 -r 432999eabbaa97528a2bd37f1b99de86b39a924f lib/galaxy/jobs/runners/drmaa.py --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -37,29 +37,10 @@ drmaa.JobState.FAILED: 'job finished, but failed', } -# The last four lines (following the last fi) will: -# - setup the env -# - move to the job wrapper's working directory -# - execute the command -# - take the command's exit code ($?) and write it to a file. -drm_template = """#!/bin/sh -GALAXY_LIB="%s" -if [ "$GALAXY_LIB" != "None" ]; then - if [ -n "$PYTHONPATH" ]; then - PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" - else - PYTHONPATH="$GALAXY_LIB" - fi - export PYTHONPATH -fi -%s -cd %s -%s -echo $? > %s -""" DRMAA_jobTemplate_attributes = [ 'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification', - 'jobName','email','project' ] + 'jobName', 'email', 'project' ] + class DRMAAJobRunner( AsynchronousJobRunner ): """ @@ -138,12 +119,7 @@ jt.nativeSpecification = native_spec # fill in the DRM's job run template - script = drm_template % ( job_wrapper.galaxy_lib_dir, - job_wrapper.get_env_setup_clause(), - os.path.abspath( job_wrapper.working_directory ), - command_line, - ajs.exit_code_file ) - + script = self.get_job_file(job_wrapper, exit_code_path=ajs.exit_code_file) try: fh = file( ajs.job_file, "w" ) fh.write( script ) diff -r b681b2aeddbc52c3d953abcb431688df6fb8a005 -r 432999eabbaa97528a2bd37f1b99de86b39a924f lib/galaxy/jobs/runners/util/__init__.py --- /dev/null +++ b/lib/galaxy/jobs/runners/util/__init__.py @@ -0,0 +1,9 @@ +""" +This module and its submodules contains utilities for running external +processes and interfacing with job managers. This module should contain +functionality shared between Galaxy and the LWR. +""" +try: + from galaxy.util.bunch import Bunch +except ImportError: + from lwr.util import Bunch diff -r b681b2aeddbc52c3d953abcb431688df6fb8a005 -r 432999eabbaa97528a2bd37f1b99de86b39a924f lib/galaxy/jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh --- /dev/null +++ b/lib/galaxy/jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh @@ -0,0 +1,11 @@ +export GALAXY_SLOTS_CONFIGURED="1" +if [ -n "$SLURM_JOB_NUM_NODES" ]; then + GALAXY_SLOTS="$SLURM_JOB_NUM_NODES" +elif [ -n "$NSLOTS" ]; then + GALAXY_SLOTS="$NSLOTS" +elif [ -f "$PBS_NODEFILE" ]; then + GALAXY_SLOTS=`wc -l < $PBS_NODEFILE` +else + GALAXY_SLOTS="1" + unset GALAXY_SLOTS_CONFIGURED +fi diff -r b681b2aeddbc52c3d953abcb431688df6fb8a005 -r 432999eabbaa97528a2bd37f1b99de86b39a924f lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh --- /dev/null +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -0,0 +1,17 @@ +#!/bin/sh +$headers +$slots_statement +export GALAXY_SLOTS +GALAXY_LIB="$galaxy_lib" +if [ "$GALAXY_LIB" != "None" ]; then + if [ -n "$PYTHONPATH" ]; then + PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" + else + PYTHONPATH="$GALAXY_LIB" + fi + export PYTHONPATH +fi +$env_setup_commands +cd $working_directory +$command +echo $? > $exit_code_path diff -r b681b2aeddbc52c3d953abcb431688df6fb8a005 -r 432999eabbaa97528a2bd37f1b99de86b39a924f lib/galaxy/jobs/runners/util/job_script/__init__.py --- /dev/null +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -0,0 +1,54 @@ +from string import Template +from pkg_resources import resource_string + +DEFAULT_JOB_FILE_TEMPLATE = Template( + resource_string(__name__, 'DEFAULT_JOB_FILE_TEMPLATE.sh') +) + +SLOTS_STATEMENT_CLUSTER_DEFAULT = \ + resource_string(__name__, 'CLUSTER_SLOTS_STATEMENT.sh') + +SLOTS_STATEMENT_SINGLE = """ +GALAXY_SLOTS="1" +""" + +REQUIRED_TEMPLATE_PARAMS = ['working_directory', 'command', 'exit_code_path'] +OPTIONAL_TEMPLATE_PARAMS = { + 'galaxy_lib': None, + 'headers': '', + 'env_setup_commands': '', + 'slots_statement': SLOTS_STATEMENT_CLUSTER_DEFAULT, +} + + +def job_script(template=DEFAULT_JOB_FILE_TEMPLATE, **kwds): + """ + + >>> has_exception = False + >>> try: job_script() + ... except Exception as e: has_exception = True + >>> has_exception + True + >>> script = job_script(working_directory='wd', command='uptime', exit_code_path='ec') + >>> '\\nuptime\\n' in script + True + >>> 'echo $? > ec' in script + True + >>> 'GALAXY_LIB="None"' in script + True + >>> script.startswith('#!/bin/sh\\n#PBS -test\\n') + False + >>> script = job_script(working_directory='wd', command='uptime', exit_code_path='ec', headers='#PBS -test') + >>> script.startswith('#!/bin/sh\\n#PBS -test\\n') + True + >>> script = job_script(working_directory='wd', command='uptime', exit_code_path='ec', slots_statement='GALAXY_SLOTS="$SLURM_JOB_NUM_NODES"') + >>> script.find('GALAXY_SLOTS="$SLURM_JOB_NUM_NODES"\\nexport GALAXY_SLOTS\\n') > 0 + True + """ + if any([param not in kwds for param in REQUIRED_TEMPLATE_PARAMS]): + raise Exception("Failed to create job_script, a required parameter is missing.") + template_params = OPTIONAL_TEMPLATE_PARAMS.copy() + template_params.update(**kwds) + if not isinstance(template, Template): + template = Template(template) + return template.safe_substitute(template_params) https://bitbucket.org/galaxy/galaxy-central/commits/126d5c5b9a49/ Changeset: 126d5c5b9a49 User: jmchilton Date: 2013-10-17 00:37:37 Summary: Condor job runner enhancements. Implement exit code handling and submission script parameters (job destination parameter <param id="submit_xxxx">yyyy</param> will be passed through as xxxx = yyyy. Bring in LWR helper scripts to implement most of these and clean up things. Bring in LWR external.py for parsing ID's comding from resource managers, seems like overkill for now in Galaxy because just used in condor, but in LWR also used for its version of the CLI runners (will merge those changes in down the road). Utilize job_scripts module for building script (this is what fixes exit code handling). TODO: GALAXY_SLOTS logic still not implemented, but should be easier now. Affected #: 3 files diff -r 432999eabbaa97528a2bd37f1b99de86b39a924f -r 126d5c5b9a4938909b518cb176dc47890e1c13e5 lib/galaxy/jobs/runners/condor.py --- a/lib/galaxy/jobs/runners/condor.py +++ b/lib/galaxy/jobs/runners/condor.py @@ -3,40 +3,18 @@ """ import os -import re -import sys -import time import logging -import subprocess from galaxy import model from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner - -from galaxy.util import asbool +from galaxy.jobs.runners.util.condor import submission_params, build_submit_description +from galaxy.jobs.runners.util.condor import condor_submit, condor_stop +from galaxy.jobs.runners.util.condor import summarize_condor_log log = logging.getLogger( __name__ ) __all__ = [ 'CondorJobRunner' ] -drm_template = """#!/bin/sh -GALAXY_LIB="%s" -if [ "$GALAXY_LIB" != "None" ]; then - if [ -n "$PYTHONPATH" ]; then - PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" - else - PYTHONPATH="$GALAXY_LIB" - fi - export PYTHONPATH -fi -cd %s -%s -""" - -default_query_classad = dict( - universe = 'vanilla', - getenv = 'true', - notification = 'NEVER', -) class CondorJobState( AsynchronousJobState ): def __init__( self, **kwargs ): @@ -49,21 +27,22 @@ self.user_log = None self.user_log_size = 0 + class CondorJobRunner( AsynchronousJobRunner ): """ Job runner backed by a finite pool of worker threads. FIFO scheduling """ runner_name = "CondorRunner" + def __init__( self, app, nworkers ): """Initialize this job runner and start the monitor thread""" super( CondorJobRunner, self ).__init__( app, nworkers ) self._init_monitor_thread() self._init_worker_threads() - # superclass url_to_destination is fine - condor runner does not take params - def queue_job( self, job_wrapper ): """Create job script and submit it to the DRM""" + # prepare the job if not self.prepare_job( job_wrapper, include_metadata=True ): return @@ -78,25 +57,30 @@ galaxy_id_tag = job_wrapper.get_id_tag() # get destination params - query_params = default_query_classad.copy() - query_params.update( job_destination.params ) + query_params = submission_params(**job_destination.params) # define job attributes - cjs = CondorJobState( files_dir=self.app.config.cluster_files_directory, job_wrapper=job_wrapper ) - cjs.user_log = os.path.join( self.app.config.cluster_files_directory, 'galaxy_%s.condor.log' % galaxy_id_tag ) + cjs = CondorJobState( + files_dir=self.app.config.cluster_files_directory, + job_wrapper=job_wrapper + ) + + cluster_directory = self.app.config.cluster_files_directory + cjs.user_log = os.path.join( cluster_directory, 'galaxy_%s.condor.log' % galaxy_id_tag ) cjs.register_cleanup_file_attribute( 'user_log' ) - submit_file = os.path.join( self.app.config.cluster_files_directory, 'galaxy_%s.condor.desc' % galaxy_id_tag ) + submit_file = os.path.join( cluster_directory, 'galaxy_%s.condor.desc' % galaxy_id_tag ) executable = cjs.job_file - submit_desc = [ ] - for k, v in query_params.items(): - submit_desc.append( '%s = %s' % ( k, v ) ) - submit_desc.append( 'executable = ' + executable ) - submit_desc.append( 'output = ' + cjs.output_file ) - submit_desc.append( 'error = ' + cjs.error_file ) - submit_desc.append( 'log = ' + cjs.user_log ) - submit_desc.append( 'queue' ) - script = drm_template % (job_wrapper.galaxy_lib_dir, os.path.abspath( job_wrapper.working_directory ), command_line) + build_submit_params = dict( + executable=executable, + output=cjs.output_file, + error=cjs.error_file, + user_log=cjs.user_log, + query_params=query_params, + ) + + submit_file_contents = build_submit_description(**build_submit_params) + script = self.get_job_file( job_wrapper, exit_code_path=cjs.exit_code_file ) try: fh = file( executable, "w" ) fh.write( script ) @@ -108,10 +92,7 @@ return try: - fh = file( submit_file, 'w' ) - for line in submit_desc: - fh.write( line + '\n' ) - fh.close() + open(submit_file, "w").write(submit_file_contents) except: if self.app.config.cleanup_job == "always": cjs.cleanup() @@ -132,23 +113,9 @@ log.debug( "(%s) submitting file %s" % ( galaxy_id_tag, executable ) ) log.debug( "(%s) command is: %s" % ( galaxy_id_tag, command_line ) ) - s_out = '' - external_job_id = None - try: - submit = subprocess.Popen( ( 'condor_submit', submit_file ), stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) - s_out, s_err = submit.communicate() - if submit.returncode == 0: - match = re.search( 'submitted to cluster (\\d+).', s_out ) - if match is None: - s_out = 'Failed to find job id from condor_submit' - else: - external_job_id = match.group( 1 ) - except Exception, e: - # TODO Add extra except for OSError? - s_out = str(e) - + external_job_id, message = condor_submit(submit_file) if external_job_id is None: - log.debug( "condor_submit failed for job %s: %s" % (job_wrapper.get_id_tag(), s_out) ) + log.debug( "condor_submit failed for job %s: %s" % (job_wrapper.get_id_tag(), message) ) if self.app.config.cleanup_job == "always": os.unlink( submit_file ) cjs.cleanup() @@ -177,29 +144,17 @@ new_watched = [] for cjs in self.watched: job_id = cjs.job_id - log_job_id = job_id.zfill(3) galaxy_id_tag = cjs.job_wrapper.get_id_tag() - job_running = False - job_complete = False - job_failed = False try: if os.stat( cjs.user_log ).st_size == cjs.user_log_size: new_watched.append( cjs ) continue - with open(cjs.user_log, 'r') as fh: - for line in fh: - if '001 (' + log_job_id + '.' in line: - job_running = True - if '004 (' + log_job_id + '.' in line: - job_running = False - if '007 (' + log_job_id + '.' in line: - job_running = False - if '005 (' + log_job_id + '.' in line: - job_complete = True - if '009 (' + log_job_id + '.' in line: - job_failed = True - cjs.user_log_size = fh.tell() - except Exception, e: + s1, s4, s7, s5, s9, log_size = summarize_condor_log(cjs.user_log, job_id) + job_running = s1 and not (s4 or s7) + job_complete = s5 + job_failed = s9 + cjs.user_log_size = log_size + except Exception: # so we don't kill the monitor thread log.exception( "(%s/%s) Unable to check job status" % ( galaxy_id_tag, job_id ) ) log.warning( "(%s/%s) job will now be errored" % ( galaxy_id_tag, job_id ) ) @@ -230,13 +185,10 @@ def stop_job( self, job ): """Attempts to delete a job from the DRM queue""" - try: - subprocess.check_call( ( 'condor_rm', job.job_runner_external_id ) ) - log.debug( "(%s/%s) Removed from DRM queue at user's request" % ( job.id, job.job_runner_external_id ) ) - except subprocess.CalledProcessError: - log.debug( "(%s/%s) User killed running job, but condor_rm failed" % ( job.id, job.job_runner_external_id ) ) - except Exception, e: - log.debug( "(%s/%s) User killed running job, but error encountered removing from Condor queue: %s" % ( job.id, job.job_runner_external_id, e ) ) + external_id = job.job_runner_external_id + failure_message = condor_stop(external_id) + if failure_message: + log.debug("(%s/%s). Failed to stop condor %s" % (external_id, failure_message)) def recover( self, job, job_wrapper ): """Recovers jobs stuck in the queued/running state when Galaxy started""" diff -r 432999eabbaa97528a2bd37f1b99de86b39a924f -r 126d5c5b9a4938909b518cb176dc47890e1c13e5 lib/galaxy/jobs/runners/util/condor/__init__.py --- /dev/null +++ b/lib/galaxy/jobs/runners/util/condor/__init__.py @@ -0,0 +1,114 @@ +""" +Condor helper utilities. +""" +from subprocess import Popen, PIPE, STDOUT, check_call, CalledProcessError +from ..external import parse_external_id + +DEFAULT_QUERY_CLASSAD = dict( + universe='vanilla', + getenv='true', + notification='NEVER', +) + +PROBLEM_RUNNING_CONDOR_SUBMIT = \ + "Problem encountered while running condor_submit." +PROBLEM_PARSING_EXTERNAL_ID = \ + "Failed to find job id from condor_submit" + +SUBMIT_PARAM_PREFIX = "submit_" + + +def submission_params(**kwds): + submission_params = {} + for key, value in kwds.iteritems(): + key = key.lower() + if key.startswith(SUBMIT_PARAM_PREFIX): + condor_key = key[len(SUBMIT_PARAM_PREFIX):] + submission_params[condor_key] = value + return submission_params + + +def build_submit_description(executable, output, error, user_log, query_params): + """ + Build up the contents of a condor submit description file. + + >>> submit_args = dict(executable='/path/to/script', output='o', error='e', user_log='ul') + >>> submit_args['query_params'] = dict() + >>> default_description = build_submit_description(**submit_args) + >>> assert 'executable = /path/to/script' in default_description + >>> assert 'output = o' in default_description + >>> assert 'error = e' in default_description + >>> assert 'queue' in default_description + >>> assert 'universe = vanilla' in default_description + >>> assert 'universe = standard' not in default_description + >>> submit_args['query_params'] = dict(universe='standard') + >>> std_description = build_submit_description(**submit_args) + >>> assert 'universe = vanilla' not in std_description + >>> assert 'universe = standard' in std_description + """ + all_query_params = DEFAULT_QUERY_CLASSAD.copy() + all_query_params.update(query_params) + + submit_description = [] + for key, value in all_query_params.items(): + submit_description.append('%s = %s' % (key, value)) + submit_description.append('executable = ' + executable) + submit_description.append('output = ' + output) + submit_description.append('error = ' + error) + submit_description.append('log = ' + user_log) + submit_description.append('queue') + return '\n'.join(submit_description) + + +def condor_submit(submit_file): + """ + Submit a condor job described by the given file. Parse an external id for + the submission or return None and a reason for the failure. + """ + external_id = None + try: + submit = Popen(('condor_submit', submit_file), stdout=PIPE, stderr=STDOUT) + message, _ = submit.communicate() + if submit.returncode == 0: + external_id = parse_external_id(message, type='condor') + else: + message = PROBLEM_PARSING_EXTERNAL_ID + except Exception as e: + message = str(e) + return external_id, message + + +def condor_stop(external_id): + """ + Stop running condor job and return a failure_message if this + fails. + """ + failure_message = None + try: + check_call(('condor_rm', external_id)) + except CalledProcessError: + failure_message = "condor_rm failed" + except Exception as e: + "error encountered calling condor_rm: %s" % e + return failure_message + + +def summarize_condor_log(log_file, external_id): + """ + """ + log_job_id = external_id.zfill(3) + s1 = s4 = s7 = s5 = s9 = False + with open(log_file, 'r') as log_handle: + for line in log_handle: + if '001 (' + log_job_id + '.' in line: + s1 = True + if '004 (' + log_job_id + '.' in line: + s4 = True + if '007 (' + log_job_id + '.' in line: + s7 = True + if '005 (' + log_job_id + '.' in line: + s5 = True + if '009 (' + log_job_id + '.' in line: + s9 = True + file_size = log_handle.tell() + return s1, s4, s7, s5, s9, file_size diff -r 432999eabbaa97528a2bd37f1b99de86b39a924f -r 126d5c5b9a4938909b518cb176dc47890e1c13e5 lib/galaxy/jobs/runners/util/external.py --- /dev/null +++ b/lib/galaxy/jobs/runners/util/external.py @@ -0,0 +1,37 @@ +from re import search + +EXTERNAL_ID_TYPE_ANY = None + +EXTERNAL_ID_PATTERNS = [ + ('condor', r'submitted to cluster (\d+)\.'), + ('slurm', r'Submitted batch job (\w+)'), + ('torque', r'(.+)'), # Default 'pattern' assumed by Galaxy code circa August 2013. +] + + +def parse_external_id(output, type=EXTERNAL_ID_TYPE_ANY): + """ + Attempt to parse the output of job submission commands for an external id.__doc__ + + >>> parse_external_id("12345.pbsmanager") + '12345.pbsmanager' + >>> parse_external_id('Submitted batch job 185') + '185' + >>> parse_external_id('Submitted batch job 185', type='torque') + 'Submitted batch job 185' + >>> parse_external_id('submitted to cluster 125.') + '125' + >>> parse_external_id('submitted to cluster 125.', type='slurm') + >>> + """ + external_id = None + for pattern_type, pattern in EXTERNAL_ID_PATTERNS: + if type != EXTERNAL_ID_TYPE_ANY and type != pattern_type: + continue + + match = search(pattern, output) + if match: + external_id = match.group(1) + break + + return external_id https://bitbucket.org/galaxy/galaxy-central/commits/93caaf7f9bdd/ Changeset: 93caaf7f9bdd User: jmchilton Date: 2013-10-17 00:37:37 Summary: Implement GALAXY_SLOTS logic for condor runner. Just pass through submit_request_cpus as GALAXY_SLOTS if set in job_conf.xml. Affected #: 1 file diff -r 126d5c5b9a4938909b518cb176dc47890e1c13e5 -r 93caaf7f9bdd6429a250921fa080bfe71028f8c0 lib/galaxy/jobs/runners/condor.py --- a/lib/galaxy/jobs/runners/condor.py +++ b/lib/galaxy/jobs/runners/condor.py @@ -58,6 +58,11 @@ # get destination params query_params = submission_params(**job_destination.params) + galaxy_slots = query_params.get('request_cpus', None) + if galaxy_slots: + galaxy_slots_statement = 'GALAXY_SLOTS="%s"; export GALAXY_SLOTS_CONFIGURED="1"' % galaxy_slots + else: + galaxy_slots_statement = 'GALAXY_SLOTS="1"' # define job attributes cjs = CondorJobState( @@ -80,7 +85,11 @@ ) submit_file_contents = build_submit_description(**build_submit_params) - script = self.get_job_file( job_wrapper, exit_code_path=cjs.exit_code_file ) + script = self.get_job_file( + job_wrapper, + exit_code_path=cjs.exit_code_file, + slots_statement=galaxy_slots_statement, + ) try: fh = file( executable, "w" ) fh.write( script ) https://bitbucket.org/galaxy/galaxy-central/commits/5715b876f91b/ Changeset: 5715b876f91b User: jmchilton Date: 2013-10-17 00:37:37 Summary: PBS job runner improvements (including GALAXY_SLOTS). Rework to use get_job_file, this will enable GALAXY_SLOTS logic for the PBS runner. Bring in staging fixes found in pull request #194 from CSIRO (stephen_mcmahon_). Small PEP-8 tweaks. Affected #: 1 file diff -r 93caaf7f9bdd6429a250921fa080bfe71028f8c0 -r 5715b876f91bd34d179dc9d48e7ad1a9e6baf285 lib/galaxy/jobs/runners/pbs.py --- a/lib/galaxy/jobs/runners/pbs.py +++ b/lib/galaxy/jobs/runners/pbs.py @@ -1,9 +1,10 @@ -import os, logging, threading, time, traceback +import os +import logging +import time +import traceback from datetime import timedelta -from Queue import Queue, Empty from galaxy import model -from galaxy.datatypes.data import nice_size from galaxy.util.bunch import Bunch from galaxy.util import DATABASE_MAX_STRING_SIZE, shrink_stream_by_size from galaxy.jobs import JobDestination @@ -35,42 +36,14 @@ # The last two lines execute the command and then retrieve the command's # exit code ($?) and write it to a file. -pbs_template = """#!/bin/sh -GALAXY_LIB="%s" -if [ "$GALAXY_LIB" != "None" ]; then - if [ -n "$PYTHONPATH" ]; then - export PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" - else - export PYTHONPATH="$GALAXY_LIB" - fi -fi -%s -cd %s -%s -echo $? > %s -""" - -# The last two lines execute the command and then retrieve the command's -# exit code ($?) and write it to a file. -pbs_symlink_template = """#!/bin/sh -GALAXY_LIB="%s" -if [ "$GALAXY_LIB" != "None" ]; then - if [ -n "$PYTHONPATH" ]; then - export PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" - else - export PYTHONPATH="$GALAXY_LIB" - fi -fi +pbs_symlink_template = """ for dataset in %s; do dir=`dirname $dataset` file=`basename $dataset` [ ! -d $dir ] && mkdir -p $dir [ ! -e $dataset ] && ln -s %s/$file $dataset done -%s -cd %s -%s -echo $? > %s +mkdir -p %s """ PBS_ARGMAP = { @@ -109,6 +82,7 @@ -8: "exec() of user command failed", } + class PBSJobRunner( AsynchronousJobRunner ): """ Job runner backed by a finite pool of worker threads. FIFO scheduling @@ -273,6 +247,7 @@ pbs_ofile = self.app.config.pbs_application_server + ':' + ofile pbs_efile = self.app.config.pbs_application_server + ':' + efile output_files = [ str( o ) for o in output_fnames ] + output_files.append(ecfile) stagein = self.get_stage_in_out( job_wrapper.get_input_fnames() + output_files, symlink=True ) stageout = self.get_stage_in_out( output_files ) attrs = [ @@ -300,20 +275,20 @@ # write the job script if self.app.config.pbs_stage_path != '': - script = pbs_symlink_template % ( job_wrapper.galaxy_lib_dir, - " ".join( job_wrapper.get_input_fnames() + output_files ), - self.app.config.pbs_stage_path, - job_wrapper.get_env_setup_clause(), - exec_dir, - command_line, - ecfile ) + # touch the ecfile so that it gets staged + with file(ecfile, 'a'): + os.utime(ecfile, None) + stage_commands = pbs_symlink_template % ( + " ".join( job_wrapper.get_input_fnames() + output_files ), + self.app.config.pbs_stage_path, + exec_dir, + ) else: - script = pbs_template % ( job_wrapper.galaxy_lib_dir, - job_wrapper.get_env_setup_clause(), - exec_dir, - command_line, - ecfile ) + stage_commands = '' + + env_setup_commands = '%s\n%s' % (stage_commands, job_wrapper.get_env_setup_clause()) + script = self.get_job_file(job_wrapper, exit_code_path=ecfile, env_setup_commands=env_setup_commands) job_file = "%s/%s.sh" % (self.app.config.cluster_files_directory, job_wrapper.job_id) fh = file(job_file, "w") fh.write(script) https://bitbucket.org/galaxy/galaxy-central/commits/84784515c048/ Changeset: 84784515c048 User: jmchilton Date: 2013-10-17 00:37:37 Summary: Add GALAXY_SLOTS logic to local runner. Affected #: 1 file diff -r 5715b876f91bd34d179dc9d48e7ad1a9e6baf285 -r 84784515c0481cfddbfc05e983efced55972e9a0 lib/galaxy/jobs/runners/local.py --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -42,6 +42,21 @@ super( LocalJobRunner, self ).__init__( app, nworkers ) self._init_worker_threads() + def __command_line( self, job_wrapper ): + """ + """ + command_line = job_wrapper.runner_command_line + + ## slots would be cleaner name, but don't want deployers to see examples and think it + ## is going to work with other job runners. + slots = job_wrapper.job_destination.params.get( "local_slots", None ) + command_line = command_line.lstrip( " ;" ) + if slots: + command_line = 'export GALAXY_SLOTS="%d"; export GALAXY_SLOTS_CONFIGURED="1"; %s' % ( int( slots ), command_line ) + else: + command_line = 'export GALAXY_SLOTS="1"; %s' % command_line + return command_line + def queue_job( self, job_wrapper ): # prepare the job if not self.prepare_job( job_wrapper ): @@ -51,7 +66,7 @@ exit_code = 0 # command line has been added to the wrapper by prepare_job() - command_line = job_wrapper.runner_command_line + command_line = self.__command_line( job_wrapper ) job_id = job_wrapper.get_id_tag() https://bitbucket.org/galaxy/galaxy-central/commits/3825c8d49364/ Changeset: 3825c8d49364 User: jmchilton Date: 2013-10-17 00:37:37 Summary: Improved fix for tools producing command lines with trailing semi-colons. Affected #: 1 file diff -r 84784515c0481cfddbfc05e983efced55972e9a0 -r 3825c8d493649a72c6fc65bdd06b2362f0bf5930 lib/galaxy/jobs/command_factory.py --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -21,7 +21,7 @@ # Remove trailing semi-colon so we can start hacking up this command. # TODO: Refactor to compose a list and join with ';', would be more clean. - commands = commands.rstrip(";") + commands = commands.rstrip("; ") # Prepend version string if job_wrapper.version_string_cmd: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Enhance the Tool Shed API feature supporting resetting metadata on certain repositories to allow for individual repositories per request.
by commits-noreply＠bitbucket.org 16 Oct '13

16 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b681b2aeddbc/ Changeset: b681b2aeddbc User: greg Date: 2013-10-16 22:29:00 Summary: Enhance the Tool Shed API feature supporting resetting metadata on certain repositories to allow for individual repositories per request. Affected #: 3 files diff -r 2a4f4be3c377de638ee280b7a3ae83c62af6038a -r b681b2aeddbc52c3d953abcb431688df6fb8a005 lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -177,6 +177,42 @@ return message @web.expose_api + def repository_ids_for_setting_metadata( self, trans, my_writable=False, **kwd ): + """ + GET /api/get_repository_ids_for_setting_metadata + + Displays a collection (list) of repository ids ordered for setting metadata. + + :param key: the API key of the Tool Shed user. + :param my_writable (optional): if the API key is associated with an admin user in the Tool Shed, setting this param value + to True will restrict resetting metadata to only repositories that are writable by the user + in addition to those repositories of type tool_dependency_definition. This param is ignored + if the current user is not an admin user, in which case this same restriction is automatic. + """ + try: + if trans.user_is_admin(): + my_writable = util.asbool( my_writable ) + else: + my_writable = True + handled_repository_ids = [] + repository_ids = [] + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) + # Make sure repositories of type tool_dependency_definition are first in the list. + for repository in query: + if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + repository_ids.append( trans.security.encode_id( repository.id ) ) + # Now add all remaining repositories to the list. + for repository in query: + if repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + repository_ids.append( trans.security.encode_id( repository.id ) ) + return repository_ids + except Exception, e: + message = "Error in the Tool Shed repositories API in repository_ids_for_setting_metadata: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + + @web.expose_api def reset_metadata_on_repositories( self, trans, payload, **kwd ): """ PUT /api/repositories/reset_metadata_on_repositories @@ -239,6 +275,48 @@ trans.response.status = 500 return message + @web.expose_api + def reset_metadata_on_repository( self, trans, payload, **kwd ): + """ + PUT /api/repositories/reset_metadata_on_repository + + Resets all metadata on a specified repository in the Tool Shed. + + :param key: the API key of the Tool Shed user. + + The following parameters must be included in the payload. + :param repository_id: the encoded id of the repository on which metadata is to be reset. + """ + def handle_repository( trans, start_time, repository ): + results = dict( start_time=start_time, + repository_status=[] ) + try: + invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans, + trans.security.encode_id( repository.id ) ) + if invalid_file_tups: + message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) + else: + message = "Successfully reset metadata on repository %s" % str( repository.name ) + except Exception, e: + message = "Error resetting metadata on repository %s: %s" % ( str( repository.name ), str( e ) ) + + results[ 'repository_status' ].append( message ) + return results + try: + repository_id = payload.get( 'repository_id', None ) + if repository_id is not None: + repository = suc.get_repository_in_tool_shed( trans, repository_id ) + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results = handle_repository( trans, start_time, repository ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) + except Exception, e: + message = "Error in the Tool Shed repositories API in reset_metadata_on_repositories: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + @web.expose_api_anonymous def show( self, trans, id, **kwd ): """ diff -r 2a4f4be3c377de638ee280b7a3ae83c62af6038a -r b681b2aeddbc52c3d953abcb431688df6fb8a005 lib/galaxy/webapps/tool_shed/buildapp.py --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -85,7 +85,9 @@ controller='repositories', collection={ 'get_repository_revision_install_info' : 'GET', 'get_ordered_installable_revisions' : 'GET', - 'reset_metadata_on_repositories' : 'POST' }, + 'repository_ids_for_setting_metadata' : 'GET', + 'reset_metadata_on_repositories' : 'POST', + 'reset_metadata_on_repository' : 'POST' }, name_prefix='repository_', path_prefix='/api', parent_resources=dict( member_name='repository', collection_name='repositories' ) ) diff -r 2a4f4be3c377de638ee280b7a3ae83c62af6038a -r b681b2aeddbc52c3d953abcb431688df6fb8a005 lib/tool_shed/scripts/api/reset_metadata_on_repositories.py --- a/lib/tool_shed/scripts/api/reset_metadata_on_repositories.py +++ b/lib/tool_shed/scripts/api/reset_metadata_on_repositories.py @@ -16,20 +16,37 @@ import os import sys sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import get from common import submit +def string_as_bool( string ): + if str( string ).lower() in [ 'true' ]: + return True + else: + return False + def main( options ): api_key = options.api my_writable = options.my_writable + one_per_request = options.one_per_request base_tool_shed_url = options.tool_shed_url.rstrip( '/' ) - data = dict( my_writable=my_writable ) - url = '%s/api/repositories/reset_metadata_on_repositories' % base_tool_shed_url - submit( url, data, options.api ) + if string_as_bool( one_per_request ): + url = '%s/api/repositories/repository_ids_for_setting_metadata?key=%s&my_writable=%s' % ( base_tool_shed_url, api_key, str( my_writable ) ) + repository_ids = get( url, api_key ) + for repository_id in repository_ids: + data = dict( repository_id=repository_id ) + url = '%s/api/repositories/reset_metadata_on_repository' % base_tool_shed_url + submit( url, data, options.api ) + else: + data = dict( my_writable=my_writable ) + url = '%s/api/repositories/reset_metadata_on_repositories' % base_tool_shed_url + submit( url, data, options.api ) if __name__ == '__main__': parser = argparse.ArgumentParser( description='Reset metadata on certain repositories in the Tool Shed via the Tool Shed API.' ) parser.add_argument( "-a", "--api", dest="api", required=True, help="API Key" ) parser.add_argument( "-m", "--my_writable", dest="my_writable", required=False, default='False', help="Restrict to my writable repositories" ) + parser.add_argument( "-o", "--one_per_request", dest="one_per_request", required=False, default='False', help="One repository per request" ) parser.add_argument( "-u", "--url", dest="tool_shed_url", required=True, help="Tool Shed URL" ) options = parser.parse_args() main( options ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Allow user field (__user_email__, __user_name__, etc..) in hidden fields as outlined in PR 206, but do so in a way that centralizes some related logic (i.e. logic used by Cheetah in job templates and used by dynamic filter options).
by commits-noreply＠bitbucket.org 16 Oct '13

16 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/2a4f4be3c377/ Changeset: 2a4f4be3c377 User: jmchilton Date: 2013-09-19 21:46:45 Summary: Allow user field (__user_email__, __user_name__, etc..) in hidden fields as outlined in PR 206, but do so in a way that centralizes some related logic (i.e. logic used by Cheetah in job templates and used by dynamic filter options). Affected #: 4 files diff -r 64f283100f960c1ae7645fae17394334b1b7f349 -r 2a4f4be3c377de638ee280b7a3ae83c62af6038a lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -680,20 +680,8 @@ out_data[ "output_file" ] = FakeDatasetAssociation( dataset=special.dataset ) # These can be passed on the command line if wanted as $__user_*__ - if job.history and job.history.user: - user = job.history.user - user_id = '%d' % job.history.user.id - user_email = str(job.history.user.email) - user_name = str(job.history.user.username) - else: - user = None - user_id = 'Anonymous' - user_email = 'Anonymous' - user_name = 'Anonymous' - incoming['__user__'] = user - incoming['__user_id__'] = incoming['userId'] = user_id - incoming['__user_email__'] = incoming['userEmail'] = user_email - incoming['__user_name__'] = user_name + incoming.update( model.User.user_template_environment( job.history and job.history.user ) ) + # Build params, done before hook so hook can use param_dict = self.tool.build_param_dict( incoming, inp_data, out_data, diff -r 64f283100f960c1ae7645fae17394334b1b7f349 -r 2a4f4be3c377de638ee280b7a3ae83c62af6038a lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -18,6 +18,7 @@ import simplejson import socket import time +from string import Template import galaxy.datatypes import galaxy.datatypes.registry @@ -159,6 +160,47 @@ total += hda.dataset.get_total_size() return total + @staticmethod + def user_template_environment( user ): + """ + + >>> env = User.user_template_environment(None) + >>> env['__user_email__'] + 'Anonymous' + >>> env['__user_id__'] + 'Anonymous' + >>> user = User('foo(a)example.com') + >>> user.id = 6 + >>> user.username = 'foo2' + >>> env = User.user_template_environment(user) + >>> env['__user_id__'] + '6' + >>> env['__user_name__'] + 'foo2' + """ + if user: + user_id = '%d' % user.id + user_email = str( user.email ) + user_name = str( user.username ) + else: + user = None + user_id = 'Anonymous' + user_email = 'Anonymous' + user_name = 'Anonymous' + environment = {} + environment[ '__user__' ] = user + environment[ '__user_id__' ] = environment[ 'userId' ] = user_id + environment[ '__user_email__' ] = environment[ 'userEmail' ] = user_email + environment[ '__user_name__' ] = user_name + return environment + + @staticmethod + def expand_user_properties( user, in_string ): + """ + """ + environment = User.user_template_environment( user ) + return Template( in_string ).safe_substitute( environment ) + class Job( object, Dictifiable ): dict_collection_visible_keys = [ 'id' ] diff -r 64f283100f960c1ae7645fae17394334b1b7f349 -r 2a4f4be3c377de638ee280b7a3ae83c62af6038a lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2291,7 +2291,7 @@ args = dict() for key, param in self.inputs.iteritems(): if isinstance( param, HiddenToolParameter ): - args[key] = param.value + args[key] = model.User.expand_user_properties( trans.user, param.value ) elif isinstance( param, BaseURLToolParameter ): args[key] = param.get_value( trans ) else: diff -r 64f283100f960c1ae7645fae17394334b1b7f349 -r 2a4f4be3c377de638ee280b7a3ae83c62af6038a lib/galaxy/tools/parameters/dynamic_options.py --- a/lib/galaxy/tools/parameters/dynamic_options.py +++ b/lib/galaxy/tools/parameters/dynamic_options.py @@ -6,6 +6,7 @@ import operator, sys, os, logging import basic, validation from galaxy.util import string_as_bool +from galaxy.model import User import galaxy.tools log = logging.getLogger(__name__) @@ -55,8 +56,7 @@ rval = [] filter_value = self.value try: - if trans.user.email: - filter_value = filter_value.replace('$__user_email__',trans.user.email) + filter_value = User.expand_user_properties( trans.user, filter_value) except: pass for fields in options: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Enhance the Tool Shed and Galaxy APIs to support resetting metadata on repositories in the Tool Shed or installed into Galaxy respectively. When resetting repository metadata in the Tool Shed, if the API key is associated with an admin user, then setting the my_writable param value to True will restrict resetting metadata to only repositories that are writable by the user in addition to those repositories of type tool_dependency_definition. This param is ignored
by commits-noreply＠bitbucket.org 16 Oct '13

16 Oct '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/64f283100f96/ Changeset: 64f283100f96 User: greg Date: 2013-10-16 20:19:26 Summary: Enhance the Tool Shed and Galaxy APIs to support resetting metadata on repositories in the Tool Shed or installed into Galaxy respectively. When resetting repository metadata in the Tool Shed, if the API key is associated with an admin user, then setting the my_writable param value to True will restrict resetting metadata to only repositories that are writable by the user in addition to those repositories of type tool_dependency_definition. This param is ignored if the current user is not an admin user, in which case this same restriction is automatic. Also, when resetting metadata in the Tool Shed, repositories of type tool_dependency_definition are reset before repositories of type unrestricted. When setting metadata for installed repositories in Galaxy, neither repository types nor order are considered since these concepts are irrelevant in that environment. Affected #: 10 files diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -1,6 +1,7 @@ import logging from paste.httpexceptions import HTTPBadRequest, HTTPForbidden +from time import strftime from galaxy import util from galaxy import web @@ -10,6 +11,7 @@ from tool_shed.galaxy_install import repository_util from tool_shed.util import common_util from tool_shed.util import encoding_util +from tool_shed.util import metadata_util from tool_shed.util import workflow_util import tool_shed.util.shed_util_common as suc @@ -488,6 +490,49 @@ return tool_shed_repositories @web.expose_api + def reset_metadata_on_installed_repositories( self, trans, payload, **kwd ): + """ + PUT /api/tool_shed_repositories/reset_metadata_on_installed_repositories + + Resets all metadata on all repositories installed into Galaxy in an "orderly fashion". + + :param key: the API key of the Galaxy admin user. + """ + try: + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results = dict( start_time=start_time, + successful_count=0, + unsuccessful_count=0, + repository_status=[] ) + # Make sure the current user's API key proves he is an admin user in this Galaxy instance. + if not trans.user_is_admin(): + raise HTTPForbidden( detail='You are not authorized to reset metadata on repositories installed into this Galaxy instance.' ) + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=False ) + # Now reset metadata on all remaining repositories. + for repository in query: + repository_id = trans.security.encode_id( repository.id ) + try: + invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_installed_repository( trans, repository_id ) + if invalid_file_tups: + message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) + results[ 'unsuccessful_count' ] += 1 + else: + message = "Successfully reset metadata on repository %s" % str( repository.name ) + results[ 'successful_count' ] += 1 + except Exception, e: + message = "Error resetting metadata on repository %s: %s" % ( str( repository.name ), str( e ) ) + results[ 'unsuccessful_count' ] += 1 + results[ 'repository_status' ].append( message ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) + except Exception, e: + message = "Error in the Galaxy tool_shed_repositories API in reset_metadata_on_installed_repositories: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + + @web.expose_api def show( self, trans, id, **kwd ): """ GET /api/tool_shed_repositories/{encoded_tool_shed_repsository_id} diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -187,6 +187,7 @@ 'exported_workflows' : 'GET', 'import_workflow' : 'POST', 'import_workflows' : 'POST' }, + collection={ 'reset_metadata_on_installed_repositories' : 'POST' }, controller='tool_shed_repositories', name_prefix='tool_shed_repository_', path_prefix='/api', diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -1,11 +1,16 @@ import logging +from time import strftime from galaxy.web.framework.helpers import time_ago from galaxy import eggs from galaxy import web from galaxy import util +from galaxy.util import json from galaxy.web.base.controller import BaseAPIController +import tool_shed.repository_types.util as rt_util import tool_shed.util.shed_util_common as suc from tool_shed.galaxy_install import repository_util +from tool_shed.util import metadata_util +from tool_shed.util import tool_util eggs.require( 'mercurial' ) @@ -171,6 +176,69 @@ trans.response.status = 500 return message + @web.expose_api + def reset_metadata_on_repositories( self, trans, payload, **kwd ): + """ + PUT /api/repositories/reset_metadata_on_repositories + + Resets all metadata on all repositories in the Tool Shed in an "orderly fashion". Since there are currently only two + repository types (tool_dependecy_definition and unrestricted), the order in which metadata is reset is repositories of + type tool_dependecy_definition first followed by repositories of type unrestricted, and only one pass is necessary. If + a new repository type is introduced, the process will undoubtedly need to be revisited. To facilitate this order, an + in-memory list of repository ids that have been processed is maintained. + + :param key: the API key of the Tool Shed user. + + The following parameters can optionally be included in the payload. + :param my_writable (optional): if the API key is associated with an admin user in the Tool Shed, setting this param value + to True will restrict resetting metadata to only repositories that are writable by the user + in addition to those repositories of type tool_dependency_definition. This param is ignored + if the current user is not an admin user, in which case this same restriction is automatic. + """ + def handle_repository( trans, repository, results ): + repository_id = trans.security.encode_id( repository.id ) + try: + invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans, repository_id ) + if invalid_file_tups: + message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) + results[ 'unsuccessful_count' ] += 1 + else: + message = "Successfully reset metadata on repository %s" % str( repository.name ) + results[ 'successful_count' ] += 1 + except Exception, e: + message = "Error resetting metadata on repository %s: %s" % ( str( repository.name ), str( e ) ) + results[ 'unsuccessful_count' ] += 1 + results[ 'repository_status' ].append( message ) + return results + try: + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results = dict( start_time=start_time, + repository_status=[], + successful_count=0, + unsuccessful_count=0 ) + handled_repository_ids = [] + if trans.user_is_admin(): + my_writable = util.asbool( payload.get( 'my_writable', False ) ) + else: + my_writable = True + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) + # First reset metadata on all repositories of type repository_dependency_definition. + for repository in query: + if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + results = handle_repository( trans, repository, results ) + # Now reset metadata on all remaining repositories. + for repository in query: + if repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + results = handle_repository( trans, repository, results ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) + except Exception, e: + message = "Error in the Tool Shed repositories API in reset_metadata_on_repositories: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + @web.expose_api_anonymous def show( self, trans, id, **kwd ): """ diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/galaxy/webapps/tool_shed/buildapp.py --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -84,7 +84,8 @@ 'repositories', controller='repositories', collection={ 'get_repository_revision_install_info' : 'GET', - 'get_ordered_installable_revisions' : 'GET' }, + 'get_ordered_installable_revisions' : 'GET', + 'reset_metadata_on_repositories' : 'POST' }, name_prefix='repository_', path_prefix='/api', parent_resources=dict( member_name='repository', collection_name='repositories' ) ) diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/tool_shed/scripts/api/reset_metadata_on_repositories.py --- /dev/null +++ b/lib/tool_shed/scripts/api/reset_metadata_on_repositories.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +""" +Script to reset metadata on certain repositories in the Tool Shed. If the received API key is associated +with an admin user in the Tool Shed, setting the my_writable param value to True will restrict resetting +metadata to only repositories that are writable by the user in addition to those repositories of type +tool_dependency_definition. The my_writable param is ignored if the current user is not an admin user, +in which case this same restriction is automatic. + +usage: reset_metadata_on_repositories.py key <my_writable> + +Here is a working example of how to use this script to reset metadata on certain repositories in a specified Tool Shed. +python ./reset_metadata_on_repositories.py -a 22be3b -m True -u http://localhost:9009/ +""" + +import argparse +import os +import sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +def main( options ): + api_key = options.api + my_writable = options.my_writable + base_tool_shed_url = options.tool_shed_url.rstrip( '/' ) + data = dict( my_writable=my_writable ) + url = '%s/api/repositories/reset_metadata_on_repositories' % base_tool_shed_url + submit( url, data, options.api ) + +if __name__ == '__main__': + parser = argparse.ArgumentParser( description='Reset metadata on certain repositories in the Tool Shed via the Tool Shed API.' ) + parser.add_argument( "-a", "--api", dest="api", required=True, help="API Key" ) + parser.add_argument( "-m", "--my_writable", dest="my_writable", required=False, default='False', help="Restrict to my writable repositories" ) + parser.add_argument( "-u", "--url", dest="tool_shed_url", required=True, help="Tool Shed URL" ) + options = parser.parse_args() + main( options ) diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -1782,10 +1782,10 @@ log.debug( message ) unsuccessful_count += 1 else: - log.debug( "Successfully reset metadata on repository %s" % repository.name ) + log.debug( "Successfully reset metadata on repository %s" % str( repository.name ) ) successful_count += 1 except: - log.exception( "Error attempting to reset metadata on repository %s", repository.name ) + log.exception( "Error attempting to reset metadata on repository %s", str( repository.name ) ) unsuccessful_count += 1 message = "Successfully reset metadata on %d %s. " % ( successful_count, inflector.cond_plural( successful_count, "repository" ) ) if unsuccessful_count: diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -20,6 +20,7 @@ from tool_shed.util import common_util from tool_shed.util import encoding_util from tool_shed.util import xml_util +import tool_shed.repository_types.util as rt_util from xml.etree import ElementTree as XmlET from urllib2 import HTTPError @@ -104,45 +105,15 @@ def build_repository_ids_select_field( trans, name='repository_ids', multiple=True, display='checkboxes', my_writable=False ): """Method called from both Galaxy and the Tool Shed to generate the current list of repositories for resetting metadata.""" repositories_select_field = SelectField( name=name, multiple=multiple, display=display ) - if trans.webapp.name == 'tool_shed': - # We're in the tool shed. - if my_writable: - username = trans.user.username - clause_list = [] - for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.deleted == False ): - allow_push = repository.allow_push( trans.app ) - if allow_push: - allow_push_usernames = allow_push.split( ',' ) - if username in allow_push_usernames: - clause_list.append( trans.model.Repository.table.c.id == repository.id ) - if clause_list: - for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( or_( *clause_list ) ) \ - .order_by( trans.model.Repository.table.c.name, - trans.model.Repository.table.c.user_id ): - owner = repository.user.username - option_label = '%s (%s)' % ( repository.name, owner ) - option_value = '%s' % trans.security.encode_id( repository.id ) - repositories_select_field.add_option( option_label, option_value ) + query = get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=True ) + for repository in query: + if trans.webapp.name == 'tool_shed': + owner = str( repository.user.username ) else: - for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.deleted == False ) \ - .order_by( trans.model.Repository.table.c.name, - trans.model.Repository.table.c.user_id ): - owner = repository.user.username - option_label = '%s (%s)' % ( repository.name, owner ) - option_value = '%s' % trans.security.encode_id( repository.id ) - repositories_select_field.add_option( option_label, option_value ) - else: - # We're in Galaxy. - for repository in trans.sa_session.query( trans.model.ToolShedRepository ) \ - .filter( trans.model.ToolShedRepository.table.c.uninstalled == False ) \ - .order_by( trans.model.ToolShedRepository.table.c.name, - trans.model.ToolShedRepository.table.c.owner ): - option_label = '%s (%s)' % ( repository.name, repository.owner ) - option_value = trans.security.encode_id( repository.id ) - repositories_select_field.add_option( option_label, option_value ) + owner = str( repository.owner ) + option_label = '%s (%s)' % ( str( repository.name ), owner ) + option_value = '%s' % trans.security.encode_id( repository.id ) + repositories_select_field.add_option( option_label, option_value ) return repositories_select_field def build_tool_dependencies_select_field( trans, tool_shed_repository, name, multiple=True, display='checkboxes', uninstalled_only=False ): @@ -836,6 +807,63 @@ prior_import_or_install_required_dict[ encoded_repository_id ] = prior_import_or_install_ids return prior_import_or_install_required_dict +def get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=True ): + """ + Return a query containing repositories for resetting metadata. This method is called from both the Tool Shed and Galaxy. The + my_writable parameter is ignored unless called from the Tool Shed, and the order parameter is used for displaying the list of + repositories ordered alphabetically for display on a page. When called from wither the Tool Shed or Galaxy API, order is False. + """ + if trans.webapp.name == 'tool_shed': + # When called from the Tool Shed API, the metadata is reset on all repositories of type tool_dependency_definition in addition + # to other selected repositories. + if my_writable: + username = trans.user.username + clause_list = [] + for repository in trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.deleted == False ): + allow_push = repository.allow_push( trans.app ) + if not order: + # We've been called from the Tool Shed API, so reset metadata on all repositories of type tool_dependency_definition. + if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION: + clause_list.append( trans.model.Repository.table.c.id == repository.id ) + elif allow_push: + # Include all repositories that are writable by the current user. + allow_push_usernames = allow_push.split( ',' ) + if username in allow_push_usernames: + clause_list.append( trans.model.Repository.table.c.id == repository.id ) + if clause_list: + if order: + return trans.sa_session.query( trans.model.Repository ) \ + .filter( or_( *clause_list ) ) \ + .order_by( trans.model.Repository.table.c.name, + trans.model.Repository.table.c.user_id ) + else: + return trans.sa_session.query( trans.model.Repository ) \ + .filter( or_( *clause_list ) ) + else: + # Return an empty query. + return trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.id == -1 ) + else: + if order: + return trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.deleted == False ) \ + .order_by( trans.model.Repository.table.c.name, + trans.model.Repository.table.c.user_id ) + else: + return trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.deleted == False ) + else: + # We're in Galaxy. + if order: + return trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( trans.model.ToolShedRepository.table.c.uninstalled == False ) \ + .order_by( trans.model.ToolShedRepository.table.c.name, + trans.model.ToolShedRepository.table.c.owner ) + else: + return trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( trans.model.ToolShedRepository.table.c.uninstalled == False ) + def get_repo_info_tuple_contents( repo_info_tuple ): """Take care in handling the repo_info_tuple as it evolves over time as new tool shed features are introduced.""" if len( repo_info_tuple ) == 6: diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 scripts/api/import_workflows_from_installed_tool_shed_repository.py --- a/scripts/api/import_workflows_from_installed_tool_shed_repository.py +++ b/scripts/api/import_workflows_from_installed_tool_shed_repository.py @@ -45,7 +45,7 @@ data = {} # NOTE: to import a single workflow, add an index to data (e.g., # data[ 'index' ] = 0 - # and change the url to be ~/import_workflow (simgular). For example, + # and change the url to be ~/import_workflow (singular). For example, # url = '%s%s' % ( base_galaxy_url, '/api/tool_shed_repositories/%s/import_workflow' % str( tool_shed_repository_id ) ) url = '%s%s' % ( base_galaxy_url, '/api/tool_shed_repositories/%s/import_workflows' % str( tool_shed_repository_id ) ) submit( options.api, url, data ) diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 scripts/api/reset_metadata_on_installed_repositories.py --- /dev/null +++ b/scripts/api/reset_metadata_on_installed_repositories.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +""" +Script to reset metadata on all Tool Shed repositories installed into a Galaxy instance. The received +API key must be associated with a Galaxy admin user. + +usage: reset_metadata_on_installed_repositories.py key + +Here is a working example of how to use this script. +python ./reset_metadata_on_installed_repositories.py -a 22be3b -u http://localhost:8763/ +""" + +import argparse +import os +import sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +def main( options ): + api_key = options.api + base_galaxy_url = options.galaxy_url.rstrip( '/' ) + url = '%s/api/tool_shed_repositories/reset_metadata_on_installed_repositories' % base_galaxy_url + submit( options.api, url, {} ) + +if __name__ == '__main__': + parser = argparse.ArgumentParser( description='Reset metadata on all Tool Shed repositories installed into Galaxy via the Galaxy API.' ) + parser.add_argument( "-a", "--api", dest="api", required=True, help="API Key" ) + parser.add_argument( "-u", "--url", dest="galaxy_url", required=True, help="Galaxy URL" ) + options = parser.parse_args() + main( options ) diff -r 00764f2c46dff45f208e22203c6498777725bba7 -r 64f283100f960c1ae7645fae17394334b1b7f349 test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -969,7 +969,7 @@ self.check_for_strings( [ 'Metadata has been reset' ] ) def reset_metadata_on_selected_repositories( self, repository_ids ): - self.visit_url( '/admin/reset_metadata_on_selected_repositories_in_tool_shed' ) + self.visit_url( '/admin/reset_metadata_on_selected_repositories' ) kwd = dict( repository_ids=repository_ids ) self.submit_form( form_no=1, button="reset_metadata_on_selected_repositories_button", **kwd ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 16 Oct '13

16 Oct '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e786022dc67e/ Changeset: e786022dc67e User: jmchilton Date: 2013-10-16 20:01:15 Summary: Improved handling of tools producing unicode in standard error/output. Without this change (modification on idea by Bjoern), sqlite reports errors as follows for such tools: ProgrammingError: (ProgrammingError) You must not use 8-bit bytestrings unless you use a text_factory that can interpret 8-bit bytestrings (like text_factory = str). It is highly recommended that you instead just switch your application to Unicode strings. u'UPDATE job SET update_time=?, stdout=?, stderr=? WHERE job.id = ?' ('2013-10-10 05:00:05.789309', '', '/dev/fd/2 unicode \xc3\xaa\xe0\xaf\xb3\xe0\xbe\x85\xe1\x9d\xb1\xe3\x8e\xb0\n', 11). Affected #: 1 file diff -r 1c22e43fc21e049da34af60f603f2e19bde912d2 -r e786022dc67ed918050bd81b9ac679ac958e4f75 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -25,6 +25,7 @@ from galaxy.util.bunch import Bunch from galaxy.util.expressions import ExpressionContext from galaxy.util.json import from_json_string +from galaxy.util import unicodify from .output_checker import check_output log = logging.getLogger( __name__ ) @@ -869,6 +870,9 @@ the output datasets based on stderr and stdout from the command, and the contents of the output files. """ + stdout = unicodify( stdout ) + stderr = unicodify( stderr ) + # default post job setup self.sa_session.expunge_all() job = self.get_job() @@ -1518,6 +1522,9 @@ the output datasets based on stderr and stdout from the command, and the contents of the output files. """ + stdout = unicodify( stdout ) + stderr = unicodify( stderr ) + # This may have ended too soon log.debug( 'task %s for job %d ended; exit code: %d' % (self.task_id, self.job_id, https://bitbucket.org/galaxy/galaxy-central/commits/00764f2c46df/ Changeset: 00764f2c46df User: jmchilton Date: 2013-10-16 20:01:15 Summary: Improved handling of tools producing unicode in stardard error/output. Without this modification paste cannot render the stderr and/or stdout of such tools. This is a modification on an idea from Simon Guest, utilizing some code (now available as galaxy.util.smart_str) from an older version of Django to solve this problem. Affected #: 2 files diff -r e786022dc67ed918050bd81b9ac679ac958e4f75 -r 00764f2c46dff45f208e22203c6498777725bba7 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -547,6 +547,30 @@ except: return default + +def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'): + """ + Returns a bytestring version of 's', encoded as specified in 'encoding'. + + If strings_only is True, don't convert (some) non-string-like objects. + + Adapted from an older, simpler version of django.utils.encoding.smart_str. + """ + if strings_only and isinstance(s, (type(None), int)): + return s + if not isinstance(s, basestring): + try: + return str(s) + except UnicodeEncodeError: + return unicode(s).encode(encoding, errors) + elif isinstance(s, unicode): + return s.encode(encoding, errors) + elif s and encoding != 'utf-8': + return s.decode('utf-8', errors).encode(encoding, errors) + else: + return s + + def object_to_string( obj ): return binascii.hexlify( pickle.dumps( obj, 2 ) ) diff -r e786022dc67ed918050bd81b9ac679ac958e4f75 -r 00764f2c46dff45f208e22203c6498777725bba7 lib/galaxy/webapps/galaxy/controllers/dataset.py --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -8,7 +8,7 @@ from galaxy import datatypes, eggs, model, util, web from galaxy.datatypes.display_applications.util import decode_dataset_user, encode_dataset_user from galaxy.model.item_attrs import UsesAnnotations, UsesItemRatings -from galaxy.util import inflector +from galaxy.util import inflector, smart_str from galaxy.util.sanitize_html import sanitize_html from galaxy.web.base.controller import BaseUIController, ERROR, SUCCESS, url_for, UsesHistoryDatasetAssociationMixin, UsesHistoryMixin from galaxy.web.framework.helpers import grids, iff, time_ago @@ -184,7 +184,7 @@ stdout = job.stdout except: stdout = "Invalid dataset ID or you are not allowed to access this dataset" - return stdout + return smart_str( stdout ) @web.expose # TODO: Migrate stderr and stdout to use _get_job_for_dataset; it wasn't tested. @@ -196,7 +196,7 @@ stderr = job.stderr except: stderr = "Invalid dataset ID or you are not allowed to access this dataset" - return stderr + return smart_str( stderr ) @web.expose def exit_code( self, trans, dataset_id=None, **kwargs ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 4 new changesets
by commits-noreply＠bitbucket.org 16 Oct '13

16 Oct '13

4 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3a156f9e008d/ Changeset: 3a156f9e008d User: nsoranzo Date: 2013-09-18 14:43:59 Summary: Prevent no choice in multiple selections. Affected #: 2 files diff -r 750e687b468a27978d51dfac9d542a0cc113e64b -r 3a156f9e008d3e61a951aa9efba2c982cb826e0a tools/ngs_simulation/ngs_simulation.py --- a/tools/ngs_simulation/ngs_simulation.py +++ b/tools/ngs_simulation/ngs_simulation.py @@ -70,11 +70,11 @@ num_sims = int( options.num_sims ) except TypeError, e: stop_err( 'Make sure the number of simulations is an integer value: %s' % str( e ) ) - if len( options.polymorphism ) > 0: + if options.polymorphism != 'None': polymorphisms = [ float( p ) for p in options.polymorphism.split( ',' ) ] else: stop_err( 'Select at least one polymorphism value to use' ) - if len( options.detection_thresh ) > 0: + if options.detection_thresh != 'None': detection_threshes = [ float( dt ) for dt in options.detection_thresh.split( ',' ) ] else: stop_err( 'Select at least one detection threshold to use' ) diff -r 750e687b468a27978d51dfac9d542a0cc113e64b -r 3a156f9e008d3e61a951aa9efba2c982cb826e0a tools/ngs_simulation/ngs_simulation.xml --- a/tools/ngs_simulation/ngs_simulation.xml +++ b/tools/ngs_simulation/ngs_simulation.xml @@ -72,6 +72,7 @@ <option value="0.8">0.8</option><option value="0.9">0.9</option><option value="1.0">1.0</option> + <validator type="no_options" message="You must select at least one value" /></param><param name="detection_thresh" type="select" multiple="true" label="Detection thresholds"><option value="0.001">0.001</option> @@ -102,6 +103,7 @@ <option value="0.8">0.8</option><option value="0.9">0.9</option><option value="1.0">1.0</option> + <validator type="no_options" message="You must select at least one value" /></param><param name="summary_out" type="boolean" truevalue="true" falsevalue="false" checked="true" label="Include a (text) summary file for all the simulations" /><!-- <param name="sim_results" type="boolean" truevalue="true" falsevalue="false" checked="false" label="Output all tabular simulation results" help="Number of polymorphisms times number of detection thresholds"/> https://bitbucket.org/galaxy/galaxy-central/commits/ff4bbde97901/ Changeset: ff4bbde97901 User: nsoranzo Date: 2013-09-18 14:44:42 Summary: Fix reference to 'Fetch taxonomic representation' tool. Affected #: 1 file diff -r 3a156f9e008d3e61a951aa9efba2c982cb826e0a -r ff4bbde97901cc3dd3ab9c36432e2d17a655265a tools/taxonomy/t2ps_wrapper.xml --- a/tools/taxonomy/t2ps_wrapper.xml +++ b/tools/taxonomy/t2ps_wrapper.xml @@ -47,7 +47,7 @@ **What it does** -Given taxonomy representation (produced by *Taxonomy manipulation->Fetch Taxonomic Ranks* tool) this utility produces a graphical representations of phylogenetic tree in PDF format. +Given taxonomy representation (produced by *Fetch taxonomic representation* tool) this utility produces a graphical representations of phylogenetic tree in PDF format. -------- https://bitbucket.org/galaxy/galaxy-central/commits/c61cc2f91c59/ Changeset: c61cc2f91c59 User: nsoranzo Date: 2013-10-04 18:15:22 Summary: Fix for "syntax error near unexpected token `;;'". <command> should not end with a semicolon, otherwise BaseJobRunner.build_command_line() would create a bash command containing ';;' when appending metadata setting commands. Affected #: 1 file diff -r ff4bbde97901cc3dd3ab9c36432e2d17a655265a -r c61cc2f91c59c54e585ef92b77f650774ebdbd55 tools/sr_mapping/bowtie2_wrapper.xml --- a/tools/sr_mapping/bowtie2_wrapper.xml +++ b/tools/sr_mapping/bowtie2_wrapper.xml @@ -95,15 +95,15 @@ #end if ## view/sort and output file - | samtools view -Su - | samtools sort -o - - > $output; + | samtools view -Su - | samtools sort -o - - > $output ## rename unaligned sequence files #if $library.type == "paired" and $output_unaligned_reads_l and $output_unaligned_reads_r: #set left = str($output_unaligned_reads_l).replace( '.dat', '.1.dat' ) #set right = str($output_unaligned_reads_l).replace( '.dat', '.2.dat' ) - mv $left $output_unaligned_reads_l; - mv $right $output_unaligned_reads_r; + ; mv $left $output_unaligned_reads_l; + mv $right $output_unaligned_reads_r #end if </command> https://bitbucket.org/galaxy/galaxy-central/commits/1c22e43fc21e/ Changeset: 1c22e43fc21e User: jmchilton Date: 2013-10-16 19:49:34 Summary: Merged in nsoranzo/galaxy-central-tools2 (pull request #234) Bug fixes for 3 tools (rebased #222) Affected #: 4 files diff -r e8fbf32ba1ccb4edec436e9ca76a40d6f64917be -r 1c22e43fc21e049da34af60f603f2e19bde912d2 tools/ngs_simulation/ngs_simulation.py --- a/tools/ngs_simulation/ngs_simulation.py +++ b/tools/ngs_simulation/ngs_simulation.py @@ -70,11 +70,11 @@ num_sims = int( options.num_sims ) except TypeError, e: stop_err( 'Make sure the number of simulations is an integer value: %s' % str( e ) ) - if len( options.polymorphism ) > 0: + if options.polymorphism != 'None': polymorphisms = [ float( p ) for p in options.polymorphism.split( ',' ) ] else: stop_err( 'Select at least one polymorphism value to use' ) - if len( options.detection_thresh ) > 0: + if options.detection_thresh != 'None': detection_threshes = [ float( dt ) for dt in options.detection_thresh.split( ',' ) ] else: stop_err( 'Select at least one detection threshold to use' ) diff -r e8fbf32ba1ccb4edec436e9ca76a40d6f64917be -r 1c22e43fc21e049da34af60f603f2e19bde912d2 tools/ngs_simulation/ngs_simulation.xml --- a/tools/ngs_simulation/ngs_simulation.xml +++ b/tools/ngs_simulation/ngs_simulation.xml @@ -72,6 +72,7 @@ <option value="0.8">0.8</option><option value="0.9">0.9</option><option value="1.0">1.0</option> + <validator type="no_options" message="You must select at least one value" /></param><param name="detection_thresh" type="select" multiple="true" label="Detection thresholds"><option value="0.001">0.001</option> @@ -102,6 +103,7 @@ <option value="0.8">0.8</option><option value="0.9">0.9</option><option value="1.0">1.0</option> + <validator type="no_options" message="You must select at least one value" /></param><param name="summary_out" type="boolean" truevalue="true" falsevalue="false" checked="true" label="Include a (text) summary file for all the simulations" /><!-- <param name="sim_results" type="boolean" truevalue="true" falsevalue="false" checked="false" label="Output all tabular simulation results" help="Number of polymorphisms times number of detection thresholds"/> diff -r e8fbf32ba1ccb4edec436e9ca76a40d6f64917be -r 1c22e43fc21e049da34af60f603f2e19bde912d2 tools/sr_mapping/bowtie2_wrapper.xml --- a/tools/sr_mapping/bowtie2_wrapper.xml +++ b/tools/sr_mapping/bowtie2_wrapper.xml @@ -95,15 +95,15 @@ #end if ## view/sort and output file - | samtools view -Su - | samtools sort -o - - > $output; + | samtools view -Su - | samtools sort -o - - > $output ## rename unaligned sequence files #if $library.type == "paired" and $output_unaligned_reads_l and $output_unaligned_reads_r: #set left = str($output_unaligned_reads_l).replace( '.dat', '.1.dat' ) #set right = str($output_unaligned_reads_l).replace( '.dat', '.2.dat' ) - mv $left $output_unaligned_reads_l; - mv $right $output_unaligned_reads_r; + ; mv $left $output_unaligned_reads_l; + mv $right $output_unaligned_reads_r #end if </command> diff -r e8fbf32ba1ccb4edec436e9ca76a40d6f64917be -r 1c22e43fc21e049da34af60f603f2e19bde912d2 tools/taxonomy/t2ps_wrapper.xml --- a/tools/taxonomy/t2ps_wrapper.xml +++ b/tools/taxonomy/t2ps_wrapper.xml @@ -47,7 +47,7 @@ **What it does** -Given taxonomy representation (produced by *Taxonomy manipulation->Fetch Taxonomic Ranks* tool) this utility produces a graphical representations of phylogenetic tree in PDF format. +Given taxonomy representation (produced by *Fetch taxonomic representation* tool) this utility produces a graphical representations of phylogenetic tree in PDF format. -------- Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0