1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a3cff55b129f/ Changeset: a3cff55b129f User: jmchilton Date: 2014-10-07 17:28:43+00:00 Summary: Method for using local network (e.g. NFS) to distribute cached docker images across clusters. General idea and outline for short caching script embedded in containers.py from Kyle Ellrott. Affected #: 4 files diff -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 -r a3cff55b129fcbf9b8e359ff0f71cb844a7e9277 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -143,6 +143,7 @@ self.cluster_files_directory = os.path.abspath( kwargs.get( "cluster_files_directory", "database/pbs" ) ) self.job_working_directory = resolve_path( kwargs.get( "job_working_directory", "database/job_working_directory" ), self.root ) self.cleanup_job = kwargs.get( "cleanup_job", "always" ) + self.container_image_cache_path = self.resolve_path( kwargs.get( "container_image_cache_path", "database/container_images" ) ) self.outputs_to_working_directory = string_as_bool( kwargs.get( 'outputs_to_working_directory', False ) ) self.output_size_limit = int( kwargs.get( 'output_size_limit', 0 ) ) self.retry_job_output_collection = int( kwargs.get( 'retry_job_output_collection', 0 ) ) @@ -640,7 +641,8 @@ app_info = containers.AppInfo( galaxy_root_dir, default_file_path=file_path, - outputs_to_working_directory=self.config.outputs_to_working_directory + outputs_to_working_directory=self.config.outputs_to_working_directory, + container_image_cache_path=self.config.container_image_cache_path, ) self.container_finder = galaxy.tools.deps.containers.ContainerFinder(app_info) diff -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 -r a3cff55b129fcbf9b8e359ff0f71cb844a7e9277 lib/galaxy/tools/deps/containers.py --- a/lib/galaxy/tools/deps/containers.py +++ b/lib/galaxy/tools/deps/containers.py @@ -13,6 +13,29 @@ DEFAULT_CONTAINER_TYPE = "docker" +LOAD_CACHED_IMAGE_COMMAND_TEMPLATE = ''' +python << EOF +import re, tarfile, json, subprocess +t = tarfile.TarFile("${cached_image_file}") +meta_str = t.extractfile('repositories').read() +meta = json.loads(meta_str) +tag, tag_value = meta.items()[0] +rev, rev_value = tag_value.items()[0] +cmd = "${images_cmd}" +proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) +stdo, stde = proc.communicate() +found = False +for line in stdo.split("\\n"): + tmp = re.split(r'\s+', line) + if tmp[0] == tag and tmp[1] == rev and tmp[2] == rev_value: + found = True +if not found: + print "Loading image" + cmd = "cat ${cached_image_file} | ${load_cmd}" + subprocess.check_call(cmd, shell=True) +EOF +''' + class ContainerFinder(object): @@ -117,11 +140,18 @@ class AppInfo(object): - def __init__(self, galaxy_root_dir=None, default_file_path=None, outputs_to_working_directory=False): + def __init__( + self, + galaxy_root_dir=None, + default_file_path=None, + outputs_to_working_directory=False, + container_image_cache_path=None, + ): self.galaxy_root_dir = galaxy_root_dir self.default_file_path = default_file_path # TODO: Vary default value for docker_volumes based on this... self.outputs_to_working_directory = outputs_to_working_directory + self.container_image_cache_path = container_image_cache_path class ToolInfo(object): @@ -199,7 +229,12 @@ host=prop("host", docker_util.DEFAULT_HOST), ) - cache_command = docker_util.build_docker_cache_command(self.container_id, **docker_host_props) + cached_image_file = self.__get_cached_image_file() + if not cached_image_file: + # TODO: Add option to cache it once here and create cached_image_file. + cache_command = docker_util.build_docker_cache_command(self.container_id, **docker_host_props) + else: + cache_command = self.__cache_from_file_command(cached_image_file, docker_host_props) run_command = docker_util.build_docker_run_command( command, self.container_id, @@ -213,6 +248,29 @@ ) return "%s\n%s" % (cache_command, run_command) + def __cache_from_file_command(self, cached_image_file, docker_host_props): + images_cmd = docker_util.build_docker_images_command(truncate=False, **docker_host_props) + load_cmd = docker_util.build_docker_load_command(**docker_host_props) + + return string.Template(LOAD_CACHED_IMAGE_COMMAND_TEMPLATE).safe_substitute( + cached_image_file=cached_image_file, + images_cmd=images_cmd, + load_cmd=load_cmd + ) + + def __get_cached_image_file(self): + container_id = self.container_id + cache_directory = os.path.abspath(self.__get_destination_overridable_property("container_image_cache_path")) + cache_path = docker_cache_path(cache_directory, container_id) + return cache_path if os.path.exists(cache_path) else None + + def __get_destination_overridable_property(self, name): + prop_name = "docker_%s" % name + if prop_name in self.destination_info: + return self.destination_info[prop_name] + else: + return getattr(self.app_info, name) + def __expand_str(self, value): if not value: return value @@ -248,6 +306,12 @@ return template.safe_substitute(variables) +def docker_cache_path(cache_directory, container_id): + file_container_id = container_id.replace("/", "_slash_") + cache_file_name = "docker_%s.tar" % file_container_id + return os.path.join(cache_directory, cache_file_name) + + CONTAINER_CLASSES = dict( docker=DockerContainer, ) diff -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 -r a3cff55b129fcbf9b8e359ff0f71cb844a7e9277 lib/galaxy/tools/deps/docker_util.py --- a/lib/galaxy/tools/deps/docker_util.py +++ b/lib/galaxy/tools/deps/docker_util.py @@ -55,14 +55,11 @@ def build_command( image, docker_build_path, - docker_cmd=DEFAULT_DOCKER_COMMAND, - sudo=DEFAULT_SUDO, - sudo_cmd=DEFAULT_SUDO_COMMAND, - host=DEFAULT_HOST, + **kwds ): if os.path.isfile(docker_build_path): docker_build_path = os.path.dirname(os.path.abspath(docker_build_path)) - build_command_parts = __docker_prefix(docker_cmd, sudo, sudo_cmd, host) + build_command_parts = __docker_prefix(**kwds) build_command_parts.extend(["build", "-t", image, docker_build_path]) return build_command_parts @@ -70,34 +67,42 @@ def build_save_image_command( image, destination, - docker_cmd=DEFAULT_DOCKER_COMMAND, - sudo=DEFAULT_SUDO, - sudo_cmd=DEFAULT_SUDO_COMMAND, - host=DEFAULT_HOST, + **kwds ): - build_command_parts = __docker_prefix(docker_cmd, sudo, sudo_cmd, host) + build_command_parts = __docker_prefix(**kwds) build_command_parts.extend(["save", "-o", destination, image]) return build_command_parts def build_docker_cache_command( image, - docker_cmd=DEFAULT_DOCKER_COMMAND, - sudo=DEFAULT_SUDO, - sudo_cmd=DEFAULT_SUDO_COMMAND, - host=DEFAULT_HOST, + **kwds ): - inspect_command_parts = __docker_prefix(docker_cmd, sudo, sudo_cmd, host) + inspect_command_parts = __docker_prefix(**kwds) inspect_command_parts.extend(["inspect", image]) inspect_image_command = " ".join(inspect_command_parts) - pull_command_parts = __docker_prefix(docker_cmd, sudo, sudo_cmd, host) + pull_command_parts = __docker_prefix(**kwds) pull_command_parts.extend(["pull", image]) pull_image_command = " ".join(pull_command_parts) cache_command = "%s > /dev/null 2>&1\n[ $? -ne 0 ] && %s > /dev/null 2>&1\n" % (inspect_image_command, pull_image_command) return cache_command +def build_docker_images_command(truncate=True, **kwds): + images_command_parts = __docker_prefix(**kwds) + images_command_parts.append("images") + if not truncate: + images_command_parts.append("--no-trunc") + return " ".join(images_command_parts) + + +def build_docker_load_command(**kwds): + load_command_parts = __docker_prefix(**kwds) + load_command_parts.append("load") + return " ".join(load_command_parts) + + def build_docker_run_command( container_command, image, @@ -116,7 +121,12 @@ auto_rm=DEFAULT_AUTO_REMOVE, host=DEFAULT_HOST, ): - command_parts = __docker_prefix(docker_cmd, sudo, sudo_cmd, host) + command_parts = __docker_prefix( + docker_cmd=docker_cmd, + sudo=sudo, + sudo_cmd=sudo_cmd, + host=host + ) command_parts.append("run") if interactive: command_parts.append("-i") @@ -144,7 +154,13 @@ return " ".join(command_parts) -def __docker_prefix(docker_cmd, sudo, sudo_cmd, host): +def __docker_prefix( + docker_cmd=DEFAULT_DOCKER_COMMAND, + sudo=DEFAULT_SUDO, + sudo_cmd=DEFAULT_SUDO_COMMAND, + host=DEFAULT_HOST, + **kwds +): """ Prefix to issue a docker command. """ command_parts = [] diff -r 6b7782f17e84b357c968c7e8e14d1f50c3668008 -r a3cff55b129fcbf9b8e359ff0f71cb844a7e9277 lib/galaxy/tools/deps/dockerfiles.py --- a/lib/galaxy/tools/deps/dockerfiles.py +++ b/lib/galaxy/tools/deps/dockerfiles.py @@ -2,6 +2,7 @@ from ..deps import commands from ..deps import docker_util +from ..deps.containers import docker_cache_path from ..deps.requirements import parse_requirements_from_xml from ...tools import loader_directory @@ -20,11 +21,13 @@ def dockerfile_build(path, dockerfile=None, error=log.error, **kwds): expected_container_names = set() + tool_directories = set() for (tool_path, tool_xml) in loader_directory.load_tool_elements_from_path(path): requirements, containers = parse_requirements_from_xml(tool_xml) for container in containers: if container.type == "docker": expected_container_names.add(container.identifier) + tool_directories.add(os.path.dirname(tool_path)) break if len(expected_container_names) == 0: @@ -34,9 +37,8 @@ error("Multiple different docker identifiers found for selected tools [%s]", expected_container_names) image_identifier = expected_container_names.pop() - if dockerfile is None: - dockerfile = "Dockerfile" + dockerfile = __find_dockerfile(dockerfile, tool_directories) docker_command_parts = docker_util.build_command( image_identifier, dockerfile, @@ -45,10 +47,24 @@ commands.execute(docker_command_parts) docker_image_cache = kwds['docker_image_cache'] if docker_image_cache: - destination = os.path.join(docker_image_cache, image_identifier + ".tar") + destination = docker_cache_path(docker_image_cache, image_identifier) save_image_command_parts = docker_util.build_save_image_command( image_identifier, destination, **docker_host_args(**kwds) ) commands.execute(save_image_command_parts) + + +def __find_dockerfile(dockerfile, tool_directories): + if dockerfile is not None: + return dockerfile + search_directories = ["."] + if len(tool_directories) == 1: + tool_directory = tool_directories.pop() + search_directories.insert(0, tool_directory) + for directory in search_directories: + potential_dockerfile = os.path.join(directory, "Dockerfile") + if os.path.exists(potential_dockerfile): + return potential_dockerfile + raise Exception("Could not find dockerfile to build.") Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.