1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/45df9d2018e1/ Changeset: 45df9d2018e1 User: jmchilton Date: 2014-06-17 06:14:59 Summary: Update LWR client and synchronized modules through LWR changeset 148132f. Mostly style fixes but also adds ability to configure LWR file staging actions with YAML instead of JSON. Affected #: 6 files diff -r 0a428afbc23538b6f8d0a8d9dcc29b9a576c1ddc -r 45df9d2018e1c452478549b933476937595bf6a4 lib/galaxy/jobs/runners/lwr_client/action_mapper.py --- a/lib/galaxy/jobs/runners/lwr_client/action_mapper.py +++ b/lib/galaxy/jobs/runners/lwr_client/action_mapper.py @@ -11,6 +11,7 @@ from re import escape import galaxy.util from galaxy.util.bunch import Bunch +from .config_util import read_file from .util import directory_files from .util import unique_path_prefix from .transport import get_file @@ -160,7 +161,7 @@ def __client_to_config(self, client): action_config_path = client.action_config_path if action_config_path: - config = load(open(action_config_path, 'rb')) + config = read_file(action_config_path) else: config = dict() config["default_action"] = client.default_file_action @@ -434,7 +435,7 @@ elif value is REQUIRED_ACTION_KWD: message_template = "action_type %s requires key word argument %s" message = message_template % (action_type, key) - raise Exception( message ) + raise Exception(message) self.action_type = action_type self.action_kwds = action_kwds path_types_str = config.get('path_types', "*defaults*") diff -r 0a428afbc23538b6f8d0a8d9dcc29b9a576c1ddc -r 45df9d2018e1c452478549b933476937595bf6a4 lib/galaxy/jobs/runners/lwr_client/amqp_exchange.py --- a/lib/galaxy/jobs/runners/lwr_client/amqp_exchange.py +++ b/lib/galaxy/jobs/runners/lwr_client/amqp_exchange.py @@ -73,7 +73,8 @@ except (IOError, socket.error), exc: # In testing, errno is None log.warning('Got %s, will retry: %s', exc.__class__.__name__, exc) - heartbeat_thread.join() + if heartbeat_thread: + heartbeat_thread.join() def heartbeat(self, connection): log.debug('AMQP heartbeat thread alive') diff -r 0a428afbc23538b6f8d0a8d9dcc29b9a576c1ddc -r 45df9d2018e1c452478549b933476937595bf6a4 lib/galaxy/jobs/runners/lwr_client/config_util.py --- /dev/null +++ b/lib/galaxy/jobs/runners/lwr_client/config_util.py @@ -0,0 +1,80 @@ +""" Generic interface for reading YAML/INI/JSON config files into nested dictionaries. +""" + +try: + from galaxy import eggs + eggs.require('PyYAML') +except Exception: + # If not in Galaxy, ignore this. + pass +try: + import yaml +except ImportError: + yaml = None +try: + from ConfigParser import ConfigParser +except ImportError: + from configparser import ConfigParser +import json + + +CONFIG_TYPE_JSON = "json" +CONFIG_TYPE_YAML = "yaml" +CONFIG_TYPE_INI = "ini" + +DEFAULT_CONFIG_TYPE = CONFIG_TYPE_YAML + +JSON_EXTS = [".json"] +YAML_EXTS = [".yaml", ".yml"] +INI_EXTS = [".ini"] + +EXT_MAP = { + CONFIG_TYPE_JSON: JSON_EXTS, + CONFIG_TYPE_YAML: YAML_EXTS, + CONFIG_TYPE_INI: INI_EXTS, +} + + +def read_file(path, type=None, default_type=DEFAULT_CONFIG_TYPE): + if path is None: + raise ValueError("Undefined path supplied.") + + config_type = __find_type(path, type, default_type) + return EXT_READERS[config_type](path) + + +def __find_type(path, explicit_type, default_type): + if explicit_type: + return explicit_type + + for config_type, config_exts in EXT_MAP.items(): + for ext in config_exts: + if path.endswith(ext): + return config_type + + return default_type + + +def __read_yaml(path): + if yaml is None: + raise ImportError("Attempting to read YAML configuration file - but PyYAML dependency unavailable.") + + with open(path, "rb") as f: + return yaml.load(f) + + +def __read_ini(path): + config = ConfigParser() + config.read(path) + return config._sections + + +def __read_json(path): + with open(path, "rb") as f: + return json.load(f) + +EXT_READERS = { + CONFIG_TYPE_JSON: __read_json, + CONFIG_TYPE_YAML: __read_yaml, + CONFIG_TYPE_INI: __read_ini, +} diff -r 0a428afbc23538b6f8d0a8d9dcc29b9a576c1ddc -r 45df9d2018e1c452478549b933476937595bf6a4 lib/galaxy/jobs/runners/lwr_client/transport/curl.py --- a/lib/galaxy/jobs/runners/lwr_client/transport/curl.py +++ b/lib/galaxy/jobs/runners/lwr_client/transport/curl.py @@ -10,7 +10,7 @@ PYCURL_UNAVAILABLE_MESSAGE = \ - "You are attempting to use the Pycurl version of the LWR client by pycurl is unavailable." + "You are attempting to use the Pycurl version of the LWR client but pycurl is unavailable." class PycurlTransport(object): diff -r 0a428afbc23538b6f8d0a8d9dcc29b9a576c1ddc -r 45df9d2018e1c452478549b933476937595bf6a4 lib/galaxy/jobs/runners/lwr_client/util.py --- a/lib/galaxy/jobs/runners/lwr_client/util.py +++ b/lib/galaxy/jobs/runners/lwr_client/util.py @@ -135,7 +135,7 @@ message_template = "Cannot compute new path for file %s, does not start with %s." message = message_template % (posix_path, old_base) raise Exception(message) - stripped_path = posix_path[ len(old_base): ] + stripped_path = posix_path[len(old_base):] while stripped_path.startswith("/"): stripped_path = stripped_path[1:] path_parts = stripped_path.split(self.separator) diff -r 0a428afbc23538b6f8d0a8d9dcc29b9a576c1ddc -r 45df9d2018e1c452478549b933476937595bf6a4 lib/galaxy/jobs/runners/util/cli/job/slurm.py --- a/lib/galaxy/jobs/runners/util/cli/job/slurm.py +++ b/lib/galaxy/jobs/runners/util/cli/job/slurm.py @@ -1,4 +1,4 @@ -# A simple CLI runner for slurm that can be used when running Galaxy from a +# A simple CLI runner for slurm that can be used when running Galaxy from a # non-submit host and using a Slurm cluster. try: @@ -22,6 +22,7 @@ 'partition': '-p' } + class Slurm(BaseJobExec): def __init__(self, **params): @@ -31,8 +32,8 @@ def job_script_kwargs(self, ofile, efile, job_name): scriptargs = {'-o': ofile, - '-e': efile, - '-J': job_name} + '-e': efile, + '-J': job_name} # Map arguments using argmap. for k, v in self.params.items(): @@ -66,7 +67,7 @@ def parse_status(self, status, job_ids): # Get status for each job, skipping header. rval = {} - for line in status.splitlines()[ 1: ]: + for line in status.splitlines()[1:]: id, state = line.split() if id in job_ids: # map job states to Galaxy job states. @@ -75,7 +76,7 @@ def parse_single_status(self, status, job_id): status = status.splitlines() - if len( status ) > 1: + if len(status) > 1: # Job still on cluster and has state. id, state = status[1].split() return self._get_job_state(state) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.