1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/752929cea6d3/ Changeset: 752929cea6d3 User: jmchilton Date: 2013-12-01 13:14:32 Summary: Update LWR client through LWR revision acecc74f7016. Mostly small modifications related Python 3 compat. (no iteritems, diff. StringIO, exception syntax, urllib diffs,) and improved handling of unicode data. Affected #: 6 files diff -r 9a5a8d18cd16ed3331ba996de21ed4d4dfa93da9 -r 752929cea6d37423c14f46f317ef3815cfc3c3d0 lib/galaxy/jobs/runners/lwr_client/action_mapper.py --- a/lib/galaxy/jobs/runners/lwr_client/action_mapper.py +++ b/lib/galaxy/jobs/runners/lwr_client/action_mapper.py @@ -21,7 +21,7 @@ >>> from tempfile import NamedTemporaryFile >>> from os import unlink >>> f = NamedTemporaryFile(delete=False) - >>> f.write(json_string) + >>> write_result = f.write(json_string.encode('UTF-8')) >>> f.close() >>> class MockClient(): ... default_file_action = 'none' @@ -30,23 +30,23 @@ >>> mapper = FileActionMapper(MockClient()) >>> unlink(f.name) >>> # Test first config line above, implicit path prefix mapper - >>> mapper.action('/opt/galaxy/tools/filters/catWrapper.py', 'input') - ('none',) + >>> mapper.action('/opt/galaxy/tools/filters/catWrapper.py', 'input')[0] == u'none' + True >>> # Test another (2nd) mapper, this one with a different action - >>> mapper.action('/galaxy/data/files/000/dataset_1.dat', 'input') - ('transfer',) + >>> mapper.action('/galaxy/data/files/000/dataset_1.dat', 'input')[0] == u'transfer' + True >>> # Always at least copy work_dir outputs. - >>> mapper.action('/opt/galaxy/database/working_directory/45.sh', 'work_dir') - ('copy',) + >>> mapper.action('/opt/galaxy/database/working_directory/45.sh', 'work_dir')[0] == u'copy' + True >>> # Test glob mapper (matching test) - >>> mapper.action('/cool/bamfiles/projectABC/study1/patient3.bam', 'input') - ('copy',) + >>> mapper.action('/cool/bamfiles/projectABC/study1/patient3.bam', 'input')[0] == u'copy' + True >>> # Test glob mapper (non-matching test) - >>> mapper.action('/cool/bamfiles/projectABC/study1/patient3.bam.bai', 'input') - ('none',) + >>> mapper.action('/cool/bamfiles/projectABC/study1/patient3.bam.bai', 'input')[0] == u'none' + True >>> # Regex mapper test. - >>> mapper.action('/old/galaxy/data/dataset_10245.dat', 'input') - ('copy',) + >>> mapper.action('/old/galaxy/data/dataset_10245.dat', 'input')[0] == u'copy' + True """ def __init__(self, client): diff -r 9a5a8d18cd16ed3331ba996de21ed4d4dfa93da9 -r 752929cea6d37423c14f46f317ef3815cfc3c3d0 lib/galaxy/jobs/runners/lwr_client/destination.py --- a/lib/galaxy/jobs/runners/lwr_client/destination.py +++ b/lib/galaxy/jobs/runners/lwr_client/destination.py @@ -51,9 +51,10 @@ >>> destination_params = {"private_token": "12345", "submit_native_specification": "-q batch"} >>> result = submit_params(destination_params) - >>> result.items() - [('native_specification', '-q batch')] + >>> result + {'native_specification': '-q batch'} """ - return dict([(key[len(SUBMIT_PREFIX):], value) - for key, value in (destination_params or {}).iteritems() + destination_params = destination_params or {} + return dict([(key[len(SUBMIT_PREFIX):], destination_params[key]) + for key in destination_params if key.startswith(SUBMIT_PREFIX)]) diff -r 9a5a8d18cd16ed3331ba996de21ed4d4dfa93da9 -r 752929cea6d37423c14f46f317ef3815cfc3c3d0 lib/galaxy/jobs/runners/lwr_client/manager.py --- a/lib/galaxy/jobs/runners/lwr_client/manager.py +++ b/lib/galaxy/jobs/runners/lwr_client/manager.py @@ -5,8 +5,18 @@ from queue import Queue from threading import Thread from os import getenv -from urllib import urlencode -from StringIO import StringIO +try: + from urllib import urlencode +except ImportError: + from urllib.parse import urlencode +try: + from StringIO import StringIO as BytesIO +except ImportError: + from io import BytesIO +try: + from six import text_type +except ImportError: + from galaxy.util import unicodify as text_type from .client import Client, InputCachingClient from .transport import get_transport @@ -27,10 +37,10 @@ """ def __init__(self, **kwds): if 'job_manager' in kwds: - self.job_manager_interface_class = LocalJobManagerInterface + self.job_manager_interface_class = LocalLwrInterface self.job_manager_interface_args = dict(job_manager=kwds['job_manager'], file_cache=kwds['file_cache']) else: - self.job_manager_interface_class = HttpJobManagerInterface + self.job_manager_interface_class = HttpLwrInterface transport_type = kwds.get('transport_type', None) transport = get_transport(transport_type) self.job_manager_interface_args = dict(transport=transport) @@ -55,7 +65,11 @@ return self.client_class(destination_params, job_id, job_manager_interface, **self.extra_client_kwds) def __parse_destination_params(self, destination_params): - if isinstance(destination_params, str) or isinstance(destination_params, unicode): + try: + unicode_type = unicode + except NameError: + unicode_type = str + if isinstance(destination_params, str) or isinstance(destination_params, unicode_type): destination_params = url_to_destination_params(destination_params) return destination_params @@ -76,7 +90,7 @@ """ -class HttpJobManagerInterface(object): +class HttpLwrInterface(object): def __init__(self, destination_params, transport): self.transport = transport @@ -92,12 +106,13 @@ def __build_url(self, command, args): if self.private_key: args["private_key"] = self.private_key - data = urlencode(args) + arg_bytes = dict([(k, text_type(args[k]).encode('utf-8')) for k in args]) + data = urlencode(arg_bytes) url = self.remote_host + command + "?" + data return url -class LocalJobManagerInterface(object): +class LocalLwrInterface(object): def __init__(self, destination_params, job_manager, file_cache): self.job_manager = job_manager @@ -113,6 +128,7 @@ } def execute(self, command, args={}, data=None, input_path=None, output_path=None): + # If data set, should be unicode (on Python 2) or str (on Python 3). from lwr import routes from lwr.framework import build_func_args controller = getattr(routes, command) @@ -129,9 +145,9 @@ def __build_body(self, data, input_path): if data is not None: - return StringIO(data) + return BytesIO(data.encode('utf-8')) elif input_path is not None: - return open(input_path, 'r') + return open(input_path, 'rb') else: return None @@ -188,4 +204,4 @@ int_val = int(val) return int_val -__all__ = [ClientManager, HttpJobManagerInterface] +__all__ = [ClientManager, HttpLwrInterface] diff -r 9a5a8d18cd16ed3331ba996de21ed4d4dfa93da9 -r 752929cea6d37423c14f46f317ef3815cfc3c3d0 lib/galaxy/jobs/runners/lwr_client/stager.py --- a/lib/galaxy/jobs/runners/lwr_client/stager.py +++ b/lib/galaxy/jobs/runners/lwr_client/stager.py @@ -1,6 +1,7 @@ from os.path import abspath, basename, join, exists from os import listdir, sep from re import findall +from io import open from .action_mapper import FileActionMapper @@ -24,22 +25,24 @@ >>> import tempfile >>> tf = tempfile.NamedTemporaryFile() >>> def setup_inputs(tf): - ... open(tf.name, "w").write("world /path/to/input the rest") - ... inputs = JobInputs("hello /path/to/input", [tf.name]) + ... open(tf.name, "w").write(u"world /path/to/input the rest") + ... inputs = JobInputs(u"hello /path/to/input", [tf.name]) ... return inputs >>> inputs = setup_inputs(tf) - >>> inputs.rewrite_paths("/path/to/input", 'C:\\input') - >>> inputs.rewritten_command_line - 'hello C:\\\\input' - >>> inputs.rewritten_config_files[tf.name] - 'world C:\\\\input the rest' + >>> inputs.rewrite_paths(u"/path/to/input", u'C:\\input') + >>> inputs.rewritten_command_line == u'hello C:\\\\input' + True + >>> inputs.rewritten_config_files[tf.name] == u'world C:\\\\input the rest' + True >>> tf.close() >>> tf = tempfile.NamedTemporaryFile() >>> inputs = setup_inputs(tf) - >>> inputs.find_referenced_subfiles('/path/to') - ['/path/to/input'] + >>> inputs.find_referenced_subfiles('/path/to') == [u'/path/to/input'] + True >>> inputs.path_referenced('/path/to') True + >>> inputs.path_referenced(u'/path/to') + True >>> inputs.path_referenced('/path/to/input') True >>> inputs.path_referenced('/path/to/notinput') @@ -92,7 +95,7 @@ self.rewritten_command_line = self.rewritten_command_line.replace(local_path, remote_path) def __rewrite_config_files(self, local_path, remote_path): - for config_file, rewritten_contents in self.rewritten_config_files.iteritems(): + for config_file, rewritten_contents in self.rewritten_config_files.items(): self.rewritten_config_files[config_file] = rewritten_contents.replace(local_path, remote_path) def __items(self): @@ -140,7 +143,7 @@ For each file that has been transferred and renamed, updated command_line and configfiles to reflect that rewrite. """ - for local_path, remote_path in self.file_renames.iteritems(): + for local_path, remote_path in self.file_renames.items(): self.job_inputs.rewrite_paths(local_path, remote_path) def __action(self, path, type): @@ -283,7 +286,7 @@ self.transfer_tracker.rewrite_input_paths() def __upload_rewritten_config_files(self): - for config_file, new_config_contents in self.job_inputs.rewritten_config_files.iteritems(): + for config_file, new_config_contents in self.job_inputs.rewritten_config_files.items(): self.client.put_file(config_file, input_type='config', contents=new_config_contents) def get_rewritten_command_line(self): @@ -304,7 +307,7 @@ try: action = action_mapper.action(output_file, 'output') client.fetch_work_dir_output(source_file, working_directory, output_file, action[0]) - except Exception, e: + except Exception as e: download_failure_exceptions.append(e) # Remove from full output_files list so don't try to download directly. output_files.remove(output_file) @@ -312,7 +315,7 @@ try: action = action_mapper.action(output_file, 'output') client.fetch_output(output_file, working_directory=working_directory, action=action[0]) - except Exception, e: + except Exception as e: download_failure_exceptions.append(e) return __clean(download_failure_exceptions, cleanup_job, client) @@ -340,9 +343,9 @@ def _read(path): """ Utility method to quickly read small files (config files and tool - wrappers) into memory as strings. + wrappers) into memory as bytes. """ - input = open(path, "r") + input = open(path, "r", encoding="utf-8") try: return input.read() finally: diff -r 9a5a8d18cd16ed3331ba996de21ed4d4dfa93da9 -r 752929cea6d37423c14f46f317ef3815cfc3c3d0 lib/galaxy/jobs/runners/lwr_client/transport/curl.py --- a/lib/galaxy/jobs/runners/lwr_client/transport/curl.py +++ b/lib/galaxy/jobs/runners/lwr_client/transport/curl.py @@ -1,4 +1,7 @@ -from cStringIO import StringIO +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO try: from pycurl import Curl except: @@ -25,6 +28,8 @@ c.setopt(c.INFILESIZE, filesize) if data: c.setopt(c.POST, 1) + if type(data).__name__ == 'unicode': + data = data.encode('UTF-8') c.setopt(c.POSTFIELDS, data) c.perform() if not output_path: diff -r 9a5a8d18cd16ed3331ba996de21ed4d4dfa93da9 -r 752929cea6d37423c14f46f317ef3815cfc3c3d0 lib/galaxy/jobs/runners/lwr_client/transport/standard.py --- a/lib/galaxy/jobs/runners/lwr_client/transport/standard.py +++ b/lib/galaxy/jobs/runners/lwr_client/transport/standard.py @@ -3,16 +3,23 @@ """ from __future__ import with_statement import mmap -import urllib2 +try: + from urllib2 import urlopen +except ImportError: + from urllib.request import urlopen +try: + from urllib2 import Request +except ImportError: + from urllib.request import Request class Urllib2Transport(object): def _url_open(self, request, data): - return urllib2.urlopen(request, data) + return urlopen(request, data) def execute(self, url, data=None, input_path=None, output_path=None): - request = urllib2.Request(url=url, data=data) + request = Request(url=url, data=data) input = None try: if input_path: @@ -26,7 +33,7 @@ with open(output_path, 'wb') as output: while True: buffer = response.read(1024) - if buffer == "": + if not buffer: break output.write(buffer) return response Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.