[galaxy-commits] commit/galaxy-central: 2 new changesets

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e69024c183a7/ Changeset: e69024c183a7 User: jmchilton Date: 2014-05-29 07:15:38 Summary: Unit tests for RunnerParams. Small changes to make this easier - mostly just defining constants. Affected #: 2 files diff -r cc0f9182fcb0b18730dd79c6889de4c8a58d4446 -r e69024c183a7403bae7b0801faa8f0db5fc1e5c0 lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -24,20 +24,27 @@ STOP_SIGNAL = object() +JOB_RUNNER_PARAMETER_UNKNOWN_MESSAGE = "Invalid job runner parameter for this plugin: %s" +JOB_RUNNER_PARAMETER_MAP_PROBLEM_MESSAGE = "Job runner parameter '%s' value '%s' could not be converted to the correct type" +JOB_RUNNER_PARAMETER_VALIDATION_FAILED_MESSAGE = "Job runner parameter %s failed validation" + + class RunnerParams( object ): def __init__( self, specs=None, params=None ): self.specs = specs or dict() self.params = params or dict() for name, value in self.params.items(): - assert name in self.specs, 'Invalid job runner parameter for this plugin: %s' % name + assert name in self.specs, JOB_RUNNER_PARAMETER_UNKNOWN_MESSAGE % name if 'map' in self.specs[ name ]: try: self.params[ name ] = self.specs[ name ][ 'map' ]( value ) - except Exception, e: - raise Exception( 'Job runner parameter "%s" value "%s" could not be converted to the correct type: %s' % ( name, value, e ) ) + except Exception: + message = JOB_RUNNER_PARAMETER_MAP_PROBLEM_MESSAGE % ( name, value ) + log.exception(message) + raise Exception( message ) if 'valid' in self.specs[ name ]: - assert self.specs[ name ][ 'valid' ]( value ), 'Job runner parameter %s failed validation' % name + assert self.specs[ name ][ 'valid' ]( value ), JOB_RUNNER_PARAMETER_VALIDATION_FAILED_MESSAGE % name def __getattr__( self, name ): return self.params.get( name, self.specs[ name ][ 'default' ] ) @@ -46,13 +53,15 @@ class BaseJobRunner( object ): + DEFAULT_SPECS = dict( recheck_missing_job_retries=dict( map=int, valid=lambda x: x >= 0, default=0 ) ) + def __init__( self, app, nworkers, **kwargs ): """Start the job runner """ self.app = app self.sa_session = app.model.context self.nworkers = nworkers - runner_param_specs = dict( recheck_missing_job_retries=dict( map=int, valid=lambda x: x >= 0, default=0 ) ) + runner_param_specs = self.DEFAULT_SPECS.copy() if 'runner_param_specs' in kwargs: runner_param_specs.update( kwargs.pop( 'runner_param_specs' ) ) if kwargs: diff -r cc0f9182fcb0b18730dd79c6889de4c8a58d4446 -r e69024c183a7403bae7b0801faa8f0db5fc1e5c0 test/unit/jobs/test_runner_params.py --- /dev/null +++ b/test/unit/jobs/test_runner_params.py @@ -0,0 +1,48 @@ +from galaxy.jobs import runners + + +def test_default_specs(): + # recheck_missing_job_retries is integer >= 0 + params = runners.RunnerParams( specs=runners.BaseJobRunner.DEFAULT_SPECS, params=dict( recheck_missing_job_retries="1" ) ) + assert params.recheck_missing_job_retries == 1 + assert params["recheck_missing_job_retries"] == 1 + + exception_raised = False + try: + runners.RunnerParams( specs=runners.BaseJobRunner.DEFAULT_SPECS, params=dict( recheck_missing_job_retries=-1 ) ) + except Exception: + exception_raised = True + assert exception_raised + + +def test_missing_parameter(): + exception = None + try: + runners.RunnerParams( specs={}, params=dict( foo="bar" ) ) + except Exception as e: + exception = e + assert exception.message == runners.JOB_RUNNER_PARAMETER_UNKNOWN_MESSAGE % "foo" + + +def test_invalid_parameter(): + exception = None + try: + runners.RunnerParams( specs=dict( foo=dict( valid=lambda x: x != "bar", defualt="baz" ) ), params=dict( foo="bar" ) ) + except Exception as e: + exception = e + assert exception.message == runners.JOB_RUNNER_PARAMETER_VALIDATION_FAILED_MESSAGE % "foo" + + +def test_map_problem(): + exception = None + try: + runners.RunnerParams( specs=dict( foo=dict( map=lambda x: 1 / 0, default="baz" ) ), params=dict( foo="bar" ) ) + except Exception as e: + exception = e + assert exception.message == runners.JOB_RUNNER_PARAMETER_MAP_PROBLEM_MESSAGE % ( "foo", "bar" ) + + +def test_param_default(): + runner_params = runners.RunnerParams( specs=dict( foo=dict( default="baz" ) ), params={} ) + assert runner_params["foo"] == "baz" + assert runner_params.foo == "baz" https://bitbucket.org/galaxy/galaxy-central/commits/34d179b55c85/ Changeset: 34d179b55c85 User: jmchilton Date: 2014-05-29 07:15:38 Summary: Generalize RunnerParams. Create ParamsWithSpecs class in galaxy.util. If we are going to build out specs for all LWR client parameters - would be nice if they could be reused on the LWR server side where relevant (e.g. AMQP connection parameters). Slightly modified implementation that extends collections.defaultdict so that operations like iterating over parameters, fetching keys, etc... are available so LWR runner param handling code can look like destination param handling code. Affected #: 2 files diff -r e69024c183a7403bae7b0801faa8f0db5fc1e5c0 -r 34d179b55c859f9c669c73733bde2684cf9d4a9f lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -16,6 +16,7 @@ from galaxy import model from galaxy.util import DATABASE_MAX_STRING_SIZE, shrink_stream_by_size from galaxy.util import in_directory +from galaxy.util import ParamsWithSpecs from galaxy.jobs.runners.util.job_script import job_script from galaxy.jobs.runners.util.env import env_to_statement @@ -29,27 +30,16 @@ JOB_RUNNER_PARAMETER_VALIDATION_FAILED_MESSAGE = "Job runner parameter %s failed validation" -class RunnerParams( object ): +class RunnerParams( ParamsWithSpecs ): - def __init__( self, specs=None, params=None ): - self.specs = specs or dict() - self.params = params or dict() - for name, value in self.params.items(): - assert name in self.specs, JOB_RUNNER_PARAMETER_UNKNOWN_MESSAGE % name - if 'map' in self.specs[ name ]: - try: - self.params[ name ] = self.specs[ name ][ 'map' ]( value ) - except Exception: - message = JOB_RUNNER_PARAMETER_MAP_PROBLEM_MESSAGE % ( name, value ) - log.exception(message) - raise Exception( message ) - if 'valid' in self.specs[ name ]: - assert self.specs[ name ][ 'valid' ]( value ), JOB_RUNNER_PARAMETER_VALIDATION_FAILED_MESSAGE % name + def _param_unknown_error( self, name ): + raise Exception( JOB_RUNNER_PARAMETER_UNKNOWN_MESSAGE % name ) - def __getattr__( self, name ): - return self.params.get( name, self.specs[ name ][ 'default' ] ) + def _param_map_error( self, name, value ): + raise Exception( JOB_RUNNER_PARAMETER_MAP_PROBLEM_MESSAGE % ( name, value ) ) - __getitem__ = __getattr__ + def _param_vaildation_error( self, name, value ): + raise Exception( JOB_RUNNER_PARAMETER_VALIDATION_FAILED_MESSAGE % name ) class BaseJobRunner( object ): diff -r e69024c183a7403bae7b0801faa8f0db5fc1e5c0 -r 34d179b55c859f9c669c73733bde2684cf9d4a9f lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -7,6 +7,7 @@ from __future__ import absolute_import import binascii +import collections import errno import grp import json @@ -693,6 +694,44 @@ def string_to_object( s ): return pickle.loads( binascii.unhexlify( s ) ) + +class ParamsWithSpecs( collections.defaultdict ): + """ + """ + + def __init__( self, specs=None, params=None ): + self.specs = specs or dict() + self.params = params or dict() + for name, value in self.params.items(): + if name not in self.specs: + self._param_unknown_error( name ) + if 'map' in self.specs[ name ]: + try: + self.params[ name ] = self.specs[ name ][ 'map' ]( value ) + except Exception: + self._param_map_error( name, value ) + if 'valid' in self.specs[ name ]: + if not self.specs[ name ][ 'valid' ]( value ): + self._param_vaildation_error( name, value ) + + self.update( self.params ) + + def __missing__( self, name ): + return self.specs[ name ][ 'default' ] + + def __getattr__( self, name ): + return self[ name ] + + def _param_unknown_error( self, name ): + raise NotImplementedError() + + def _param_map_error( self, name, value ): + raise NotImplementedError() + + def _param_vaildation_error( self, name, value ): + raise NotImplementedError() + + def compare_urls( url1, url2, compare_scheme=True, compare_hostname=True, compare_path=True ): url1 = urlparse( url1 ) url2 = urlparse( url2 ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.