1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/702004fc9ca2/ Changeset: 702004fc9ca2 User: jmchilton Date: 2014-06-09 23:58:10 Summary: More verbose and defensive exception handling when responding to LWR job updates. Handler becoming non-responsive in some cases - this may fix it or there may be a race condition somewhere when caching status messages. Affected #: 2 files diff -r 2006b8020cb2b3908dc186fc02bf0635a232bffb -r 702004fc9ca216594ba7c88deb1153cafd37a573 lib/galaxy/jobs/runners/lwr.py --- a/lib/galaxy/jobs/runners/lwr.py +++ b/lib/galaxy/jobs/runners/lwr.py @@ -153,10 +153,16 @@ return job_state def __async_update( self, full_status ): - job_id = full_status[ "job_id" ] - job, job_wrapper = self.app.job_manager.job_handler.job_queue.job_pair_for_id( job_id ) - job_state = self.__job_state( job, job_wrapper ) - self.__update_job_state_for_lwr_status(job_state, full_status["status"]) + job_id = None + try: + job_id = full_status[ "job_id" ] + job, job_wrapper = self.app.job_manager.job_handler.job_queue.job_pair_for_id( job_id ) + job_state = self.__job_state( job, job_wrapper ) + self.__update_job_state_for_lwr_status(job_state, full_status["status"]) + except Exception: + log.exception( "Failed to update LWR job status for job_id %s" % job_id ) + raise + # Nothing else to do? - Attempt to fail the job? def queue_job(self, job_wrapper): job_destination = job_wrapper.job_destination diff -r 2006b8020cb2b3908dc186fc02bf0635a232bffb -r 702004fc9ca216594ba7c88deb1153cafd37a573 lib/galaxy/jobs/runners/lwr_client/manager.py --- a/lib/galaxy/jobs/runners/lwr_client/manager.py +++ b/lib/galaxy/jobs/runners/lwr_client/manager.py @@ -103,7 +103,10 @@ callback(body) except Exception: log.exception("Failure processing job status update message.") - message.ack() + except BaseException as e: + log.exception("Failure processing job status update message - BaseException type %s" % type(e)) + finally: + message.ack() def run(): self.exchange.consume("status_update", callback_wrapper, check=self) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.