Hi all, We had a problem on the NAS where we couldn't write anymore. This problem was solved and I decided that I would profit to upgrade galaxy. Now we have a problem where jobs do not get launched and the handler log shows a lot of error messages. The first error message is copied below. Afterwards it alternates between galaxy.jobs.handler ERROR 2015-04-07 15:58:24,959 Exception in monitor_step galaxy.jobs.runners ERROR 2015-04-07 15:58:23,542 Unhandled exception checking active jobs with, mainly, the same error stack (second error message shown below). Any idea what is going wrong? My first impression is that it is trying to recuperate some job that galaxy believes is still running but the machine has been stopped and rebooted in the mean time. Any help is welcome! Cristian
galaxy.jobs.runners ERROR 2015-04-07 15:44:04,809 (27764) Unhandled exception calling queue_jobTraceback (most recent call last): File "/home/galaxy/galaxy-dist/lib/galaxy/jobs/runners/__init__.py", line 97, in run_next method(arg) File "/home/galaxy/galaxy-dist/lib/galaxy/jobs/runners/drmaa.py", line 115, in queue_job if not self.prepare_job( job_wrapper, include_metadata=True ): File "/home/galaxy/galaxy-dist/lib/galaxy/jobs/runners/__init__.py", line 140, in prepare_job job_id = job_wrapper.get_id_tag() File "/home/galaxy/galaxy-dist/lib/galaxy/jobs/__init__.py", line 794, in get_id_tag return self.get_job().get_id_tag() File "/home/galaxy/galaxy-dist/lib/galaxy/jobs/__init__.py", line 790, in get_job return self.sa_session.query( model.Job ).get( self.job_id ) File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/query.py", line 840, in get return loading.load_on_ident(self, key) File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/loading.py", line 231, in load_on_ident return q.one() File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/query.py", line 2395, in one ret = list(self) File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/query.py", line 2434, in __iter__ context = self._compile_context() File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/query.py", line 2810, in _compile_context entity.setup_context(self, context) File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/query.py", line 3194, in setup_context column_collection=context.primary_columns File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/interfaces.py", line 466, in setup strat.setup_query(context, entity, path, loader, adapter, **kwargs) File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/strategies.py", line 1111, in setup_query column_collection, parentmapper, chained_from_outerjoin File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/strategies.py", line 1234, in _generate_row_adapter use_mapper_path=True) File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/util.py", line 364, in __init__ else mapper.with_polymorphic_mappers, File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/util/langhelpers.py", line 725, in __get__ obj.__dict__[self.__name__] = result = self.fget(obj) File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/mapper.py", line 1877, in _with_polymorphic_mappers configure_mappers() File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/mapper.py", line 2589, in configure_mappers mapper._post_configure_properties() File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/mapper.py", line 1694, in _post_configure_properties prop.init() File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/interfaces.py", line 144, in init self.do_init() File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/relationships.py", line 1549, in do_init self._process_dependent_arguments() File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/relationships.py", line 1605, in _process_dependent_arguments self.target = self.mapper.mapped_table File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/util/langhelpers.py", line 725, in __get__ obj.__dict__[self.__name__] = result = self.fget(obj) File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/relationships.py", line 1522, in mapper argument = self.argument() File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/ext/declarative/clsregistry.py", line 283, in __call__ (self.prop.parent, self.arg, n.args[0], self.cls)InvalidRequestError: When initializing mapper Mapper|Queue|kombu_queue, expression 'Message' failed to locate a name ("name 'Message' is not defined"). If this is a class name, consider adding this relationship() to the <class 'kombu.transport.sqlalchemy.Queue'> class after both dependent classes have been defined. Repeated error messages:galaxy.jobs.runners ERROR 2015-04-07 15:58:23,542 Unhandled exception checking active jobsTraceback (most recent call last): File "/home/galaxy/galaxy-dist/lib/galaxy/jobs/runners/__init__.py", line 501, in monitor self.check_watched_items() File "/home/galaxy/galaxy-dist/lib/galaxy/jobs/runners/drmaa.py", line 238, in check_watched_items galaxy_id_tag = ajs.job_wrapper.get_id_tag() File "/home/galaxy/galaxy-dist/lib/galaxy/jobs/__init__.py", line 794, in get_id_tag return self.get_job().get_id_tag() File "/home/galaxy/galaxy-dist/lib/galaxy/jobs/__init__.py", line 790, in get_job return self.sa_session.query( model.Job ).get( self.job_id ) File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/scoping.py", line 150, in do return getattr(self.registry(), name)(*args, **kwargs) File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/session.py", line 1165, in query return self._query_cls(entities, self, **kwargs) File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/query.py", line 108,in __init__ self._set_entities(entities) File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/query.py", line 118,in _set_entities self._set_entity_selectables(self._entities) File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/query.py", line 151,in _set_entity_selectables ent.setup_entity(*d[entity]) File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/query.py", line 3036, in setup_entity self._with_polymorphic = ext_info.with_polymorphic_mappers File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/util/langhelpers.py", line 725, in __get__ obj.__dict__[self.__name__] = result = self.fget(obj) File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/mapper.py", line 1877, in _with_polymorphic_mappers configure_mappers() File "/home/galaxy/galaxy-dist/eggs/SQLAlchemy-0.9.8-py2.7-linux-x86_64-ucs4.egg/sqlalchemy/orm/mapper.py", line 2586, in configure_mappers raise eInvalidRequestError: One or more mappers failed to initialize - can't proceed with initialization of other mappers. Original exception was: When initializing mapper Mapper|Queue|kombu_queue, expression 'Message' failed to locate a name ("name 'Message' is not defined"). If this is a class name, consider adding this relationship() to the <class 'kombu.transport.sqlalchemy.Queue'> class after both dependent classes have been defined.