Hi Galaxy Team, I sent this query last week and was wondering if anyone new anything about this error. I closed and re-started Galaxy to get it going again, however I have had to do this twice this week whenever we have multiple people using the server (3) and one is running a large workflow (15 steps).
Also, we are having an additional problem when trying to convert large tabular files to interval. The file is over 12 million lines, we are getting a server time out error as I assume Galaxy tries to inspect the entire file before changing format.
Thanks Shaun Webb
Hi, our Galaxy server is showing a timeout error:
TimeoutError: QueuePool limit of size 10 overflow 20 reached, connection timed out, timeout 30
I assume this is due to overuse as it is consistent with the database settings in the universe.wsgi file.
First of all, is there anyway to view and cancel queued jobs? I usually use the admin page but this is not accessible at the moment.
Secondly, we plan for Galaxy to be used on a large scale, is there anyway of preventing this type of error in future?
I've included the full traceback below.
Thanks in advance for any help Shaun Webb
URL: http://bifx3.bio.ed.ac.uk:8080/admin Module paste.exceptions.errormiddleware:143 in __call__ << try: __traceback_supplement__ = Supplement, self, environ app_iter = self.application(environ, start_response) return self.make_catching_iter(app_iter, environ) except:>> app_iter = self.application(environ, start_response) Module paste.debug.prints:98 in __call__ << try: status, headers, body = wsgilib.intercept_output( environ, self.app) if status is None: # Some error occurred >> environ, self.app) Module paste.wsgilib:539 in intercept_output << data.append(headers) return output.write app_iter = application(environ, replacement_start_response) if data[0] is None: return (None, None, app_iter)>> app_iter = application(environ, replacement_start_response) Module paste.recursive:80 in __call__ << environ['paste.recursive.script_name'] = my_script_name try: return self.application(environ, start_response) except ForwardRequestException, e: middleware = CheckForRecursionMiddleware(>> return self.application(environ, start_response) Module paste.httpexceptions:632 in __call__ << []).append(HTTPException) try: return self.application(environ, start_response) except HTTPException, exc: return exc(environ, start_response)>> return self.application(environ, start_response) Module galaxy.web.framework.base:99 in __call__ << rc.environ = environ # Setup the transaction trans = self.transaction_factory( environ ) rc.redirect = trans.response.send_redirect # Get the controller class >> trans = self.transaction_factory( environ ) Module galaxy.web.framework:110 in <lambda> << def __init__( self, galaxy_app, session_cookie='galaxysession' ): base.WebApplication.__init__( self ) self.set_transaction_factory( lambda e: UniverseWebTransaction( e, galaxy_app, self, session_cookie ) ) # Mako support self.mako_template_lookup = mako.lookup.TemplateLookup(>> self.set_transaction_factory( lambda e: UniverseWebTransaction( e, galaxy_app, self, session_cookie ) ) Module galaxy.web.framework:150 in __init__ << self.workflow_building_mode = False # Always have a valid galaxy session self.__ensure_valid_session( session_cookie ) # Prevent deleted users from accessing Galaxy if self.app.config.use_remote_user and self.galaxy_session.user.deleted:>> self.__ensure_valid_session( session_cookie ) Module galaxy.web.framework:279 in __ensure_valid_session << galaxy_session = self.sa_session.query( self.app.model.GalaxySession ) \ .filter( and_( self.app.model.GalaxySession.table.c.session_key==session_key,
self.app.model.GalaxySession.table.c.is_valid==True ) ) \ .first() # If remote user is in use it can invalidate the session, so we need to to check some things now. >> self.app.model.GalaxySession.table.c.is_valid==True ) ) Module sqlalchemy.orm.query:1300 in first << ret = list(self)[0:1] else: ret = list(self[0:1]) if len(ret) > 0: return ret[0]>> ret = list(self[0:1]) Module sqlalchemy.orm.query:1221 in __getitem__ << return list(res)[None:None:item.step] else: return list(res) else: return list(self[item:item+1])[0]>> return list(res) Module sqlalchemy.orm.query:1361 in __iter__ << if self._autoflush and not self._populate_existing: self.session._autoflush() return self._execute_and_instances(context)
def _execute_and_instances(self, querycontext):>> return self._execute_and_instances(context) Module sqlalchemy.orm.query:1364 in _execute_and_instances << def _execute_and_instances(self, querycontext): result = self.session.execute(querycontext.statement, params=self._params, mapper=self._mapper_zero_or_none()) return self.instances(result, querycontext)>> result = self.session.execute(querycontext.statement, params=self._params, mapper=self._mapper_zero_or_none()) Module sqlalchemy.orm.session:754 in execute << engine = self.get_bind(mapper, clause=clause, **kw)
return self.__connection(engine, close_with_result=True).execute( clause, params or {})>> return self.__connection(engine, close_with_result=True).execute( Module sqlalchemy.orm.session:721 in __connection << return self.transaction._connection_for_bind(engine) else: return engine.contextual_connect(**kwargs)
def execute(self, clause, params=None, mapper=None, **kw):>> return engine.contextual_connect(**kwargs) Module sqlalchemy.engine.base:1229 in contextual_connect << """
return self.Connection(self, self.pool.connect(), close_with_result=close_with_result, **kwargs)
def table_names(self, schema=None, connection=None):>> return self.Connection(self, self.pool.connect(), close_with_result=close_with_result, **kwargs) Module sqlalchemy.pool:142 in connect << def connect(self): if not self._use_threadlocal: return _ConnectionFairy(self).checkout()
try:>> return _ConnectionFairy(self).checkout() Module sqlalchemy.pool:304 in __init__ << self.__counter = 0 try: rec = self._connection_record = pool.get() conn = self.connection = self._connection_record.get_connection() self._connection_record.backref = weakref.ref(self, lambda ref:_finalize_fairy(conn, rec, pool, ref))>> rec = self._connection_record = pool.get() Module sqlalchemy.pool:161 in get << def get(self): return self.do_get()
def do_get(self):>> return self.do_get() Module sqlalchemy.pool:628 in do_get << return self.do_get() else: raise exc.TimeoutError("QueuePool limit of size %d overflow %d reached, connection timed out, timeout %d" % (self.size(), self.overflow(), self._timeout))
if self._overflow_lock is not None:>> raise exc.TimeoutError("QueuePool limit of size %d overflow %d reached, connection timed out, timeout %d" % (self.size(), self.overflow(), self._timeout)) TimeoutError: QueuePool limit of size 10 overflow 20 reached, connection timed out, timeout 30