Hi Shaun,
In universe_wsgi.ini, you can increase those values:
database_engine_option_pool_size = database_engine_option_max_overflow =
Although those values (10, 20) are what we use for Galaxy Main, so if you're running in to problems there, you may want to consider starting more instances and balancing the load.
Also, I assume you're not using SQLite, but if so, you'd definitely want to switch to Postgres or MySQL.
Regarding the metadata issues, the following config option could definitely alleviate those issues:
set_metadata_externally = True
This will cause metadata processing to occur as a separate job instead of in the web thread.
--nate
SHAUN WEBB wrote:
Hi Galaxy Team, I sent this query last week and was wondering if anyone new anything about this error. I closed and re-started Galaxy to get it going again, however I have had to do this twice this week whenever we have multiple people using the server (3) and one is running a large workflow (15 steps).
Also, we are having an additional problem when trying to convert large tabular files to interval. The file is over 12 million lines, we are getting a server time out error as I assume Galaxy tries to inspect the entire file before changing format.
Thanks Shaun Webb
Hi, our Galaxy server is showing a timeout error:
TimeoutError: QueuePool limit of size 10 overflow 20 reached, connection timed out, timeout 30
I assume this is due to overuse as it is consistent with the database settings in the universe.wsgi file.
First of all, is there anyway to view and cancel queued jobs? I usually use the admin page but this is not accessible at the moment.
Secondly, we plan for Galaxy to be used on a large scale, is there anyway of preventing this type of error in future?
I've included the full traceback below.
Thanks in advance for any help Shaun Webb
URL: http://bifx3.bio.ed.ac.uk:8080/admin Module paste.exceptions.errormiddleware:143 in __call__ << try: __traceback_supplement__ = Supplement, self, environ app_iter = self.application(environ, start_response) return self.make_catching_iter(app_iter, environ) except:>> app_iter = self.application(environ, start_response) Module paste.debug.prints:98 in __call__ << try: status, headers, body = wsgilib.intercept_output( environ, self.app) if status is None: # Some error occurred >> environ, self.app) Module paste.wsgilib:539 in intercept_output << data.append(headers) return output.write app_iter = application(environ, replacement_start_response) if data[0] is None: return (None, None, app_iter)>> app_iter = application(environ, replacement_start_response) Module paste.recursive:80 in __call__ << environ['paste.recursive.script_name'] = my_script_name try: return self.application(environ, start_response) except ForwardRequestException, e: middleware = CheckForRecursionMiddleware(>> return self.application(environ, start_response) Module paste.httpexceptions:632 in __call__ << []).append(HTTPException) try: return self.application(environ, start_response) except HTTPException, exc: return exc(environ, start_response)>> return self.application(environ, start_response) Module galaxy.web.framework.base:99 in __call__ << rc.environ = environ # Setup the transaction trans = self.transaction_factory( environ ) rc.redirect = trans.response.send_redirect # Get the controller class >> trans = self.transaction_factory( environ ) Module galaxy.web.framework:110 in <lambda> << def __init__( self, galaxy_app, session_cookie='galaxysession' ): base.WebApplication.__init__( self ) self.set_transaction_factory( lambda e: UniverseWebTransaction( e, galaxy_app, self, session_cookie ) ) # Mako support self.mako_template_lookup = mako.lookup.TemplateLookup(>> self.set_transaction_factory( lambda e: UniverseWebTransaction( e, galaxy_app, self, session_cookie ) ) Module galaxy.web.framework:150 in __init__ << self.workflow_building_mode = False # Always have a valid galaxy session self.__ensure_valid_session( session_cookie ) # Prevent deleted users from accessing Galaxy if self.app.config.use_remote_user and self.galaxy_session.user.deleted:>> self.__ensure_valid_session( session_cookie ) Module galaxy.web.framework:279 in __ensure_valid_session << galaxy_session = self.sa_session.query( self.app.model.GalaxySession ) \ .filter( and_( self.app.model.GalaxySession.table.c.session_key==session_key,
self.app.model.GalaxySession.table.c.is_valid==True ) ) \ .first() # If remote user is in use it can invalidate the session, so we need to to check some things now. >> self.app.model.GalaxySession.table.c.is_valid==True ) ) Module sqlalchemy.orm.query:1300 in first << ret = list(self)[0:1] else: ret = list(self[0:1]) if len(ret) > 0: return ret[0]>> ret = list(self[0:1]) Module sqlalchemy.orm.query:1221 in __getitem__ << return list(res)[None:None:item.step] else: return list(res) else: return list(self[item:item+1])[0]>> return list(res) Module sqlalchemy.orm.query:1361 in __iter__ << if self._autoflush and not self._populate_existing: self.session._autoflush() return self._execute_and_instances(context)
def _execute_and_instances(self, querycontext):>> returnself._execute_and_instances(context) Module sqlalchemy.orm.query:1364 in _execute_and_instances << def _execute_and_instances(self, querycontext): result = self.session.execute(querycontext.statement, params=self._params, mapper=self._mapper_zero_or_none()) return self.instances(result, querycontext)>> result = self.session.execute(querycontext.statement, params=self._params, mapper=self._mapper_zero_or_none()) Module sqlalchemy.orm.session:754 in execute << engine = self.get_bind(mapper, clause=clause, **kw)
return self.__connection(engine,close_with_result=True).execute( clause, params or {})>> return self.__connection(engine, close_with_result=True).execute( Module sqlalchemy.orm.session:721 in __connection << return self.transaction._connection_for_bind(engine) else: return engine.contextual_connect(**kwargs)
def execute(self, clause, params=None, mapper=None, **kw):>>return engine.contextual_connect(**kwargs) Module sqlalchemy.engine.base:1229 in contextual_connect << """
return self.Connection(self, self.pool.connect(),close_with_result=close_with_result, **kwargs)
def table_names(self, schema=None, connection=None):>>return self.Connection(self, self.pool.connect(), close_with_result=close_with_result, **kwargs) Module sqlalchemy.pool:142 in connect << def connect(self): if not self._use_threadlocal: return _ConnectionFairy(self).checkout()
try:>> return _ConnectionFairy(self).checkout()Module sqlalchemy.pool:304 in __init__ << self.__counter = 0 try: rec = self._connection_record = pool.get() conn = self.connection = self._connection_record.get_connection() self._connection_record.backref = weakref.ref(self, lambda ref:_finalize_fairy(conn, rec, pool, ref))>> rec = self._connection_record = pool.get() Module sqlalchemy.pool:161 in get << def get(self): return self.do_get()
def do_get(self):>> return self.do_get()Module sqlalchemy.pool:628 in do_get << return self.do_get() else: raise exc.TimeoutError("QueuePool limit of size %d overflow %d reached, connection timed out, timeout %d" % (self.size(), self.overflow(), self._timeout))
if self._overflow_lock is not None:>> raiseexc.TimeoutError("QueuePool limit of size %d overflow %d reached, connection timed out, timeout %d" % (self.size(), self.overflow(), self._timeout)) TimeoutError: QueuePool limit of size 10 overflow 20 reached, connection timed out, timeout 30