November 2008 - galaxy-dev - lists.galaxyproject.org

[hg] galaxy 1610: Monitor memory usage in the JobWrapper.
by Greg Von Kuster 05 Nov '08

05 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/604b4b49e93f changeset: 1610:604b4b49e93f user: Greg Von Kuster <greg(a)bx.psu.edu> date: Wed Nov 05 16:00:35 2008 -0500 description: Monitor memory usage in the JobWrapper. 1 file(s) affected in this change: lib/galaxy/jobs/__init__.py diffs (116 lines): diff -r af3fc2158202 -r 604b4b49e93f lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py Wed Nov 05 15:20:47 2008 -0500 +++ b/lib/galaxy/jobs/__init__.py Wed Nov 05 16:00:35 2008 -0500 @@ -274,6 +274,9 @@ Prepare the job to run by creating the working directory and the config files. """ + if self.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() mapping.context.current.clear() #this prevents the metadata reverting that has been seen in conjunction with the PBS job runner # Create the working directory self.working_directory = \ @@ -327,6 +330,9 @@ extra_filenames.append( param_filename ) self.param_dict = param_dict self.extra_filenames = extra_filenames + if self.app.memory_usage: + m1 = self.app.memory_usage.memory( m0, pretty=True ) + log.info("End of prepare for job id %d, memory used increased by %s" % ( job.id, m1 ) ) return extra_filenames def fail( self, message, exception=False ): @@ -334,6 +340,9 @@ Indicate job failure by setting state and message on all output datasets. """ + if self.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() job = model.Job.get( self.job_id ) job.refresh() # if the job was deleted, don't fail it @@ -353,10 +362,16 @@ if exception: job.traceback = traceback.format_exc() job.flush() + if self.app.memory_usage: + m1 = self.app.memory_usage.memory( m0, pretty=True ) + log.info("End of fail for job id %d, memory used increased by %s" % ( job.id, m1 ) ) # If the job was deleted, just clean up self.cleanup() def change_state( self, state, info = False ): + if self.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() job = model.Job.get( self.job_id ) job.refresh() for dataset_assoc in job.output_datasets: @@ -370,6 +385,9 @@ job.info = info job.state = state job.flush() + if self.app.memory_usage: + m1 = self.app.memory_usage.memory( m0, pretty=True ) + log.info("End of change_state for job id %d, memory used increased by %s" % ( job.id, m1 ) ) def get_state( self ): job = model.Job.get( self.job_id ) @@ -393,6 +411,9 @@ job can be dispatched. Otherwise, return JOB_WAIT indicating that input datasets are still being prepared. """ + if self.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() job = model.Job.get( self.job_id ) job.refresh() for dataset_assoc in job.input_datasets: @@ -412,6 +433,9 @@ elif idata.state != idata.states.OK: # need to requeue return JOB_WAIT + if self.app.memory_usage: + m1 = self.app.memory_usage.memory( m0, pretty=True ) + log.info("End of check_if_ready_to_run for job id %d, memory used increased by %s" % ( job.id, m1 ) ) if job.state == model.Job.states.DELETED: return JOB_DELETED return JOB_READY @@ -422,6 +446,9 @@ the output datasets based on stderr and stdout from the command, and the contents of the output files. """ + if self.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() # default post job setup mapping.context.current.clear() job = model.Job.get( self.job_id ) @@ -485,9 +512,15 @@ job.command_line = self.command_line mapping.context.current.flush() log.debug( 'job %d ended' % self.job_id ) + if self.app.memory_usage: + m1 = self.app.memory_usage.memory( m0, pretty=True ) + log.info("End of finish for job id %d, memory used increased by %s" % ( job.id, m1 ) ) self.cleanup() def cleanup( self ): + if self.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() # remove temporary files try: for fname in self.extra_filenames: @@ -496,6 +529,9 @@ os.rmdir( self.working_directory ) except: log.exception( "Unable to cleanup job %d" % self.job_id ) + if self.app.memory_usage: + m1 = self.app.memory_usage.memory( m0, pretty=True ) + log.info("End of cleanup for job id %d, memory used increased by %s" % ( self.job_id, m1 ) ) def get_command_line( self ): return self.command_line

1 0

[hg] galaxy 1608: Possible fix for the infinite metadata recursi...
by Greg Von Kuster 05 Nov '08

05 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/53aa8d34c20a changeset: 1608:53aa8d34c20a user: Dan Blankenberg <dan(a)bx.psu.edu> date: Wed Nov 05 11:43:22 2008 -0500 description: Possible fix for the infinite metadata recursion issue. 1 file(s) affected in this change: lib/galaxy/model/__init__.py diffs (12 lines): diff -r 4e2ad77de095 -r 53aa8d34c20a lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Wed Nov 05 10:35:06 2008 -0500 +++ b/lib/galaxy/model/__init__.py Wed Nov 05 11:43:22 2008 -0500 @@ -188,7 +188,7 @@ return datatypes_registry.get_datatype_by_extension( self.extension ) def get_metadata( self ): - if not hasattr( self, '_metadata_collection' ): + if not hasattr( self, '_metadata_collection' ) or self._metadata_collection.parent is None: #using weakref to store parent (to prevent circ ref), does a Session.clear() cause parent to be invalidated, while still copying over this non-database attribute? self._metadata_collection = MetadataCollection( self ) return self._metadata_collection def set_metadata( self, bunch ):

1 0

[hg] galaxy 1607: Fix jobs report to display jobs in deleted and...
by Greg Von Kuster 05 Nov '08

05 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/4e2ad77de095 changeset: 1607:4e2ad77de095 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Wed Nov 05 10:35:06 2008 -0500 description: Fix jobs report to display jobs in deleted and waiting states. 10 file(s) affected in this change: lib/galaxy/jobs/__init__.py lib/galaxy/webapps/reports/controllers/jobs.py lib/galaxy/webapps/reports/static/january_2008_style/blue/base.css lib/galaxy/webapps/reports/templates/jobs_all_unfinished.mako lib/galaxy/webapps/reports/templates/jobs_per_month_in_error.mako lib/galaxy/webapps/reports/templates/jobs_specified_date_all.mako lib/galaxy/webapps/reports/templates/jobs_specified_date_in_error.mako lib/galaxy/webapps/reports/templates/jobs_specified_month_in_error.mako lib/galaxy/webapps/reports/templates/jobs_tool_for_month.mako lib/galaxy/webapps/reports/templates/jobs_user_for_month.mako diffs (181 lines): diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py Wed Nov 05 09:28:48 2008 -0500 +++ b/lib/galaxy/jobs/__init__.py Wed Nov 05 10:35:06 2008 -0500 @@ -403,11 +403,11 @@ idata.dataset.refresh() #we need to refresh the base Dataset, since that is where 'state' is stored # don't run jobs for which the input dataset was deleted if idata.deleted: - self.fail( "input data %d was deleted before this job started" % idata.hid ) + self.fail( "input data %d (file: %s) was deleted before the job started" % ( idata.hid, idata.file_name ) ) return JOB_INPUT_DELETED # an error in the input data causes us to bail immediately elif idata.state == idata.states.ERROR: - self.fail( "input data %d is in an error state" % idata.hid ) + self.fail( "input data %d (file: %s) is in an error state" % ( idata.hid, idata.file_name ) ) return JOB_INPUT_ERROR elif idata.state != idata.states.OK: # need to requeue diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/webapps/reports/controllers/jobs.py --- a/lib/galaxy/webapps/reports/controllers/jobs.py Wed Nov 05 09:28:48 2008 -0500 +++ b/lib/galaxy/webapps/reports/controllers/jobs.py Wed Nov 05 10:35:06 2008 -0500 @@ -91,7 +91,8 @@ year_label = start_date.strftime( "%Y" ) q = sa.select( ( sa.func.date( galaxy.model.Job.table.c.create_time ).label( 'date' ), sa.func.count( galaxy.model.Job.table.c.id ).label( 'total_jobs' ) ), - whereclause = sa.and_( galaxy.model.Job.table.c.state == 'error', + whereclause = sa.and_( sa.or_( galaxy.model.Job.table.c.state == 'error', + galaxy.model.Job.table.c.state == 'deleted' ), galaxy.model.Job.table.c.create_time >= start_date, galaxy.model.Job.table.c.create_time < end_date ), from_obj = [ sa.outerjoin( galaxy.model.Job.table, @@ -133,7 +134,8 @@ galaxy.model.Job.table.c.info, ( galaxy.model.User.table.c.email ).label( 'user_email' ), galaxy.model.GalaxySession.table.c.remote_addr ), - whereclause = sa.and_( galaxy.model.Job.table.c.state == 'error', + whereclause = sa.and_( sa.or_( galaxy.model.Job.table.c.state == 'error', + galaxy.model.Job.table.c.state == 'deleted' ), galaxy.model.Job.table.c.create_time >= start_date, galaxy.model.Job.table.c.create_time < end_date ), from_obj = [ sa.outerjoin( galaxy.model.Job.table, @@ -245,6 +247,7 @@ galaxy.model.GalaxySession.table.c.remote_addr ), whereclause = sa.or_( galaxy.model.Job.table.c.state == 'running', galaxy.model.Job.table.c.state == 'queued', + galaxy.model.Job.table.c.state == 'waiting', galaxy.model.Job.table.c.state == 'new' ), from_obj = [ sa.outerjoin( galaxy.model.Job.table, galaxy.model.History.table ).outerjoin( galaxy.model.User.table ).outerjoin( galaxy.model.GalaxySession.table, @@ -297,7 +300,8 @@ msg = '' q = sa.select( ( sa.func.date_trunc( 'month', sa.func.date( galaxy.model.Job.table.c.create_time ) ).label( 'date' ), sa.func.count( galaxy.model.Job.table.c.id ).label( 'total_jobs' ) ), - whereclause = galaxy.model.Job.table.c.state == 'error', + whereclause = sa.or_( galaxy.model.Job.table.c.state == 'error', + galaxy.model.Job.table.c.state == 'deleted' ), from_obj = [ galaxy.model.Job.table ], group_by = [ sa.func.date_trunc( 'month', sa.func.date( galaxy.model.Job.table.c.create_time ) ) ], order_by = [ sa.desc( 'date' ) ] ) diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/webapps/reports/static/january_2008_style/blue/base.css --- a/lib/galaxy/webapps/reports/static/january_2008_style/blue/base.css Wed Nov 05 09:28:48 2008 -0500 +++ b/lib/galaxy/webapps/reports/static/january_2008_style/blue/base.css Wed Nov 05 10:35:06 2008 -0500 @@ -267,6 +267,14 @@ border-bottom: solid #d8b365 1px; font-weight: bold; } +table.colored tr.headerwaiting +{ + background: #ebd9b2; + background-repeat: repeat-x; + background-position: top; + border-bottom: solid #d8b365 1px; + font-weight: bold; +} table.colored tr.headerqueued { background: #ee82ee; diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/webapps/reports/templates/jobs_all_unfinished.mako --- a/lib/galaxy/webapps/reports/templates/jobs_all_unfinished.mako Wed Nov 05 09:28:48 2008 -0500 +++ b/lib/galaxy/webapps/reports/templates/jobs_all_unfinished.mako Wed Nov 05 10:35:06 2008 -0500 @@ -17,6 +17,8 @@ state = job[0] if state == 'new': rowdef = '<tr class="headernew">' + elif state == 'waiting': + rowdef = '<tr class="headerwaiting">' elif state == 'queued': rowdef = '<tr class="headerqueued">' elif state == 'running': diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/webapps/reports/templates/jobs_per_month_in_error.mako --- a/lib/galaxy/webapps/reports/templates/jobs_per_month_in_error.mako Wed Nov 05 09:28:48 2008 -0500 +++ b/lib/galaxy/webapps/reports/templates/jobs_per_month_in_error.mako Wed Nov 05 10:35:06 2008 -0500 @@ -2,7 +2,7 @@ <%def name="main_body()"> <div class="reportBody"> - <h3 align="center">Jobs In Error Per Month</h3> + <h3 align="center">Jobs In Error Per Month ( includes jobs deleted before finishing )</h3> <h4 align="center">Click Month to view the number of jobs in error for each day of that month</h4> %if msg: <table align="center" width="70%" class="border" cellpadding="5" cellspacing="5"> diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/webapps/reports/templates/jobs_specified_date_all.mako --- a/lib/galaxy/webapps/reports/templates/jobs_specified_date_all.mako Wed Nov 05 09:28:48 2008 -0500 +++ b/lib/galaxy/webapps/reports/templates/jobs_specified_date_all.mako Wed Nov 05 10:35:06 2008 -0500 @@ -17,11 +17,15 @@ state = job[0] if state == 'new': rowdef = '<tr class="headernew">' + elif state == 'waiting': + rowdef = '<tr class="headerwaiting">' elif state == 'queued': rowdef = '<tr class="headerqueued">' elif state == 'running': rowdef = '<tr class="headerrunning">' elif state == 'error': + rowdef = '<tr class="headererror">' + elif state == 'deleted': rowdef = '<tr class="headererror">' elif state == 'ok': rowdef = '<tr class="headerok">' diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/webapps/reports/templates/jobs_specified_date_in_error.mako --- a/lib/galaxy/webapps/reports/templates/jobs_specified_date_in_error.mako Wed Nov 05 09:28:48 2008 -0500 +++ b/lib/galaxy/webapps/reports/templates/jobs_specified_date_in_error.mako Wed Nov 05 10:35:06 2008 -0500 @@ -2,7 +2,7 @@ <%def name="main_body()"> <div class="reportBody"> - <h3 align="center">All Jobs in Error for ${day_label}, ${month_label} ${day_of_month}, ${year_label}</h3> + <h3 align="center">All Jobs in Error for ${day_label}, ${month_label} ${day_of_month}, ${year_label} ( includes jobs deleted before finishing )</h3> %if msg: <table align="center" width="70%" class="border" cellpadding="5" cellspacing="5"> <tr><td class="ok_bgr">${msg}</td></tr> diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/webapps/reports/templates/jobs_specified_month_in_error.mako --- a/lib/galaxy/webapps/reports/templates/jobs_specified_month_in_error.mako Wed Nov 05 09:28:48 2008 -0500 +++ b/lib/galaxy/webapps/reports/templates/jobs_specified_month_in_error.mako Wed Nov 05 10:35:06 2008 -0500 @@ -2,7 +2,7 @@ <%def name="main_body()"> <div class="reportBody"> - <h3 align="center">Jobs in Error for ${month_label} ${year_label}</h3> + <h3 align="center">Jobs in Error for ${month_label} ${year_label} ( includes jobs deleted before finishing )</h3> <h4 align="center">Click Jobs in Error to view jobs in error for that day</h4> %if msg: <table align="center" width="70%" class="border" cellpadding="5" cellspacing="5"> diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/webapps/reports/templates/jobs_tool_for_month.mako --- a/lib/galaxy/webapps/reports/templates/jobs_tool_for_month.mako Wed Nov 05 09:28:48 2008 -0500 +++ b/lib/galaxy/webapps/reports/templates/jobs_tool_for_month.mako Wed Nov 05 10:35:06 2008 -0500 @@ -17,11 +17,15 @@ state = job[0] if state == 'new': rowdef = '<tr class="headernew">' + elif state == 'waiting': + rowdef = '<tr class="headerwaiting">' elif state == 'queued': rowdef = '<tr class="headerqueued">' elif state == 'running': rowdef = '<tr class="headerrunning">' elif state == 'error': + rowdef = '<tr class="headererror">' + elif state == 'deleted': rowdef = '<tr class="headererror">' elif state == 'ok': rowdef = '<tr class="headerok">' diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/webapps/reports/templates/jobs_user_for_month.mako --- a/lib/galaxy/webapps/reports/templates/jobs_user_for_month.mako Wed Nov 05 09:28:48 2008 -0500 +++ b/lib/galaxy/webapps/reports/templates/jobs_user_for_month.mako Wed Nov 05 10:35:06 2008 -0500 @@ -17,11 +17,15 @@ state = job[0] if state == 'new': rowdef = '<tr class="headernew">' + elif state == 'waiting': + rowdef = '<tr class="headerwaiting">' elif state == 'queued': rowdef = '<tr class="headerqueued">' elif state == 'running': rowdef = '<tr class="headerrunning">' elif state == 'error': + rowdef = '<tr class="headererror">' + elif state == 'deleted': rowdef = '<tr class="headererror">' elif state == 'ok': rowdef = '<tr class="headerok">'

1 0

[hg] galaxy 1606: Make logging events a config setting.
by Greg Von Kuster 05 Nov '08

05 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/72252f33edd1 changeset: 1606:72252f33edd1 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Wed Nov 05 09:28:48 2008 -0500 description: Make logging events a config setting. 3 file(s) affected in this change: lib/galaxy/config.py lib/galaxy/web/framework/__init__.py universe_wsgi.ini.sample diffs (68 lines): diff -r f2e583f7cab9 -r 72252f33edd1 lib/galaxy/config.py --- a/lib/galaxy/config.py Tue Nov 04 16:04:49 2008 -0500 +++ b/lib/galaxy/config.py Wed Nov 05 09:28:48 2008 -0500 @@ -58,6 +58,7 @@ self.use_heartbeat = string_as_bool( kwargs.get( 'use_heartbeat', False ) ) self.use_memdump = string_as_bool( kwargs.get( 'use_memdump', False ) ) self.log_memory_usage = string_as_bool( kwargs.get( 'log_memory_usage', False ) ) + self.log_events = string_as_bool( kwargs.get( 'log_events', False ) ) self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea" ).lower().split(",") self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "wormbase,flybase,elegans" ).lower().split(",") self.brand = kwargs.get( 'brand', None ) diff -r f2e583f7cab9 -r 72252f33edd1 lib/galaxy/web/framework/__init__.py --- a/lib/galaxy/web/framework/__init__.py Tue Nov 04 16:04:49 2008 -0500 +++ b/lib/galaxy/web/framework/__init__.py Wed Nov 05 09:28:48 2008 -0500 @@ -127,23 +127,24 @@ return self.app.model.context.current def log_event( self, message, tool_id=None, **kwargs ): """ - Application level logging. Still needs fleshing out (log levels and - such) + Application level logging. Still needs fleshing out (log levels and such) + Logging events is a config setting - if False, do not log. """ - event = self.app.model.Event() - event.tool_id = tool_id - try: - event.message = message % kwargs - except: - event.message = message - event.history = self.history - try: - event.history_id = self.history.id - except: - event.history_id = None - event.user = self.user - event.session_id = self.galaxy_session.id - event.flush() + if self.app.config.log_events: + event = self.app.model.Event() + event.tool_id = tool_id + try: + event.message = message % kwargs + except: + event.message = message + event.history = self.history + try: + event.history_id = self.history.id + except: + event.history_id = None + event.user = self.user + event.session_id = self.galaxy_session.id + event.flush() def get_cookie( self, name='galaxysession' ): """Convienience method for getting the galaxysession cookie""" try: diff -r f2e583f7cab9 -r 72252f33edd1 universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample Tue Nov 04 16:04:49 2008 -0500 +++ b/universe_wsgi.ini.sample Wed Nov 05 09:28:48 2008 -0500 @@ -18,6 +18,9 @@ # Log memory usage log_memory_usage = False + +# Log events +log_events = True # Should jobs be tracked through the database, rather than in memory ## track_jobs_in_database = true

1 0

[hg] galaxy 1605: Add ability to track memory usage in code secm...
by Greg Von Kuster 04 Nov '08

04 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/f2e583f7cab9 changeset: 1605:f2e583f7cab9 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Tue Nov 04 16:04:49 2008 -0500 description: Add ability to track memory usage in code secments - requires config addition: # Log memory usage log_memory_usage = False Added logging statements in tool execution methods and most methods in the user and root controllers. Also fixes for checking config settings that should be boolean ( e.g., memdump, heartbeat ). 9 file(s) affected in this change: lib/galaxy/app.py lib/galaxy/config.py lib/galaxy/eggs/__init__.py lib/galaxy/tools/actions/__init__.py lib/galaxy/tools/actions/upload.py lib/galaxy/util/memory_usage.py lib/galaxy/web/controllers/root.py lib/galaxy/web/controllers/user.py universe_wsgi.ini.sample diffs (469 lines): diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/app.py --- a/lib/galaxy/app.py Tue Nov 04 14:00:30 2008 -0500 +++ b/lib/galaxy/app.py Tue Nov 04 16:04:49 2008 -0500 @@ -38,6 +38,7 @@ # Heartbeat and memdump for thread / heap profiling self.heartbeat = None self.memdump = None + self.memory_usage = None # Start the heartbeat process if configured and available if self.config.use_heartbeat: from galaxy.util import heartbeat @@ -49,6 +50,10 @@ from galaxy.util import memdump if memdump.Memdump: self.memdump = memdump.Memdump() + # Enable memory_usage logging if configured + if self.config.log_memory_usage: + from galaxy.util import memory_usage + self.memory_usage = memory_usage def shutdown( self ): self.job_manager.shutdown() if self.heartbeat: diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/config.py --- a/lib/galaxy/config.py Tue Nov 04 14:00:30 2008 -0500 +++ b/lib/galaxy/config.py Tue Nov 04 16:04:49 2008 -0500 @@ -55,8 +55,9 @@ self.pbs_dataset_server = kwargs.get('pbs_dataset_server', "" ) self.pbs_dataset_path = kwargs.get('pbs_dataset_path', "" ) self.pbs_stage_path = kwargs.get('pbs_stage_path', "" ) - self.use_heartbeat = string_as_bool( kwargs.get( 'use_heartbeat', "False" ) ) - self.use_memdump = kwargs.get( 'use_memdump', False ) + self.use_heartbeat = string_as_bool( kwargs.get( 'use_heartbeat', False ) ) + self.use_memdump = string_as_bool( kwargs.get( 'use_memdump', False ) ) + self.log_memory_usage = string_as_bool( kwargs.get( 'log_memory_usage', False ) ) self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea" ).lower().split(",") self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "wormbase,flybase,elegans" ).lower().split(",") self.brand = kwargs.get( 'brand', None ) diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/eggs/__init__.py --- a/lib/galaxy/eggs/__init__.py Tue Nov 04 14:00:30 2008 -0500 +++ b/lib/galaxy/eggs/__init__.py Tue Nov 04 16:04:49 2008 -0500 @@ -484,7 +484,7 @@ return False elif egg_name == "threadframe": try: - if self.config.get( "app:main", "use_heartbeat" ) == "True": + if self.config.get( "app:main", "use_heartbeat" ): return True else: return False @@ -492,7 +492,7 @@ return False elif egg_name == "guppy": try: - if self.config.get( "app:main", "use_memdump" ) == "True": + if self.config.get( "app:main", "use_memdump" ): return True else: return False diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/tools/actions/__init__.py --- a/lib/galaxy/tools/actions/__init__.py Tue Nov 04 14:00:30 2008 -0500 +++ b/lib/galaxy/tools/actions/__init__.py Tue Nov 04 16:04:49 2008 -0500 @@ -65,6 +65,9 @@ return input_datasets def execute(self, tool, trans, incoming={}, set_output_hid=True ): + if trans.app.memory_usage: + # Keep track of memory usage + m0 = trans.app.memory_usage.memory() out_data = {} # Collect any input datasets from the incoming parameters inp_data = self.collect_input_datasets( tool, incoming, trans ) @@ -198,6 +201,9 @@ for name, dataset in out_data.iteritems(): job.add_output_dataset( name, dataset ) trans.app.model.flush() + if trans.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info("End of tool %s execution for job id %d, memory used increased by %s" % ( tool.id, job.id, m1 ) ) # Some tools are not really executable, but jobs are still created for them ( for record keeping ). # Examples include tools that redirect to other applications ( epigraph ). These special tools must # include something that can be retrieved from the params ( e.g., REDIRECT_URL ) to keep the job diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/tools/actions/upload.py --- a/lib/galaxy/tools/actions/upload.py Tue Nov 04 14:00:30 2008 -0500 +++ b/lib/galaxy/tools/actions/upload.py Tue Nov 04 16:04:49 2008 -0500 @@ -15,6 +15,9 @@ self.line_count = None def execute( self, tool, trans, incoming={}, set_output_hid = True ): + if trans.app.memory_usage: + # Keep track of memory usage + m0 = trans.app.memory_usage.memory() data_file = incoming['file_data'] file_type = incoming['file_type'] dbkey = incoming['dbkey'] @@ -93,6 +96,9 @@ job.flush() log.info( 'job id %d ended ok, file size: %s' % ( job.id, file_size_str ) ) trans.log_event( 'job id %d ended ok, file size: %s' % ( job.id, file_size_str ), tool_id=tool.id ) + if trans.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info("End of tool %s execution for job id %d, memory used increased by %s" % ( tool.id, job.id, m1 ) ) return dict( output=hda ) def upload_empty(self, trans, job, err_code, err_msg): diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/util/memory_usage.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/util/memory_usage.py Tue Nov 04 16:04:49 2008 -0500 @@ -0,0 +1,44 @@ +import os, platform, logging +from galaxy.datatypes.data import nice_size + +log = logging.getLogger( __name__ ) + +_proc_status = '/proc/%d/status' % os.getpid() + +_scale = { 'kB': 1024.0, 'mB': 1024.0*1024.0, 'KB': 1024.0, 'MB': 1024.0*1024.0 } + +def _VmB( VmKey ): + '''Private. + ''' + global _proc_status, _scale + # get pseudo file /proc/<pid>/status + try: + t = open( _proc_status ) + v = t.read() + t.close() + except: + log.debug("memory_usage is currently supported only on Linux, your platform is %s %s" % ( platform.system(), platform.release() ) ) + return 0.0 # non-Linux? + # get VmKey line e.g. 'VmRSS: 9999 kB\n ...' + i = v.index( VmKey ) + v = v[ i: ].split( None, 3 ) # whitespace + if len( v ) < 3: + return 0.0 # invalid format? + # convert Vm value to bytes + return float( v[ 1 ] ) * _scale[ v[ 2 ] ] +def memory( since=0.0, pretty=False ): + '''Return memory usage in bytes. + ''' + size = _VmB( 'VmSize:' ) - since + if pretty: + return nice_size( size ) + else: + return size +def resident( since=0.0 ): + '''Return resident memory usage in bytes. + ''' + return _VmB( 'VmRSS:' ) - since +def stacksize( since=0.0 ): + '''Return stack size in bytes. + ''' + return _VmB( 'VmStk:' ) - since diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/web/controllers/root.py --- a/lib/galaxy/web/controllers/root.py Tue Nov 04 14:00:30 2008 -0500 +++ b/lib/galaxy/web/controllers/root.py Tue Nov 04 16:04:49 2008 -0500 @@ -116,6 +116,9 @@ Returns data directly into the browser. Sets the mime-type according to the extension """ + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() if hid is not None: try: hid = int( hid ) @@ -146,6 +149,9 @@ fname = ''.join(c in valid_chars and c or '_' for c in fname)[0:150] trans.response.headers["Content-Disposition"] = "attachment; filename=GalaxyHistoryItem-%s-[%s]%s" % (data.hid, fname, toext) trans.log_event( "Display dataset id: %s" % str(id) ) + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/display, memory used increased by %s" % m1 ) try: return open( data.file_name ) except: @@ -321,6 +327,9 @@ @web.expose def history_delete( self, trans, id=None, **kwd): """Deletes a list of histories, ensures that histories are owned by current user""" + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() history_names = [] if id: if isinstance( id, list ): @@ -348,12 +357,18 @@ trans.log_event( "History id %s marked as deleted" % str(hid) ) else: return trans.show_message( "You must select at least one history to delete." ) + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/history_delete, memory used increased by %s" % m1 ) return trans.show_message( "History deleted: %s" % ",".join(history_names), refresh_frames=['history']) @web.expose def history_undelete( self, trans, id=[], **kwd): """Undeletes a list of histories, ensures that histories are owned by current user""" + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() history_names = [] errors = [] ok_msg = "" @@ -385,22 +400,34 @@ ok_msg = "Histories (%s) have been undeleted." % ", ".join( history_names ) else: errors.append( "You must select at least one history to undelete." ) + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/history_undelete, memory used increased by %s" % m1 ) return self.history_available( trans, id=','.join( id ), show_deleted=True, ok_msg = ok_msg, error_msg = " ".join( errors ) ) @web.expose def clear_history( self, trans ): """Clears the history for a user""" + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() history = trans.get_history() for dataset in history.datasets: dataset.deleted = True dataset.clear_associated_files() self.app.model.flush() trans.log_event( "History id %s cleared" % (str(history.id)) ) + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/clear_history, memory used increased by %s" % m1 ) trans.response.send_redirect( url_for("/index" ) ) @web.expose @web.require_login( "share histories with other users" ) def history_share( self, trans, id=None, email="", **kwd ): + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() send_to_err = "" if not id: id = trans.get_history().id @@ -427,6 +454,9 @@ trans.log_event( "History share, id: %s, name: '%s': to new id: %s" % (str(history.id), history.name, str(new_history.id)) ) self.app.model.flush() return trans.show_message( "History (%s) has been shared with: %s" % (",".join(history_names),email) ) + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/history_share, memory used increased by %s" % m1 ) return trans.fill_template( "/history/share.mako", histories=histories, email=email, send_to_err=send_to_err) @web.expose @@ -435,18 +465,21 @@ """ List all available histories """ + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() if as_xml: trans.response.set_content_type('text/xml') return trans.fill_template( "/history/list_as_xml.mako" ) if not isinstance( id, list ): id = id.split( "," ) trans.log_event( "History id %s available" % str( id ) ) - history_operations = dict( share=self.history_share, rename=self.history_rename, delete=self.history_delete, undelete=self.history_undelete ) - + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/history_available, memory used increased by %s" % m1 ) if do_operation in history_operations: return history_operations[do_operation]( trans, id=id, show_deleted=show_deleted, ok_msg=ok_msg, error_msg=error_msg, **kwd ) - return trans.fill_template( "/history/list.mako", ids=id, user=trans.get_user(), current_history=trans.get_history(), @@ -457,6 +490,9 @@ @web.expose def history_import( self, trans, id=None, confirm=False, **kwd ): + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() msg = "" user = trans.get_user() user_history = trans.get_history() @@ -497,6 +533,9 @@ new_history.flush() trans.set_history( new_history ) trans.log_event( "History imported, id: %s, name: '%s': " % (str(new_history.id) , new_history.name ) ) + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/history_import, memory used increased by %s" % m1 ) return trans.show_ok_message( """ History "%s" has been imported. Click <a href="%s">here</a> to begin.""" % ( new_history.name, web.url_for( '/' ) ) ) @@ -511,6 +550,9 @@ if not id: return trans.response.send_redirect( web.url_for( action='history_available' ) ) else: + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() new_history = trans.app.model.History.get( id ) if new_history: galaxy_session = trans.get_galaxy_session() @@ -522,6 +564,9 @@ new_history.flush() trans.set_history( new_history ) trans.log_event( "History switched to id: %s, name: '%s'" % (str(new_history.id), new_history.name ) ) + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/history_switch, memory used increased by %s" % m1 ) return trans.show_message( "History switched to: %s" % new_history.name, refresh_frames=['history']) else: @@ -529,13 +574,22 @@ @web.expose def history_new( self, trans ): + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() trans.new_history() trans.log_event( "Created new History, id: %s." % str(trans.get_history().id) ) + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/history_new, memory used increased by %s" % m1 ) return trans.show_message( "New history created", refresh_frames = ['history'] ) @web.expose @web.require_login( "renames histories" ) def history_rename( self, trans, id=None, name=None, **kwd ): + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() user = trans.get_user() if not isinstance( id, list ): @@ -572,11 +626,17 @@ change_msg = change_msg + "<p>You must specify a valid name for History: "+cur_names[i]+"</p>" else: change_msg = change_msg + "<p>History: "+cur_names[i]+" does not appear to belong to you.</p>" + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/history_rename, memory used increased by %s" % m1 ) return trans.show_message( "<p>%s" % change_msg, refresh_frames=['history'] ) @web.expose def history_add_to( self, trans, history_id=None, file_data=None, name="Data Added to History",info=None,ext="txt",dbkey="?",**kwd ): """Adds a POSTed file to a History""" + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() try: history = trans.app.model.History.get( history_id ) data = trans.app.model.HistoryDatasetAssociation( name = name, info = info, extension = ext, dbkey = dbkey, create_dataset = True ) @@ -595,6 +655,9 @@ data.set_size() data.flush() trans.log_event("Added dataset %d to history %d" %(data.id, trans.history.id)) + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/history_add_to, memory used increased by %s" % m1 ) return trans.show_ok_message("Dataset "+str(data.hid)+" added to history "+str(history_id)+".") except: return trans.show_error_message("Adding File to History has Failed") @@ -602,6 +665,9 @@ @web.expose def dataset_make_primary( self, trans, id=None): """Copies a dataset and makes primary""" + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() try: old_data = self.app.model.HistoryDatasetAssociation.get( id ) new_data = old_data.copy() @@ -610,6 +676,9 @@ history = trans.get_history() history.add_dataset(new_data) new_data.flush() + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/dataset_make_primary, memory used increased by %s" % m1 ) return trans.show_message( "<p>Secondary dataset has been made primary.</p>", refresh_frames=['history'] ) except: return trans.show_error_message( "<p>Failed to make secondary dataset primary.</p>" ) diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/web/controllers/user.py --- a/lib/galaxy/web/controllers/user.py Tue Nov 04 14:00:30 2008 -0500 +++ b/lib/galaxy/web/controllers/user.py Tue Nov 04 16:04:49 2008 -0500 @@ -70,6 +70,9 @@ @web.expose def login( self, trans, email='', password='' ): + if trans.app.memory_usage: + # Keep track of memory usage + m0 = trans.app.memory_usage.memory() email_error = password_error = None # Attempt login if email or password: @@ -85,6 +88,9 @@ trans.handle_user_login( user ) trans.log_event( "User logged in" ) return trans.show_ok_message( "Now logged in as " + user.email, refresh_frames=['masthead', 'history'] ) + if trans.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of user/login, memory used increased by %s" % m1 ) return trans.show_form( web.FormBuilder( web.url_for(), "Login", submit_text="Login" ) .add_text( "email", "Email address", value=email, error=email_error ) @@ -92,13 +98,22 @@ help="<a href='%s'>Forgot password? Reset here</a>" % web.url_for( action='reset_password' ) ) ) @web.expose def logout( self, trans ): + if trans.app.memory_usage: + # Keep track of memory usage + m0 = trans.app.memory_usage.memory() # Since logging an event requires a session, we'll log prior to ending the session trans.log_event( "User logged out" ) trans.handle_user_logout() + if trans.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of user/logout, memory used increased by %s" % m1 ) return trans.show_ok_message( "You are no longer logged in", refresh_frames=['masthead', 'history'] ) @web.expose def create( self, trans, email='', password='', confirm='',subscribe=False ): + if trans.app.memory_usage: + # Keep track of memory usage + m0 = trans.app.memory_usage.memory() email_error = password_error = confirm_error = None if email: if len( email ) == 0 or "@" not in email or "." not in email: @@ -120,10 +135,13 @@ trans.log_event( "User logged in" ) #subscribe user to email list if subscribe: - mail = os.popen("%s -t" % self.app.config.sendmail_path, 'w') - mail.write("To: %s\nFrom: %s\nSubject: Join Mailing List\n\nJoin Mailing list." % (self.app.config.mailing_join_addr,email) ) + mail = os.popen("%s -t" % trans.app.config.sendmail_path, 'w') + mail.write("To: %s\nFrom: %s\nSubject: Join Mailing List\n\nJoin Mailing list." % (trans.app.config.mailing_join_addr,email) ) if mail.close(): return trans.show_warn_message( "Now logged in as " + user.email+". However, subscribing to the mailing list has failed.", refresh_frames=['masthead', 'history'] ) + if trans.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of user/create, memory used increased by %s" % m1 ) return trans.show_ok_message( "Now logged in as " + user.email, refresh_frames=['masthead', 'history'] ) return trans.show_form( web.FormBuilder( web.url_for(), "Create account", submit_text="Create" ) @@ -145,7 +163,7 @@ new_pass = "" for i in range(15): new_pass = new_pass + choice(chars) - mail = os.popen("%s -t" % self.app.config.sendmail_path, 'w') + mail = os.popen("%s -t" % trans.app.config.sendmail_path, 'w') mail.write("To: %s\nFrom: no-reply@%s\nSubject: Galaxy Password Reset\n\nYour password has been reset to \"%s\" (no quotes)." % (email, trans.request.remote_addr, new_pass) ) if mail.close(): return trans.show_ok_message( "Failed to reset password! If this problem persist, submit a bug report.") diff -r 7a5d9759c702 -r f2e583f7cab9 universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample Tue Nov 04 14:00:30 2008 -0500 +++ b/universe_wsgi.ini.sample Tue Nov 04 16:04:49 2008 -0500 @@ -15,6 +15,9 @@ # Specifies the factory for the universe WSGI application paste.app_factory = galaxy.web.buildapp:app_factory log_level = DEBUG + +# Log memory usage +log_memory_usage = False # Should jobs be tracked through the database, rather than in memory ## track_jobs_in_database = true

1 0

[hg] galaxy 1601: Apply James' faster_random patch from Nov 1.
by Greg Von Kuster 04 Nov '08

04 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/52e504c91cbc changeset: 1601:52e504c91cbc user: Greg Von Kuster <greg(a)bx.psu.edu> date: Tue Nov 04 09:27:49 2008 -0500 description: Apply James' faster_random patch from Nov 1. 1 file(s) affected in this change: lib/galaxy/web/security/__init__.py diffs (34 lines): diff -r 7ae75e6d9d6a -r 52e504c91cbc lib/galaxy/web/security/__init__.py --- a/lib/galaxy/web/security/__init__.py Mon Nov 03 15:31:38 2008 -0500 +++ b/lib/galaxy/web/security/__init__.py Tue Nov 04 09:27:49 2008 -0500 @@ -10,18 +10,17 @@ log = logging.getLogger( __name__ ) if os.path.exists( "/dev/urandom" ): - log.debug("###using /dev/urandom....") # We have urandom, use it as the source of random data random_fd = os.open( "/dev/urandom", os.O_RDONLY ) def get_random_bytes( nbytes ): value = os.read( random_fd, nbytes ) # Normally we should get as much as we need if len( value ) == nbytes: - return value + return value.encode( "hex" ) # If we don't, keep reading (this is slow and should never happen) while len( value ) < nbytes: value += os.read( random_fd, nbytes - len( value ) ) - return value + return value.encode( "hex" ) else: def get_random_bytes( nbytes ): nbits = nbytes * 8 @@ -29,7 +28,8 @@ while random_pool.entropy < nbits: random_pool.add_event() random_pool.stir() - return( str( number.getRandomNumber( nbits, random_pool.get_bytes ) ) ) + return str( number.getRandomNumber( nbits, random_pool.get_bytes ) ) + class SecurityHelper( object ): # TODO: checking if histories/datasets are owned by the current user) will be moved here.

1 0

[hg] galaxy 1603: Fixes for upload tool job creation.
by Greg Von Kuster 04 Nov '08

04 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/85347b21a1b8 changeset: 1603:85347b21a1b8 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Tue Nov 04 13:20:39 2008 -0500 description: Fixes for upload tool job creation. 1 file(s) affected in this change: lib/galaxy/tools/actions/upload.py diffs (58 lines): diff -r 531be9ea2486 -r 85347b21a1b8 lib/galaxy/tools/actions/upload.py --- a/lib/galaxy/tools/actions/upload.py Tue Nov 04 11:56:09 2008 -0500 +++ b/lib/galaxy/tools/actions/upload.py Tue Nov 04 13:20:39 2008 -0500 @@ -36,7 +36,7 @@ except: job.tool_version = "1.0.0" job.flush() - log.debug( 'tool %s created job id %d' % ( tool.id, job.id ) ) + log.info( 'tool %s created job id %d' % ( tool.id, job.id ) ) trans.log_event( 'created job id %d' % job.id, tool_id=tool.id ) if 'local_filename' in dir( data_file ): # Use the existing file @@ -88,11 +88,11 @@ return self.upload_empty( trans, job, "No data error:", "either you pasted no data, the url you specified is invalid, or you have not specified a file." ) hda = data_list[0] job.state = trans.app.model.Job.states.OK - file_size_str = nice_size( hda.dataset.file_size ) - job.info = "Uploaded file '%s', size: %s" % ( file_name, file_size_str ) + file_size_str = datatypes.data.nice_size( hda.dataset.file_size ) + job.info = "%s, size: %s" % ( hda.info, file_size_str ) job.flush() - log.debug( 'tool %s job id %d ended ok, file: %s, size: %s' % ( tool.id, job.id, file_name, file_size_str ) ) - trans.log_event( 'job id %d ended ok, file: %s, size: %s' % ( job.id, file_name, file_size_str ), tool_id=tool.id ) + log.info( 'job id %d ended ok, file size: %s' % ( job.id, file_size_str ) ) + trans.log_event( 'job id %d ended ok, file size: %s' % ( job.id, file_size_str ), tool_id=tool.id ) return dict( output=hda ) def upload_empty(self, trans, job, err_code, err_msg): @@ -111,7 +111,7 @@ job.info = err_msg # If the failure is due to a Galaxy framework exception, save the traceback job.flush() - log.debug( 'tool %s job id %d ended with errors' % ( job.tool_id, job.id ) ) + log.info( 'job id %d ended with errors' % job.id ) trans.log_event( 'job id %d ended with errors' % job.id, tool_id=job.tool_id ) return dict( output=data ) @@ -301,20 +301,5 @@ temp.close() return False -def nice_size( size ): - """Returns a readably formatted string with the size""" - words = [ 'bytes', 'Kb', 'Mb', 'Gb' ] - try: - size = float( size ) - except: - return '??? bytes' - for ind, word in enumerate( words ): - step = 1024 ** ( ind + 1 ) - if step > size: - size = size / float( 1024 ** ind ) - out = "%.1f %s" % ( size, word ) - return out - return '??? bytes' - class BadFileException( Exception ): pass

1 0

[hg] galaxy 1602: Upload tool will now create a job record, no j...
by Greg Von Kuster 04 Nov '08

04 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/531be9ea2486 changeset: 1602:531be9ea2486 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Tue Nov 04 11:56:09 2008 -0500 description: Upload tool will now create a job record, no job queued though. 1 file(s) affected in this change: lib/galaxy/tools/actions/upload.py diffs (120 lines): diff -r 52e504c91cbc -r 531be9ea2486 lib/galaxy/tools/actions/upload.py --- a/lib/galaxy/tools/actions/upload.py Tue Nov 04 09:27:49 2008 -0500 +++ b/lib/galaxy/tools/actions/upload.py Tue Nov 04 11:56:09 2008 -0500 @@ -25,7 +25,19 @@ space_to_tab = True temp_name = "" data_list = [] - + # Create the job object + job = trans.app.model.Job() + job.session_id = trans.get_galaxy_session().id + job.history_id = trans.history.id + job.tool_id = tool.id + try: + # For backward compatibility, some tools may not have versions yet. + job.tool_version = tool.version + except: + job.tool_version = "1.0.0" + job.flush() + log.debug( 'tool %s created job id %d' % ( tool.id, job.id ) ) + trans.log_event( 'created job id %d' % job.id, tool_id=tool.id ) if 'local_filename' in dir( data_file ): # Use the existing file try: @@ -34,7 +46,7 @@ file_name = file_name.split( '/' )[-1] data_list.append( self.add_file( trans, data_file.local_filename, file_name, file_type, dbkey, space_to_tab=space_to_tab ) ) except Exception, e: - return self.upload_empty( trans, "Error:", str( e ) ) + return self.upload_empty( trans, job, "Error:", str( e ) ) elif 'filename' in dir( data_file ): try: file_name = data_file.filename @@ -43,7 +55,7 @@ temp_name = sniff.stream_to_file( data_file.file ) data_list.append( self.add_file( trans, temp_name, file_name, file_type, dbkey, space_to_tab=space_to_tab ) ) except Exception, e: - return self.upload_empty( trans, "Error:", str( e ) ) + return self.upload_empty( trans, job, "Error:", str( e ) ) if url_paste not in [ None, "" ]: if url_paste.lower().find( 'http://' ) >= 0 or url_paste.lower().find( 'ftp://' ) >= 0: url_paste = url_paste.replace( '\r', '' ).split( '\n' ) @@ -54,7 +66,7 @@ temp_name = sniff.stream_to_file( urllib.urlopen( line ) ) data_list.append( self.add_file( trans, temp_name, line, file_type, dbkey, info="uploaded url", space_to_tab=space_to_tab ) ) except Exception, e: - return self.upload_empty( trans, "Error:", str( e ) ) + return self.upload_empty( trans, job, "Error:", str( e ) ) else: is_valid = False for line in url_paste: @@ -67,26 +79,40 @@ temp_name = sniff.stream_to_file( StringIO.StringIO( url_paste ) ) data_list.append( self.add_file( trans, temp_name, 'Pasted Entry', file_type, dbkey, info="pasted entry", space_to_tab=space_to_tab ) ) except Exception, e: - return self.upload_empty( trans, "Error:", str( e ) ) + return self.upload_empty( trans, job, "Error:", str( e ) ) else: - return self.upload_empty( trans, "No data error:", "you pasted no data." ) + return self.upload_empty( trans, job, "No data error:", "you pasted no data." ) if self.empty: - return self.upload_empty( trans, "Empty file error:", "you attempted to upload an empty file." ) + return self.upload_empty( trans, job, "Empty file error:", "you attempted to upload an empty file." ) elif len( data_list ) < 1: - return self.upload_empty( trans, "No data error:", "either you pasted no data, the url you specified is invalid, or you have not specified a file." ) - return dict( output=data_list[0] ) + return self.upload_empty( trans, job, "No data error:", "either you pasted no data, the url you specified is invalid, or you have not specified a file." ) + hda = data_list[0] + job.state = trans.app.model.Job.states.OK + file_size_str = nice_size( hda.dataset.file_size ) + job.info = "Uploaded file '%s', size: %s" % ( file_name, file_size_str ) + job.flush() + log.debug( 'tool %s job id %d ended ok, file: %s, size: %s' % ( tool.id, job.id, file_name, file_size_str ) ) + trans.log_event( 'job id %d ended ok, file: %s, size: %s' % ( job.id, file_name, file_size_str ), tool_id=tool.id ) + return dict( output=hda ) - def upload_empty(self, trans, err_code, err_msg): + def upload_empty(self, trans, job, err_code, err_msg): data = trans.app.model.HistoryDatasetAssociation( create_dataset = True ) data.name = err_code data.extension = "txt" data.dbkey = "?" data.info = err_msg data.file_size = 0 + data.state = data.states.EMPTY data.flush() - data.state = data.states.EMPTY trans.history.add_dataset( data ) trans.app.model.flush() + # Indicate job failure by setting state and message + job.state = trans.app.model.Job.states.ERROR + job.info = err_msg + # If the failure is due to a Galaxy framework exception, save the traceback + job.flush() + log.debug( 'tool %s job id %d ended with errors' % ( job.tool_id, job.id ) ) + trans.log_event( 'job id %d ended with errors' % job.id, tool_id=job.tool_id ) return dict( output=data ) def add_file( self, trans, temp_name, file_name, file_type, dbkey, info=None, space_to_tab=False ): @@ -275,5 +301,20 @@ temp.close() return False +def nice_size( size ): + """Returns a readably formatted string with the size""" + words = [ 'bytes', 'Kb', 'Mb', 'Gb' ] + try: + size = float( size ) + except: + return '??? bytes' + for ind, word in enumerate( words ): + step = 1024 ** ( ind + 1 ) + if step > size: + size = size / float( 1024 ** ind ) + out = "%.1f %s" % ( size, word ) + return out + return '??? bytes' + class BadFileException( Exception ): pass

1 0

[hg] galaxy 1604: Fix get_bool in config
by Greg Von Kuster 04 Nov '08

04 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/7a5d9759c702 changeset: 1604:7a5d9759c702 user: Nate Coraor <nate(a)bx.psu.edu> date: Tue Nov 04 14:00:30 2008 -0500 description: Fix get_bool in config 1 file(s) affected in this change: lib/galaxy/config.py diffs (12 lines): diff -r 85347b21a1b8 -r 7a5d9759c702 lib/galaxy/config.py --- a/lib/galaxy/config.py Tue Nov 04 13:20:39 2008 -0500 +++ b/lib/galaxy/config.py Tue Nov 04 14:00:30 2008 -0500 @@ -85,7 +85,7 @@ return self.config_dict.get( key, default ) def get_bool( self, key, default ): if key in self.config_dict: - return string_as_bool( key ) + return string_as_bool( self.config_dict[key] ) else: return default def check( self ):

1 0

[hg] galaxy 1599: A fix for the metadata reverting issue seen wi...
by Greg Von Kuster 04 Nov '08

04 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/13cbdd1bbd19 changeset: 1599:13cbdd1bbd19 user: Dan Blankenberg <dan(a)bx.psu.edu> date: Mon Nov 03 15:21:03 2008 -0500 description: A fix for the metadata reverting issue seen with PBS job runner. 1 file(s) affected in this change: lib/galaxy/jobs/__init__.py diffs (18 lines): diff -r ca640ce1abcf -r 13cbdd1bbd19 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py Mon Nov 03 15:15:22 2008 -0500 +++ b/lib/galaxy/jobs/__init__.py Mon Nov 03 15:21:03 2008 -0500 @@ -274,6 +274,7 @@ Prepare the job to run by creating the working directory and the config files. """ + mapping.context.current.clear() #this prevents the metadata reverting that has been seen in conjunction with the PBS job runner # Create the working directory self.working_directory = \ os.path.join( self.app.config.job_working_directory, str( self.job_id ) ) @@ -643,4 +644,4 @@ def put( self, *args ): return def shutdown( self ): - return \ No newline at end of file + return

1 0