details: http://www.bx.psu.edu/hg/galaxy/rev/03f1e9e78f74 changeset: 1700:03f1e9e78f74 user: Greg Von Kuster <greg@bx.psu.edu> date: Tue Jan 13 11:33:17 2009 -0500 description: Eliminate boilerplate code to track memory_usage, keeping the class and config settings for possible future use. 5 file(s) affected in this change: lib/galaxy/jobs/__init__.py lib/galaxy/tools/actions/__init__.py lib/galaxy/tools/actions/upload.py lib/galaxy/web/controllers/root.py lib/galaxy/web/controllers/user.py diffs (446 lines): diff -r 511c2b1966dc -r 03f1e9e78f74 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py Tue Jan 13 10:55:08 2009 -0500 +++ b/lib/galaxy/jobs/__init__.py Tue Jan 13 11:33:17 2009 -0500 @@ -246,9 +246,6 @@ job can be dispatched. Otherwise, return JOB_WAIT indicating that input datasets are still being prepared. """ - if self.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() for dataset_assoc in job.input_datasets: idata = dataset_assoc.dataset if not idata: @@ -264,9 +261,6 @@ elif idata.state != idata.states.OK: # need to requeue return JOB_WAIT - if self.app.memory_usage: - m1 = self.app.memory_usage.memory( m0, pretty=True ) - log.info("End of check_if_ready_to_run for job id %d, memory used increased by %s" % ( job.id, m1 ) ) if job.state == model.Job.states.DELETED: return JOB_DELETED return JOB_READY @@ -322,9 +316,6 @@ Prepare the job to run by creating the working directory and the config files. """ - if self.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() mapping.context.current.clear() #this prevents the metadata reverting that has been seen in conjunction with the PBS job runner # Create the working directory self.working_directory = \ @@ -378,9 +369,6 @@ extra_filenames.append( param_filename ) self.param_dict = param_dict self.extra_filenames = extra_filenames - if self.app.memory_usage: - m1 = self.app.memory_usage.memory( m0, pretty=True ) - log.info("End of prepare for job id %d, memory used increased by %s" % ( job.id, m1 ) ) return extra_filenames def fail( self, message, exception=False ): @@ -388,9 +376,6 @@ Indicate job failure by setting state and message on all output datasets. """ - if self.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() job = model.Job.get( self.job_id ) job.refresh() # if the job was deleted, don't fail it @@ -417,16 +402,10 @@ if exception: job.traceback = traceback.format_exc() job.flush() - if self.app.memory_usage: - m1 = self.app.memory_usage.memory( m0, pretty=True ) - log.info("End of fail for job id %d, memory used increased by %s" % ( job.id, m1 ) ) # If the job was deleted, just clean up self.cleanup() def change_state( self, state, info = False ): - if self.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() job = model.Job.get( self.job_id ) job.refresh() for dataset_assoc in job.output_datasets: @@ -440,9 +419,6 @@ job.info = info job.state = state job.flush() - if self.app.memory_usage: - m1 = self.app.memory_usage.memory( m0, pretty=True ) - log.info("End of change_state for job id %d, memory used increased by %s" % ( job.id, m1 ) ) def get_state( self ): job = model.Job.get( self.job_id ) @@ -462,9 +438,6 @@ the output datasets based on stderr and stdout from the command, and the contents of the output files. """ - if self.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() # default post job setup mapping.context.current.clear() job = model.Job.get( self.job_id ) @@ -535,15 +508,9 @@ job.command_line = self.command_line mapping.context.current.flush() log.debug( 'job %d ended' % self.job_id ) - if self.app.memory_usage: - m1 = self.app.memory_usage.memory( m0, pretty=True ) - log.info("End of finish for job id %d, memory used increased by %s" % ( job.id, m1 ) ) self.cleanup() def cleanup( self ): - if self.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() # remove temporary files try: for fname in self.extra_filenames: @@ -552,9 +519,6 @@ os.rmdir( self.working_directory ) except: log.exception( "Unable to cleanup job %d" % self.job_id ) - if self.app.memory_usage: - m1 = self.app.memory_usage.memory( m0, pretty=True ) - log.info("End of cleanup for job id %d, memory used increased by %s" % ( self.job_id, m1 ) ) def get_command_line( self ): return self.command_line diff -r 511c2b1966dc -r 03f1e9e78f74 lib/galaxy/tools/actions/__init__.py --- a/lib/galaxy/tools/actions/__init__.py Tue Jan 13 10:55:08 2009 -0500 +++ b/lib/galaxy/tools/actions/__init__.py Tue Jan 13 11:33:17 2009 -0500 @@ -65,9 +65,6 @@ return input_datasets def execute(self, tool, trans, incoming={}, set_output_hid=True ): - if trans.app.memory_usage: - # Keep track of memory usage - m0 = trans.app.memory_usage.memory() out_data = {} # Collect any input datasets from the incoming parameters inp_data = self.collect_input_datasets( tool, incoming, trans ) @@ -201,9 +198,6 @@ for name, dataset in out_data.iteritems(): job.add_output_dataset( name, dataset ) trans.app.model.flush() - if trans.app.memory_usage: - m1 = trans.app.memory_usage.memory( m0, pretty=True ) - log.info("End of tool %s execution for job id %d, memory used increased by %s" % ( tool.id, job.id, m1 ) ) # Some tools are not really executable, but jobs are still created for them ( for record keeping ). # Examples include tools that redirect to other applications ( epigraph ). These special tools must # include something that can be retrieved from the params ( e.g., REDIRECT_URL ) to keep the job diff -r 511c2b1966dc -r 03f1e9e78f74 lib/galaxy/tools/actions/upload.py --- a/lib/galaxy/tools/actions/upload.py Tue Jan 13 10:55:08 2009 -0500 +++ b/lib/galaxy/tools/actions/upload.py Tue Jan 13 11:33:17 2009 -0500 @@ -15,9 +15,6 @@ self.line_count = None def execute( self, tool, trans, incoming={}, set_output_hid = True ): - if trans.app.memory_usage: - # Keep track of memory usage - m0 = trans.app.memory_usage.memory() data_file = incoming['file_data'] file_type = incoming['file_type'] dbkey = incoming['dbkey'] @@ -129,9 +126,6 @@ job.flush() log.info( 'job id %d ended ok, file size: %s' % ( job.id, file_size_str ) ) trans.log_event( 'job id %d ended ok, file size: %s' % ( job.id, file_size_str ), tool_id=tool.id ) - if trans.app.memory_usage: - m1 = trans.app.memory_usage.memory( m0, pretty=True ) - log.info("End of tool %s execution for job id %d, memory used increased by %s" % ( tool.id, job.id, m1 ) ) return dict( output=hda ) def upload_empty( self, trans, job, err_code, err_msg ): diff -r 511c2b1966dc -r 03f1e9e78f74 lib/galaxy/web/controllers/root.py --- a/lib/galaxy/web/controllers/root.py Tue Jan 13 10:55:08 2009 -0500 +++ b/lib/galaxy/web/controllers/root.py Tue Jan 13 11:33:17 2009 -0500 @@ -116,9 +116,6 @@ Returns data directly into the browser. Sets the mime-type according to the extension """ - if trans.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() if hid is not None: try: hid = int( hid ) @@ -149,9 +146,6 @@ fname = ''.join(c in valid_chars and c or '_' for c in fname)[0:150] trans.response.headers["Content-Disposition"] = "attachment; filename=GalaxyHistoryItem-%s-[%s]%s" % (data.hid, fname, toext) trans.log_event( "Display dataset id: %s" % str(id) ) - if self.app.memory_usage: - m1 = trans.app.memory_usage.memory( m0, pretty=True ) - log.info( "End of root/display, memory used increased by %s" % m1 ) try: return open( data.file_name ) except: @@ -328,9 +322,6 @@ @web.expose def history_delete( self, trans, id=None, **kwd): """Deletes a list of histories, ensures that histories are owned by current user""" - if trans.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() history_names = [] if id: if isinstance( id, list ): @@ -358,18 +349,12 @@ trans.log_event( "History id %s marked as deleted" % str(hid) ) else: return trans.show_message( "You must select at least one history to delete." ) - if self.app.memory_usage: - m1 = trans.app.memory_usage.memory( m0, pretty=True ) - log.info( "End of root/history_delete, memory used increased by %s" % m1 ) return trans.show_message( "History deleted: %s" % ",".join(history_names), refresh_frames=['history']) @web.expose def history_undelete( self, trans, id=[], **kwd): """Undeletes a list of histories, ensures that histories are owned by current user""" - if trans.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() history_names = [] errors = [] ok_msg = "" @@ -401,34 +386,22 @@ ok_msg = "Histories (%s) have been undeleted." % ", ".join( history_names ) else: errors.append( "You must select at least one history to undelete." ) - if self.app.memory_usage: - m1 = trans.app.memory_usage.memory( m0, pretty=True ) - log.info( "End of root/history_undelete, memory used increased by %s" % m1 ) return self.history_available( trans, id=','.join( id ), show_deleted=True, ok_msg = ok_msg, error_msg = " ".join( errors ) ) @web.expose def clear_history( self, trans ): """Clears the history for a user""" - if trans.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() history = trans.get_history() for dataset in history.datasets: dataset.deleted = True dataset.clear_associated_files() self.app.model.flush() trans.log_event( "History id %s cleared" % (str(history.id)) ) - if self.app.memory_usage: - m1 = trans.app.memory_usage.memory( m0, pretty=True ) - log.info( "End of root/clear_history, memory used increased by %s" % m1 ) trans.response.send_redirect( url_for("/index" ) ) @web.expose @web.require_login( "share histories with other users" ) def history_share( self, trans, id=None, email="", **kwd ): - if trans.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() send_to_err = "" if not id: id = trans.get_history().id @@ -455,9 +428,6 @@ trans.log_event( "History share, id: %s, name: '%s': to new id: %s" % (str(history.id), history.name, str(new_history.id)) ) self.app.model.flush() return trans.show_message( "History (%s) has been shared with: %s" % (",".join(history_names),email) ) - if self.app.memory_usage: - m1 = trans.app.memory_usage.memory( m0, pretty=True ) - log.info( "End of root/history_share, memory used increased by %s" % m1 ) return trans.fill_template( "/history/share.mako", histories=histories, email=email, send_to_err=send_to_err) @web.expose @@ -466,9 +436,6 @@ """ List all available histories """ - if trans.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() if as_xml: trans.response.set_content_type('text/xml') return trans.fill_template( "/history/list_as_xml.mako" ) @@ -476,9 +443,6 @@ id = id.split( "," ) trans.log_event( "History id %s available" % str( id ) ) history_operations = dict( share=self.history_share, rename=self.history_rename, delete=self.history_delete, undelete=self.history_undelete ) - if self.app.memory_usage: - m1 = trans.app.memory_usage.memory( m0, pretty=True ) - log.info( "End of root/history_available, memory used increased by %s" % m1 ) if do_operation in history_operations: return history_operations[do_operation]( trans, id=id, show_deleted=show_deleted, ok_msg=ok_msg, error_msg=error_msg, **kwd ) return trans.fill_template( "/history/list.mako", ids=id, @@ -491,9 +455,6 @@ @web.expose def history_import( self, trans, id=None, confirm=False, **kwd ): - if trans.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() msg = "" user = trans.get_user() user_history = trans.get_history() @@ -534,9 +495,6 @@ new_history.flush() trans.set_history( new_history ) trans.log_event( "History imported, id: %s, name: '%s': " % (str(new_history.id) , new_history.name ) ) - if self.app.memory_usage: - m1 = trans.app.memory_usage.memory( m0, pretty=True ) - log.info( "End of root/history_import, memory used increased by %s" % m1 ) return trans.show_ok_message( """ History "%s" has been imported. Click <a href="%s">here</a> to begin.""" % ( new_history.name, web.url_for( '/' ) ) ) @@ -551,9 +509,6 @@ if not id: return trans.response.send_redirect( web.url_for( action='history_available' ) ) else: - if trans.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() new_history = trans.app.model.History.get( id ) if new_history: galaxy_session = trans.get_galaxy_session() @@ -565,9 +520,6 @@ new_history.flush() trans.set_history( new_history ) trans.log_event( "History switched to id: %s, name: '%s'" % (str(new_history.id), new_history.name ) ) - if self.app.memory_usage: - m1 = trans.app.memory_usage.memory( m0, pretty=True ) - log.info( "End of root/history_switch, memory used increased by %s" % m1 ) return trans.show_message( "History switched to: %s" % new_history.name, refresh_frames=['history']) else: @@ -575,22 +527,13 @@ @web.expose def history_new( self, trans ): - if trans.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() trans.new_history() trans.log_event( "Created new History, id: %s." % str(trans.get_history().id) ) - if self.app.memory_usage: - m1 = trans.app.memory_usage.memory( m0, pretty=True ) - log.info( "End of root/history_new, memory used increased by %s" % m1 ) return trans.show_message( "New history created", refresh_frames = ['history'] ) @web.expose @web.require_login( "renames histories" ) def history_rename( self, trans, id=None, name=None, **kwd ): - if trans.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() user = trans.get_user() if not isinstance( id, list ): @@ -627,17 +570,11 @@ change_msg = change_msg + "<p>You must specify a valid name for History: "+cur_names[i]+"</p>" else: change_msg = change_msg + "<p>History: "+cur_names[i]+" does not appear to belong to you.</p>" - if self.app.memory_usage: - m1 = trans.app.memory_usage.memory( m0, pretty=True ) - log.info( "End of root/history_rename, memory used increased by %s" % m1 ) return trans.show_message( "<p>%s" % change_msg, refresh_frames=['history'] ) @web.expose def history_add_to( self, trans, history_id=None, file_data=None, name="Data Added to History",info=None,ext="txt",dbkey="?",**kwd ): """Adds a POSTed file to a History""" - if trans.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() try: history = trans.app.model.History.get( history_id ) data = trans.app.model.HistoryDatasetAssociation( name = name, info = info, extension = ext, dbkey = dbkey, create_dataset = True ) @@ -656,9 +593,6 @@ data.set_peek() data.flush() trans.log_event("Added dataset %d to history %d" %(data.id, trans.history.id)) - if self.app.memory_usage: - m1 = trans.app.memory_usage.memory( m0, pretty=True ) - log.info( "End of root/history_add_to, memory used increased by %s" % m1 ) return trans.show_ok_message("Dataset "+str(data.hid)+" added to history "+str(history_id)+".") except: return trans.show_error_message("Adding File to History has Failed") @@ -666,9 +600,6 @@ @web.expose def dataset_make_primary( self, trans, id=None): """Copies a dataset and makes primary""" - if trans.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() try: old_data = self.app.model.HistoryDatasetAssociation.get( id ) new_data = old_data.copy() @@ -677,9 +608,6 @@ history = trans.get_history() history.add_dataset(new_data) new_data.flush() - if self.app.memory_usage: - m1 = trans.app.memory_usage.memory( m0, pretty=True ) - log.info( "End of root/dataset_make_primary, memory used increased by %s" % m1 ) return trans.show_message( "<p>Secondary dataset has been made primary.</p>", refresh_frames=['history'] ) except: return trans.show_error_message( "<p>Failed to make secondary dataset primary.</p>" ) diff -r 511c2b1966dc -r 03f1e9e78f74 lib/galaxy/web/controllers/user.py --- a/lib/galaxy/web/controllers/user.py Tue Jan 13 10:55:08 2009 -0500 +++ b/lib/galaxy/web/controllers/user.py Tue Jan 13 11:33:17 2009 -0500 @@ -70,9 +70,6 @@ @web.expose def login( self, trans, email='', password='' ): - if trans.app.memory_usage: - # Keep track of memory usage - m0 = trans.app.memory_usage.memory() email_error = password_error = None # Attempt login if email or password: @@ -88,9 +85,6 @@ trans.handle_user_login( user ) trans.log_event( "User logged in" ) return trans.show_ok_message( "Now logged in as " + user.email, refresh_frames=['masthead', 'history'] ) - if trans.app.memory_usage: - m1 = trans.app.memory_usage.memory( m0, pretty=True ) - log.info( "End of user/login, memory used increased by %s" % m1 ) return trans.show_form( web.FormBuilder( web.url_for(), "Login", submit_text="Login" ) .add_text( "email", "Email address", value=email, error=email_error ) @@ -98,22 +92,13 @@ help="<a href='%s'>Forgot password? Reset here</a>" % web.url_for( action='reset_password' ) ) ) @web.expose def logout( self, trans ): - if trans.app.memory_usage: - # Keep track of memory usage - m0 = trans.app.memory_usage.memory() # Since logging an event requires a session, we'll log prior to ending the session trans.log_event( "User logged out" ) trans.handle_user_logout() - if trans.app.memory_usage: - m1 = trans.app.memory_usage.memory( m0, pretty=True ) - log.info( "End of user/logout, memory used increased by %s" % m1 ) return trans.show_ok_message( "You are no longer logged in", refresh_frames=['masthead', 'history'] ) @web.expose def create( self, trans, email='', password='', confirm='',subscribe=False ): - if trans.app.memory_usage: - # Keep track of memory usage - m0 = trans.app.memory_usage.memory() email_error = password_error = confirm_error = None if email: if len( email ) == 0 or "@" not in email or "." not in email: @@ -139,9 +124,6 @@ mail.write("To: %s\nFrom: %s\nSubject: Join Mailing List\n\nJoin Mailing list." % (trans.app.config.mailing_join_addr,email) ) if mail.close(): return trans.show_warn_message( "Now logged in as " + user.email+". However, subscribing to the mailing list has failed.", refresh_frames=['masthead', 'history'] ) - if trans.app.memory_usage: - m1 = trans.app.memory_usage.memory( m0, pretty=True ) - log.info( "End of user/create, memory used increased by %s" % m1 ) return trans.show_ok_message( "Now logged in as " + user.email, refresh_frames=['masthead', 'history'] ) return trans.show_form( web.FormBuilder( web.url_for(), "Create account", submit_text="Create" )