- galaxy-dev - lists.galaxyproject.org

[hg] galaxy 1606: Make logging events a config setting.
by Greg Von Kuster 05 Nov '08

05 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/72252f33edd1 changeset: 1606:72252f33edd1 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Wed Nov 05 09:28:48 2008 -0500 description: Make logging events a config setting. 3 file(s) affected in this change: lib/galaxy/config.py lib/galaxy/web/framework/__init__.py universe_wsgi.ini.sample diffs (68 lines): diff -r f2e583f7cab9 -r 72252f33edd1 lib/galaxy/config.py --- a/lib/galaxy/config.py Tue Nov 04 16:04:49 2008 -0500 +++ b/lib/galaxy/config.py Wed Nov 05 09:28:48 2008 -0500 @@ -58,6 +58,7 @@ self.use_heartbeat = string_as_bool( kwargs.get( 'use_heartbeat', False ) ) self.use_memdump = string_as_bool( kwargs.get( 'use_memdump', False ) ) self.log_memory_usage = string_as_bool( kwargs.get( 'log_memory_usage', False ) ) + self.log_events = string_as_bool( kwargs.get( 'log_events', False ) ) self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea" ).lower().split(",") self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "wormbase,flybase,elegans" ).lower().split(",") self.brand = kwargs.get( 'brand', None ) diff -r f2e583f7cab9 -r 72252f33edd1 lib/galaxy/web/framework/__init__.py --- a/lib/galaxy/web/framework/__init__.py Tue Nov 04 16:04:49 2008 -0500 +++ b/lib/galaxy/web/framework/__init__.py Wed Nov 05 09:28:48 2008 -0500 @@ -127,23 +127,24 @@ return self.app.model.context.current def log_event( self, message, tool_id=None, **kwargs ): """ - Application level logging. Still needs fleshing out (log levels and - such) + Application level logging. Still needs fleshing out (log levels and such) + Logging events is a config setting - if False, do not log. """ - event = self.app.model.Event() - event.tool_id = tool_id - try: - event.message = message % kwargs - except: - event.message = message - event.history = self.history - try: - event.history_id = self.history.id - except: - event.history_id = None - event.user = self.user - event.session_id = self.galaxy_session.id - event.flush() + if self.app.config.log_events: + event = self.app.model.Event() + event.tool_id = tool_id + try: + event.message = message % kwargs + except: + event.message = message + event.history = self.history + try: + event.history_id = self.history.id + except: + event.history_id = None + event.user = self.user + event.session_id = self.galaxy_session.id + event.flush() def get_cookie( self, name='galaxysession' ): """Convienience method for getting the galaxysession cookie""" try: diff -r f2e583f7cab9 -r 72252f33edd1 universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample Tue Nov 04 16:04:49 2008 -0500 +++ b/universe_wsgi.ini.sample Wed Nov 05 09:28:48 2008 -0500 @@ -18,6 +18,9 @@ # Log memory usage log_memory_usage = False + +# Log events +log_events = True # Should jobs be tracked through the database, rather than in memory ## track_jobs_in_database = true

1 0

[hg] galaxy 1605: Add ability to track memory usage in code secm...
by Greg Von Kuster 04 Nov '08

04 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/f2e583f7cab9 changeset: 1605:f2e583f7cab9 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Tue Nov 04 16:04:49 2008 -0500 description: Add ability to track memory usage in code secments - requires config addition: # Log memory usage log_memory_usage = False Added logging statements in tool execution methods and most methods in the user and root controllers. Also fixes for checking config settings that should be boolean ( e.g., memdump, heartbeat ). 9 file(s) affected in this change: lib/galaxy/app.py lib/galaxy/config.py lib/galaxy/eggs/__init__.py lib/galaxy/tools/actions/__init__.py lib/galaxy/tools/actions/upload.py lib/galaxy/util/memory_usage.py lib/galaxy/web/controllers/root.py lib/galaxy/web/controllers/user.py universe_wsgi.ini.sample diffs (469 lines): diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/app.py --- a/lib/galaxy/app.py Tue Nov 04 14:00:30 2008 -0500 +++ b/lib/galaxy/app.py Tue Nov 04 16:04:49 2008 -0500 @@ -38,6 +38,7 @@ # Heartbeat and memdump for thread / heap profiling self.heartbeat = None self.memdump = None + self.memory_usage = None # Start the heartbeat process if configured and available if self.config.use_heartbeat: from galaxy.util import heartbeat @@ -49,6 +50,10 @@ from galaxy.util import memdump if memdump.Memdump: self.memdump = memdump.Memdump() + # Enable memory_usage logging if configured + if self.config.log_memory_usage: + from galaxy.util import memory_usage + self.memory_usage = memory_usage def shutdown( self ): self.job_manager.shutdown() if self.heartbeat: diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/config.py --- a/lib/galaxy/config.py Tue Nov 04 14:00:30 2008 -0500 +++ b/lib/galaxy/config.py Tue Nov 04 16:04:49 2008 -0500 @@ -55,8 +55,9 @@ self.pbs_dataset_server = kwargs.get('pbs_dataset_server', "" ) self.pbs_dataset_path = kwargs.get('pbs_dataset_path', "" ) self.pbs_stage_path = kwargs.get('pbs_stage_path', "" ) - self.use_heartbeat = string_as_bool( kwargs.get( 'use_heartbeat', "False" ) ) - self.use_memdump = kwargs.get( 'use_memdump', False ) + self.use_heartbeat = string_as_bool( kwargs.get( 'use_heartbeat', False ) ) + self.use_memdump = string_as_bool( kwargs.get( 'use_memdump', False ) ) + self.log_memory_usage = string_as_bool( kwargs.get( 'log_memory_usage', False ) ) self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea" ).lower().split(",") self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "wormbase,flybase,elegans" ).lower().split(",") self.brand = kwargs.get( 'brand', None ) diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/eggs/__init__.py --- a/lib/galaxy/eggs/__init__.py Tue Nov 04 14:00:30 2008 -0500 +++ b/lib/galaxy/eggs/__init__.py Tue Nov 04 16:04:49 2008 -0500 @@ -484,7 +484,7 @@ return False elif egg_name == "threadframe": try: - if self.config.get( "app:main", "use_heartbeat" ) == "True": + if self.config.get( "app:main", "use_heartbeat" ): return True else: return False @@ -492,7 +492,7 @@ return False elif egg_name == "guppy": try: - if self.config.get( "app:main", "use_memdump" ) == "True": + if self.config.get( "app:main", "use_memdump" ): return True else: return False diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/tools/actions/__init__.py --- a/lib/galaxy/tools/actions/__init__.py Tue Nov 04 14:00:30 2008 -0500 +++ b/lib/galaxy/tools/actions/__init__.py Tue Nov 04 16:04:49 2008 -0500 @@ -65,6 +65,9 @@ return input_datasets def execute(self, tool, trans, incoming={}, set_output_hid=True ): + if trans.app.memory_usage: + # Keep track of memory usage + m0 = trans.app.memory_usage.memory() out_data = {} # Collect any input datasets from the incoming parameters inp_data = self.collect_input_datasets( tool, incoming, trans ) @@ -198,6 +201,9 @@ for name, dataset in out_data.iteritems(): job.add_output_dataset( name, dataset ) trans.app.model.flush() + if trans.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info("End of tool %s execution for job id %d, memory used increased by %s" % ( tool.id, job.id, m1 ) ) # Some tools are not really executable, but jobs are still created for them ( for record keeping ). # Examples include tools that redirect to other applications ( epigraph ). These special tools must # include something that can be retrieved from the params ( e.g., REDIRECT_URL ) to keep the job diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/tools/actions/upload.py --- a/lib/galaxy/tools/actions/upload.py Tue Nov 04 14:00:30 2008 -0500 +++ b/lib/galaxy/tools/actions/upload.py Tue Nov 04 16:04:49 2008 -0500 @@ -15,6 +15,9 @@ self.line_count = None def execute( self, tool, trans, incoming={}, set_output_hid = True ): + if trans.app.memory_usage: + # Keep track of memory usage + m0 = trans.app.memory_usage.memory() data_file = incoming['file_data'] file_type = incoming['file_type'] dbkey = incoming['dbkey'] @@ -93,6 +96,9 @@ job.flush() log.info( 'job id %d ended ok, file size: %s' % ( job.id, file_size_str ) ) trans.log_event( 'job id %d ended ok, file size: %s' % ( job.id, file_size_str ), tool_id=tool.id ) + if trans.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info("End of tool %s execution for job id %d, memory used increased by %s" % ( tool.id, job.id, m1 ) ) return dict( output=hda ) def upload_empty(self, trans, job, err_code, err_msg): diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/util/memory_usage.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/util/memory_usage.py Tue Nov 04 16:04:49 2008 -0500 @@ -0,0 +1,44 @@ +import os, platform, logging +from galaxy.datatypes.data import nice_size + +log = logging.getLogger( __name__ ) + +_proc_status = '/proc/%d/status' % os.getpid() + +_scale = { 'kB': 1024.0, 'mB': 1024.0*1024.0, 'KB': 1024.0, 'MB': 1024.0*1024.0 } + +def _VmB( VmKey ): + '''Private. + ''' + global _proc_status, _scale + # get pseudo file /proc/<pid>/status + try: + t = open( _proc_status ) + v = t.read() + t.close() + except: + log.debug("memory_usage is currently supported only on Linux, your platform is %s %s" % ( platform.system(), platform.release() ) ) + return 0.0 # non-Linux? + # get VmKey line e.g. 'VmRSS: 9999 kB\n ...' + i = v.index( VmKey ) + v = v[ i: ].split( None, 3 ) # whitespace + if len( v ) < 3: + return 0.0 # invalid format? + # convert Vm value to bytes + return float( v[ 1 ] ) * _scale[ v[ 2 ] ] +def memory( since=0.0, pretty=False ): + '''Return memory usage in bytes. + ''' + size = _VmB( 'VmSize:' ) - since + if pretty: + return nice_size( size ) + else: + return size +def resident( since=0.0 ): + '''Return resident memory usage in bytes. + ''' + return _VmB( 'VmRSS:' ) - since +def stacksize( since=0.0 ): + '''Return stack size in bytes. + ''' + return _VmB( 'VmStk:' ) - since diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/web/controllers/root.py --- a/lib/galaxy/web/controllers/root.py Tue Nov 04 14:00:30 2008 -0500 +++ b/lib/galaxy/web/controllers/root.py Tue Nov 04 16:04:49 2008 -0500 @@ -116,6 +116,9 @@ Returns data directly into the browser. Sets the mime-type according to the extension """ + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() if hid is not None: try: hid = int( hid ) @@ -146,6 +149,9 @@ fname = ''.join(c in valid_chars and c or '_' for c in fname)[0:150] trans.response.headers["Content-Disposition"] = "attachment; filename=GalaxyHistoryItem-%s-[%s]%s" % (data.hid, fname, toext) trans.log_event( "Display dataset id: %s" % str(id) ) + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/display, memory used increased by %s" % m1 ) try: return open( data.file_name ) except: @@ -321,6 +327,9 @@ @web.expose def history_delete( self, trans, id=None, **kwd): """Deletes a list of histories, ensures that histories are owned by current user""" + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() history_names = [] if id: if isinstance( id, list ): @@ -348,12 +357,18 @@ trans.log_event( "History id %s marked as deleted" % str(hid) ) else: return trans.show_message( "You must select at least one history to delete." ) + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/history_delete, memory used increased by %s" % m1 ) return trans.show_message( "History deleted: %s" % ",".join(history_names), refresh_frames=['history']) @web.expose def history_undelete( self, trans, id=[], **kwd): """Undeletes a list of histories, ensures that histories are owned by current user""" + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() history_names = [] errors = [] ok_msg = "" @@ -385,22 +400,34 @@ ok_msg = "Histories (%s) have been undeleted." % ", ".join( history_names ) else: errors.append( "You must select at least one history to undelete." ) + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/history_undelete, memory used increased by %s" % m1 ) return self.history_available( trans, id=','.join( id ), show_deleted=True, ok_msg = ok_msg, error_msg = " ".join( errors ) ) @web.expose def clear_history( self, trans ): """Clears the history for a user""" + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() history = trans.get_history() for dataset in history.datasets: dataset.deleted = True dataset.clear_associated_files() self.app.model.flush() trans.log_event( "History id %s cleared" % (str(history.id)) ) + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/clear_history, memory used increased by %s" % m1 ) trans.response.send_redirect( url_for("/index" ) ) @web.expose @web.require_login( "share histories with other users" ) def history_share( self, trans, id=None, email="", **kwd ): + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() send_to_err = "" if not id: id = trans.get_history().id @@ -427,6 +454,9 @@ trans.log_event( "History share, id: %s, name: '%s': to new id: %s" % (str(history.id), history.name, str(new_history.id)) ) self.app.model.flush() return trans.show_message( "History (%s) has been shared with: %s" % (",".join(history_names),email) ) + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/history_share, memory used increased by %s" % m1 ) return trans.fill_template( "/history/share.mako", histories=histories, email=email, send_to_err=send_to_err) @web.expose @@ -435,18 +465,21 @@ """ List all available histories """ + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() if as_xml: trans.response.set_content_type('text/xml') return trans.fill_template( "/history/list_as_xml.mako" ) if not isinstance( id, list ): id = id.split( "," ) trans.log_event( "History id %s available" % str( id ) ) - history_operations = dict( share=self.history_share, rename=self.history_rename, delete=self.history_delete, undelete=self.history_undelete ) - + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/history_available, memory used increased by %s" % m1 ) if do_operation in history_operations: return history_operations[do_operation]( trans, id=id, show_deleted=show_deleted, ok_msg=ok_msg, error_msg=error_msg, **kwd ) - return trans.fill_template( "/history/list.mako", ids=id, user=trans.get_user(), current_history=trans.get_history(), @@ -457,6 +490,9 @@ @web.expose def history_import( self, trans, id=None, confirm=False, **kwd ): + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() msg = "" user = trans.get_user() user_history = trans.get_history() @@ -497,6 +533,9 @@ new_history.flush() trans.set_history( new_history ) trans.log_event( "History imported, id: %s, name: '%s': " % (str(new_history.id) , new_history.name ) ) + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/history_import, memory used increased by %s" % m1 ) return trans.show_ok_message( """ History "%s" has been imported. Click <a href="%s">here</a> to begin.""" % ( new_history.name, web.url_for( '/' ) ) ) @@ -511,6 +550,9 @@ if not id: return trans.response.send_redirect( web.url_for( action='history_available' ) ) else: + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() new_history = trans.app.model.History.get( id ) if new_history: galaxy_session = trans.get_galaxy_session() @@ -522,6 +564,9 @@ new_history.flush() trans.set_history( new_history ) trans.log_event( "History switched to id: %s, name: '%s'" % (str(new_history.id), new_history.name ) ) + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/history_switch, memory used increased by %s" % m1 ) return trans.show_message( "History switched to: %s" % new_history.name, refresh_frames=['history']) else: @@ -529,13 +574,22 @@ @web.expose def history_new( self, trans ): + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() trans.new_history() trans.log_event( "Created new History, id: %s." % str(trans.get_history().id) ) + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/history_new, memory used increased by %s" % m1 ) return trans.show_message( "New history created", refresh_frames = ['history'] ) @web.expose @web.require_login( "renames histories" ) def history_rename( self, trans, id=None, name=None, **kwd ): + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() user = trans.get_user() if not isinstance( id, list ): @@ -572,11 +626,17 @@ change_msg = change_msg + "You must specify a valid name for History: "+cur_names[i]+"" else: change_msg = change_msg + "History: "+cur_names[i]+" does not appear to belong to you." + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/history_rename, memory used increased by %s" % m1 ) return trans.show_message( "%s" % change_msg, refresh_frames=['history'] ) @web.expose def history_add_to( self, trans, history_id=None, file_data=None, name="Data Added to History",info=None,ext="txt",dbkey="?",**kwd ): """Adds a POSTed file to a History""" + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() try: history = trans.app.model.History.get( history_id ) data = trans.app.model.HistoryDatasetAssociation( name = name, info = info, extension = ext, dbkey = dbkey, create_dataset = True ) @@ -595,6 +655,9 @@ data.set_size() data.flush() trans.log_event("Added dataset %d to history %d" %(data.id, trans.history.id)) + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/history_add_to, memory used increased by %s" % m1 ) return trans.show_ok_message("Dataset "+str(data.hid)+" added to history "+str(history_id)+".") except: return trans.show_error_message("Adding File to History has Failed") @@ -602,6 +665,9 @@ @web.expose def dataset_make_primary( self, trans, id=None): """Copies a dataset and makes primary""" + if trans.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() try: old_data = self.app.model.HistoryDatasetAssociation.get( id ) new_data = old_data.copy() @@ -610,6 +676,9 @@ history = trans.get_history() history.add_dataset(new_data) new_data.flush() + if self.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of root/dataset_make_primary, memory used increased by %s" % m1 ) return trans.show_message( "Secondary dataset has been made primary.", refresh_frames=['history'] ) except: return trans.show_error_message( "Failed to make secondary dataset primary." ) diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/web/controllers/user.py --- a/lib/galaxy/web/controllers/user.py Tue Nov 04 14:00:30 2008 -0500 +++ b/lib/galaxy/web/controllers/user.py Tue Nov 04 16:04:49 2008 -0500 @@ -70,6 +70,9 @@ @web.expose def login( self, trans, email='', password='' ): + if trans.app.memory_usage: + # Keep track of memory usage + m0 = trans.app.memory_usage.memory() email_error = password_error = None # Attempt login if email or password: @@ -85,6 +88,9 @@ trans.handle_user_login( user ) trans.log_event( "User logged in" ) return trans.show_ok_message( "Now logged in as " + user.email, refresh_frames=['masthead', 'history'] ) + if trans.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of user/login, memory used increased by %s" % m1 ) return trans.show_form( web.FormBuilder( web.url_for(), "Login", submit_text="Login" ) .add_text( "email", "Email address", value=email, error=email_error ) @@ -92,13 +98,22 @@ help="<a href='%s'>Forgot password? Reset here</a>" % web.url_for( action='reset_password' ) ) ) @web.expose def logout( self, trans ): + if trans.app.memory_usage: + # Keep track of memory usage + m0 = trans.app.memory_usage.memory() # Since logging an event requires a session, we'll log prior to ending the session trans.log_event( "User logged out" ) trans.handle_user_logout() + if trans.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of user/logout, memory used increased by %s" % m1 ) return trans.show_ok_message( "You are no longer logged in", refresh_frames=['masthead', 'history'] ) @web.expose def create( self, trans, email='', password='', confirm='',subscribe=False ): + if trans.app.memory_usage: + # Keep track of memory usage + m0 = trans.app.memory_usage.memory() email_error = password_error = confirm_error = None if email: if len( email ) == 0 or "@" not in email or "." not in email: @@ -120,10 +135,13 @@ trans.log_event( "User logged in" ) #subscribe user to email list if subscribe: - mail = os.popen("%s -t" % self.app.config.sendmail_path, 'w') - mail.write("To: %s\nFrom: %s\nSubject: Join Mailing List\n\nJoin Mailing list." % (self.app.config.mailing_join_addr,email) ) + mail = os.popen("%s -t" % trans.app.config.sendmail_path, 'w') + mail.write("To: %s\nFrom: %s\nSubject: Join Mailing List\n\nJoin Mailing list." % (trans.app.config.mailing_join_addr,email) ) if mail.close(): return trans.show_warn_message( "Now logged in as " + user.email+". However, subscribing to the mailing list has failed.", refresh_frames=['masthead', 'history'] ) + if trans.app.memory_usage: + m1 = trans.app.memory_usage.memory( m0, pretty=True ) + log.info( "End of user/create, memory used increased by %s" % m1 ) return trans.show_ok_message( "Now logged in as " + user.email, refresh_frames=['masthead', 'history'] ) return trans.show_form( web.FormBuilder( web.url_for(), "Create account", submit_text="Create" ) @@ -145,7 +163,7 @@ new_pass = "" for i in range(15): new_pass = new_pass + choice(chars) - mail = os.popen("%s -t" % self.app.config.sendmail_path, 'w') + mail = os.popen("%s -t" % trans.app.config.sendmail_path, 'w') mail.write("To: %s\nFrom: no-reply@%s\nSubject: Galaxy Password Reset\n\nYour password has been reset to \"%s\" (no quotes)." % (email, trans.request.remote_addr, new_pass) ) if mail.close(): return trans.show_ok_message( "Failed to reset password! If this problem persist, submit a bug report.") diff -r 7a5d9759c702 -r f2e583f7cab9 universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample Tue Nov 04 14:00:30 2008 -0500 +++ b/universe_wsgi.ini.sample Tue Nov 04 16:04:49 2008 -0500 @@ -15,6 +15,9 @@ # Specifies the factory for the universe WSGI application paste.app_factory = galaxy.web.buildapp:app_factory log_level = DEBUG + +# Log memory usage +log_memory_usage = False # Should jobs be tracked through the database, rather than in memory ## track_jobs_in_database = true

1 0

[hg] galaxy 1604: Fix get_bool in config
by Greg Von Kuster 04 Nov '08

04 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/7a5d9759c702 changeset: 1604:7a5d9759c702 user: Nate Coraor <nate(a)bx.psu.edu> date: Tue Nov 04 14:00:30 2008 -0500 description: Fix get_bool in config 1 file(s) affected in this change: lib/galaxy/config.py diffs (12 lines): diff -r 85347b21a1b8 -r 7a5d9759c702 lib/galaxy/config.py --- a/lib/galaxy/config.py Tue Nov 04 13:20:39 2008 -0500 +++ b/lib/galaxy/config.py Tue Nov 04 14:00:30 2008 -0500 @@ -85,7 +85,7 @@ return self.config_dict.get( key, default ) def get_bool( self, key, default ): if key in self.config_dict: - return string_as_bool( key ) + return string_as_bool( self.config_dict[key] ) else: return default def check( self ):

1 0

[hg] galaxy 1602: Upload tool will now create a job record, no j...
by Greg Von Kuster 04 Nov '08

04 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/531be9ea2486 changeset: 1602:531be9ea2486 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Tue Nov 04 11:56:09 2008 -0500 description: Upload tool will now create a job record, no job queued though. 1 file(s) affected in this change: lib/galaxy/tools/actions/upload.py diffs (120 lines): diff -r 52e504c91cbc -r 531be9ea2486 lib/galaxy/tools/actions/upload.py --- a/lib/galaxy/tools/actions/upload.py Tue Nov 04 09:27:49 2008 -0500 +++ b/lib/galaxy/tools/actions/upload.py Tue Nov 04 11:56:09 2008 -0500 @@ -25,7 +25,19 @@ space_to_tab = True temp_name = "" data_list = [] - + # Create the job object + job = trans.app.model.Job() + job.session_id = trans.get_galaxy_session().id + job.history_id = trans.history.id + job.tool_id = tool.id + try: + # For backward compatibility, some tools may not have versions yet. + job.tool_version = tool.version + except: + job.tool_version = "1.0.0" + job.flush() + log.debug( 'tool %s created job id %d' % ( tool.id, job.id ) ) + trans.log_event( 'created job id %d' % job.id, tool_id=tool.id ) if 'local_filename' in dir( data_file ): # Use the existing file try: @@ -34,7 +46,7 @@ file_name = file_name.split( '/' )[-1] data_list.append( self.add_file( trans, data_file.local_filename, file_name, file_type, dbkey, space_to_tab=space_to_tab ) ) except Exception, e: - return self.upload_empty( trans, "Error:", str( e ) ) + return self.upload_empty( trans, job, "Error:", str( e ) ) elif 'filename' in dir( data_file ): try: file_name = data_file.filename @@ -43,7 +55,7 @@ temp_name = sniff.stream_to_file( data_file.file ) data_list.append( self.add_file( trans, temp_name, file_name, file_type, dbkey, space_to_tab=space_to_tab ) ) except Exception, e: - return self.upload_empty( trans, "Error:", str( e ) ) + return self.upload_empty( trans, job, "Error:", str( e ) ) if url_paste not in [ None, "" ]: if url_paste.lower().find( 'http://' ) >= 0 or url_paste.lower().find( 'ftp://' ) >= 0: url_paste = url_paste.replace( '\r', '' ).split( '\n' ) @@ -54,7 +66,7 @@ temp_name = sniff.stream_to_file( urllib.urlopen( line ) ) data_list.append( self.add_file( trans, temp_name, line, file_type, dbkey, info="uploaded url", space_to_tab=space_to_tab ) ) except Exception, e: - return self.upload_empty( trans, "Error:", str( e ) ) + return self.upload_empty( trans, job, "Error:", str( e ) ) else: is_valid = False for line in url_paste: @@ -67,26 +79,40 @@ temp_name = sniff.stream_to_file( StringIO.StringIO( url_paste ) ) data_list.append( self.add_file( trans, temp_name, 'Pasted Entry', file_type, dbkey, info="pasted entry", space_to_tab=space_to_tab ) ) except Exception, e: - return self.upload_empty( trans, "Error:", str( e ) ) + return self.upload_empty( trans, job, "Error:", str( e ) ) else: - return self.upload_empty( trans, "No data error:", "you pasted no data." ) + return self.upload_empty( trans, job, "No data error:", "you pasted no data." ) if self.empty: - return self.upload_empty( trans, "Empty file error:", "you attempted to upload an empty file." ) + return self.upload_empty( trans, job, "Empty file error:", "you attempted to upload an empty file." ) elif len( data_list ) < 1: - return self.upload_empty( trans, "No data error:", "either you pasted no data, the url you specified is invalid, or you have not specified a file." ) - return dict( output=data_list[0] ) + return self.upload_empty( trans, job, "No data error:", "either you pasted no data, the url you specified is invalid, or you have not specified a file." ) + hda = data_list[0] + job.state = trans.app.model.Job.states.OK + file_size_str = nice_size( hda.dataset.file_size ) + job.info = "Uploaded file '%s', size: %s" % ( file_name, file_size_str ) + job.flush() + log.debug( 'tool %s job id %d ended ok, file: %s, size: %s' % ( tool.id, job.id, file_name, file_size_str ) ) + trans.log_event( 'job id %d ended ok, file: %s, size: %s' % ( job.id, file_name, file_size_str ), tool_id=tool.id ) + return dict( output=hda ) - def upload_empty(self, trans, err_code, err_msg): + def upload_empty(self, trans, job, err_code, err_msg): data = trans.app.model.HistoryDatasetAssociation( create_dataset = True ) data.name = err_code data.extension = "txt" data.dbkey = "?" data.info = err_msg data.file_size = 0 + data.state = data.states.EMPTY data.flush() - data.state = data.states.EMPTY trans.history.add_dataset( data ) trans.app.model.flush() + # Indicate job failure by setting state and message + job.state = trans.app.model.Job.states.ERROR + job.info = err_msg + # If the failure is due to a Galaxy framework exception, save the traceback + job.flush() + log.debug( 'tool %s job id %d ended with errors' % ( job.tool_id, job.id ) ) + trans.log_event( 'job id %d ended with errors' % job.id, tool_id=job.tool_id ) return dict( output=data ) def add_file( self, trans, temp_name, file_name, file_type, dbkey, info=None, space_to_tab=False ): @@ -275,5 +301,20 @@ temp.close() return False +def nice_size( size ): + """Returns a readably formatted string with the size""" + words = [ 'bytes', 'Kb', 'Mb', 'Gb' ] + try: + size = float( size ) + except: + return '??? bytes' + for ind, word in enumerate( words ): + step = 1024 ** ( ind + 1 ) + if step > size: + size = size / float( 1024 ** ind ) + out = "%.1f %s" % ( size, word ) + return out + return '??? bytes' + class BadFileException( Exception ): pass

1 0

[hg] galaxy 1601: Apply James' faster_random patch from Nov 1.
by Greg Von Kuster 04 Nov '08

04 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/52e504c91cbc changeset: 1601:52e504c91cbc user: Greg Von Kuster <greg(a)bx.psu.edu> date: Tue Nov 04 09:27:49 2008 -0500 description: Apply James' faster_random patch from Nov 1. 1 file(s) affected in this change: lib/galaxy/web/security/__init__.py diffs (34 lines): diff -r 7ae75e6d9d6a -r 52e504c91cbc lib/galaxy/web/security/__init__.py --- a/lib/galaxy/web/security/__init__.py Mon Nov 03 15:31:38 2008 -0500 +++ b/lib/galaxy/web/security/__init__.py Tue Nov 04 09:27:49 2008 -0500 @@ -10,18 +10,17 @@ log = logging.getLogger( __name__ ) if os.path.exists( "/dev/urandom" ): - log.debug("###using /dev/urandom....") # We have urandom, use it as the source of random data random_fd = os.open( "/dev/urandom", os.O_RDONLY ) def get_random_bytes( nbytes ): value = os.read( random_fd, nbytes ) # Normally we should get as much as we need if len( value ) == nbytes: - return value + return value.encode( "hex" ) # If we don't, keep reading (this is slow and should never happen) while len( value ) < nbytes: value += os.read( random_fd, nbytes - len( value ) ) - return value + return value.encode( "hex" ) else: def get_random_bytes( nbytes ): nbits = nbytes * 8 @@ -29,7 +28,8 @@ while random_pool.entropy < nbits: random_pool.add_event() random_pool.stir() - return( str( number.getRandomNumber( nbits, random_pool.get_bytes ) ) ) + return str( number.getRandomNumber( nbits, random_pool.get_bytes ) ) + class SecurityHelper( object ): # TODO: checking if histories/datasets are owned by the current user) will be moved here.

1 0

[hg] galaxy 1603: Fixes for upload tool job creation.
by Greg Von Kuster 04 Nov '08

04 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/85347b21a1b8 changeset: 1603:85347b21a1b8 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Tue Nov 04 13:20:39 2008 -0500 description: Fixes for upload tool job creation. 1 file(s) affected in this change: lib/galaxy/tools/actions/upload.py diffs (58 lines): diff -r 531be9ea2486 -r 85347b21a1b8 lib/galaxy/tools/actions/upload.py --- a/lib/galaxy/tools/actions/upload.py Tue Nov 04 11:56:09 2008 -0500 +++ b/lib/galaxy/tools/actions/upload.py Tue Nov 04 13:20:39 2008 -0500 @@ -36,7 +36,7 @@ except: job.tool_version = "1.0.0" job.flush() - log.debug( 'tool %s created job id %d' % ( tool.id, job.id ) ) + log.info( 'tool %s created job id %d' % ( tool.id, job.id ) ) trans.log_event( 'created job id %d' % job.id, tool_id=tool.id ) if 'local_filename' in dir( data_file ): # Use the existing file @@ -88,11 +88,11 @@ return self.upload_empty( trans, job, "No data error:", "either you pasted no data, the url you specified is invalid, or you have not specified a file." ) hda = data_list[0] job.state = trans.app.model.Job.states.OK - file_size_str = nice_size( hda.dataset.file_size ) - job.info = "Uploaded file '%s', size: %s" % ( file_name, file_size_str ) + file_size_str = datatypes.data.nice_size( hda.dataset.file_size ) + job.info = "%s, size: %s" % ( hda.info, file_size_str ) job.flush() - log.debug( 'tool %s job id %d ended ok, file: %s, size: %s' % ( tool.id, job.id, file_name, file_size_str ) ) - trans.log_event( 'job id %d ended ok, file: %s, size: %s' % ( job.id, file_name, file_size_str ), tool_id=tool.id ) + log.info( 'job id %d ended ok, file size: %s' % ( job.id, file_size_str ) ) + trans.log_event( 'job id %d ended ok, file size: %s' % ( job.id, file_size_str ), tool_id=tool.id ) return dict( output=hda ) def upload_empty(self, trans, job, err_code, err_msg): @@ -111,7 +111,7 @@ job.info = err_msg # If the failure is due to a Galaxy framework exception, save the traceback job.flush() - log.debug( 'tool %s job id %d ended with errors' % ( job.tool_id, job.id ) ) + log.info( 'job id %d ended with errors' % job.id ) trans.log_event( 'job id %d ended with errors' % job.id, tool_id=job.tool_id ) return dict( output=data ) @@ -301,20 +301,5 @@ temp.close() return False -def nice_size( size ): - """Returns a readably formatted string with the size""" - words = [ 'bytes', 'Kb', 'Mb', 'Gb' ] - try: - size = float( size ) - except: - return '??? bytes' - for ind, word in enumerate( words ): - step = 1024 ** ( ind + 1 ) - if step > size: - size = size / float( 1024 ** ind ) - out = "%.1f %s" % ( size, word ) - return out - return '??? bytes' - class BadFileException( Exception ): pass

1 0

[hg] galaxy 1600: Integrate "session_create_speed" and "faster_r...
by Greg Von Kuster 04 Nov '08

04 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/7ae75e6d9d6a changeset: 1600:7ae75e6d9d6a user: Greg Von Kuster <greg(a)bx.psu.edu> date: Mon Nov 03 15:31:38 2008 -0500 description: Integrate "session_create_speed" and "faster_random" patches from James. 8 file(s) affected in this change: lib/galaxy/model/mapping.py lib/galaxy/tools/actions/__init__.py lib/galaxy/web/controllers/root.py lib/galaxy/web/controllers/tool_runner.py lib/galaxy/web/controllers/user.py lib/galaxy/web/framework/__init__.py lib/galaxy/web/security/__init__.py templates/root/history.mako diffs (650 lines): diff -r 13cbdd1bbd19 -r 7ae75e6d9d6a lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Mon Nov 03 15:21:03 2008 -0500 +++ b/lib/galaxy/model/mapping.py Mon Nov 03 15:31:38 2008 -0500 @@ -326,6 +326,7 @@ assign_mapper( context, GalaxySession, GalaxySession.table, properties=dict( histories=relation( GalaxySessionToHistoryAssociation ), + current_history=relation( History ), user=relation( User.mapper ) ) ) assign_mapper( context, GalaxySessionToHistoryAssociation, GalaxySessionToHistoryAssociation.table, diff -r 13cbdd1bbd19 -r 7ae75e6d9d6a lib/galaxy/tools/actions/__init__.py --- a/lib/galaxy/tools/actions/__init__.py Mon Nov 03 15:21:03 2008 -0500 +++ b/lib/galaxy/tools/actions/__init__.py Mon Nov 03 15:31:38 2008 -0500 @@ -177,7 +177,7 @@ trans.app.model.flush() # Create the job object job = trans.app.model.Job() - job.session_id = trans.get_galaxy_session( create=True ).id + job.session_id = trans.get_galaxy_session().id job.history_id = trans.history.id job.tool_id = tool.id try: diff -r 13cbdd1bbd19 -r 7ae75e6d9d6a lib/galaxy/web/controllers/root.py --- a/lib/galaxy/web/controllers/root.py Mon Nov 03 15:21:03 2008 -0500 +++ b/lib/galaxy/web/controllers/root.py Mon Nov 03 15:31:38 2008 -0500 @@ -54,10 +54,7 @@ NOTE: No longer accepts "id" or "template" options for security reasons. """ - try: - history = trans.get_history() - except: - return self.history_new(trans) + history = trans.get_history() if as_xml: trans.response.set_content_type('text/xml') return trans.fill_template_mako( "root/history_as_xml.mako", history=history ) diff -r 13cbdd1bbd19 -r 7ae75e6d9d6a lib/galaxy/web/controllers/tool_runner.py --- a/lib/galaxy/web/controllers/tool_runner.py Mon Nov 03 15:21:03 2008 -0500 +++ b/lib/galaxy/web/controllers/tool_runner.py Mon Nov 03 15:31:38 2008 -0500 @@ -41,7 +41,6 @@ return "Tool '%s' does not exist, kwd=%s " % (tool_id, kwd) params = util.Params( kwd, sanitize=tool.options.sanitize, tool=tool ) history = trans.get_history() - trans.ensure_valid_galaxy_session() template, vars = tool.handle_input( trans, params.__dict__ ) if len(params) > 0: trans.log_event( "Tool params: %s" % (str(params)), tool_id=tool_id ) diff -r 13cbdd1bbd19 -r 7ae75e6d9d6a lib/galaxy/web/controllers/user.py --- a/lib/galaxy/web/controllers/user.py Mon Nov 03 15:21:03 2008 -0500 +++ b/lib/galaxy/web/controllers/user.py Mon Nov 03 15:31:38 2008 -0500 @@ -82,10 +82,7 @@ elif not user.check_password( password ): password_error = "Invalid password" else: - trans.set_user( user ) - trans.ensure_valid_galaxy_session() - # Associate user with galaxy_session and history - trans.make_associations() + trans.handle_user_login( user ) trans.log_event( "User logged in" ) return trans.show_ok_message( "Now logged in as " + user.email, refresh_frames=['masthead', 'history'] ) return trans.show_form( @@ -97,7 +94,7 @@ def logout( self, trans ): # Since logging an event requires a session, we'll log prior to ending the session trans.log_event( "User logged out" ) - new_galaxy_session = trans.logout_galaxy_session() + trans.handle_user_logout() return trans.show_ok_message( "You are no longer logged in", refresh_frames=['masthead', 'history'] ) @web.expose @@ -118,12 +115,7 @@ user = trans.app.model.User( email=email ) user.set_password_cleartext( password ) user.flush() - trans.set_user( user ) - trans.ensure_valid_galaxy_session() - """ - Associate user with galaxy_session and history - """ - trans.make_associations() + trans.handle_user_login( user ) trans.log_event( "User created a new account" ) trans.log_event( "User logged in" ) #subscribe user to email list diff -r 13cbdd1bbd19 -r 7ae75e6d9d6a lib/galaxy/web/framework/__init__.py --- a/lib/galaxy/web/framework/__init__.py Mon Nov 03 15:21:03 2008 -0500 +++ b/lib/galaxy/web/framework/__init__.py Mon Nov 03 15:31:38 2008 -0500 @@ -109,12 +109,14 @@ self.__history = NOT_SET self.__galaxy_session = NOT_SET base.DefaultWebTransaction.__init__( self, environ ) - self.app.model.context.current.clear() + self.sa_session.clear() self.debug = asbool( self.app.config.get( 'debug', False ) ) # Flag indicating whether we are in workflow building mode (means # that the current history should not be used for parameter values # and such). self.workflow_building_mode = False + # Always have a valid galaxy session + self.__ensure_valid_session() @property def sa_session( self ): """ @@ -140,7 +142,6 @@ except: event.history_id = None event.user = self.user - self.ensure_valid_galaxy_session() event.session_id = self.galaxy_session.id event.flush() def get_cookie( self, name='galaxysession' ): @@ -163,277 +164,208 @@ tstamp = time.localtime ( time.time() + 3600 * 24 * age ) self.response.cookies[name]['expires'] = time.strftime( '%a, %d-%b-%Y %H:%M:%S GMT', tstamp ) self.response.cookies[name]['version'] = version + #@property + #def galaxy_session( self ): + # if not self.__galaxy_session: + # self.__ensure_valid_session() + # return self.__galaxy_session + def __ensure_valid_session( self ): + """ + Ensure that a valid Galaxy session exists and is available as + trans.session (part of initialization) + + Support for universe_session and universe_user cookies has been + removed as of 31 Oct 2008. + """ + sa_session = self.sa_session + # Try to load an existing session + secure_id = self.get_cookie( name='galaxysession' ) + galaxy_session = None + prev_galaxy_session = None + user_for_new_session = None + invalidate_existing_session = False + # Track whether the session has changed so we can avoid calling flush + # in the most common case (session exists and is valid). + galaxy_session_requires_flush = False + if secure_id: + # Decode the cookie value to get the session_key + session_key = self.security.decode_session_key( secure_id ) + # Retrive the galaxy_session id via the unique session_key + galaxy_session = sa_session.query( self.app.model.GalaxySession ).filter_by( session_key=session_key, is_valid=True ).first() + # If remote user is in use it can invalidate the session, so we need to + # to check some things now. + if self.app.config.use_remote_user: + assert "HTTP_REMOTE_USER" in self.environ, \ + "use_remote_user is set but no HTTP_REMOTE_USER variable" + remote_user_email = self.environ[ 'HTTP_REMOTE_USER' ] + if galaxy_session: + # An existing session, make sure correct association exists + if galaxy_session.user is None: + # No user, associate + galaxy_session.user = self.__get_or_create_remote_user( remote_user_email ) + galaxy_session_requires_flush = True + elif galaxy_session.user.email != remote_user_email: + # Session exists but is not associated with the correct + # remote user + invalidate_existing_session = True + user_for_new_session = self.__get_or_create_remote_user( remote_user_email ) + log.warning( "User logged in as '%s' externally, but has a cookie as '%s' invalidating session", + remote_user_email, prev_galaxy_session.user.email ) + else: + if galaxy_session is not None and galaxy_session.user and galaxy_session.user.external: + # Remote user support is not enabled, but there is an existing + # session with an external user, invalidate + invalidate_existing_session = True + log.warning( "User '%s' is an external user with an existing session, invalidating session since external auth is disabled", + galaxy_session.user.email ) + # Do we need to invalidate the session for some reason? + if invalidate_existing_session: + prev_galaxy_session = galaxy_session + prev_galaxy_session.is_valid = False + galaxy_session = None + # No relevant cookies, or couldn't find, or invalid, so create a new session + if galaxy_session is None: + galaxy_session = self.__create_new_session( prev_galaxy_session, user_for_new_session ) + galaxy_session_requires_flush = True + self.galaxy_session = galaxy_session + self.__update_session_cookie() + else: + self.galaxy_session = galaxy_session + # Do we need to flush the session? + if galaxy_session_requires_flush: + objects_to_flush = [ galaxy_session ] + # FIXME: If prev_session is a proper relation this would not + # be needed. + if prev_galaxy_session: + objects_to_flush.append( prev_galaxy_session ) + sa_session.flush( objects_to_flush ) + def __create_new_session( self, prev_galaxy_session=None, user_for_new_session=None ): + """ + Create a new GalaxySession for this request, possibly with a connection + to a previous session (in `prev_galaxy_session`) and an existing user + (in `user_for_new_session`). + + Caller is responsible for flushing the returned session. + """ + session_key = self.security.get_new_session_key() + galaxy_session = self.app.model.GalaxySession( + session_key=session_key, + is_valid=True, + remote_host = self.request.remote_host, + remote_addr = self.request.remote_addr, + referer = self.request.headers.get( 'Referer', None ) ) + # Invalidated an existing sesssion for some reason, keep track + if prev_galaxy_session: + galaxy_session.prev_session_id = prev_galaxy_session.id + # The new session should be immediately associated with a user + if user_for_new_session: + galaxy_session.user = user_for_new_session + return galaxy_session + def __get_or_create_remote_user( self, remote_user_email ): + """ + Return the user in $HTTP_REMOTE_USER and create if necessary + + Caller is responsible for flushing the returned user. + """ + # remote_user middleware ensures HTTP_REMOTE_USER exists + user = self.app.model.User.filter_by( email=remote_user_email ).first() + if user is None: + user = self.app.model.User( email=remote_user_email ) + user.set_password_cleartext( 'external' ) + user.external = True + self.log_event( "Automatically created account '%s'", user.email ) + return user + def __update_session_cookie( self ): + """ + Update the 'galaxysession' cookie to match the current session. + """ + self.set_cookie( name='galaxysession', + value=self.security.encode_session_key( self.galaxy_session.session_key ) ) + + def handle_user_login( self, user ): + """ + Login a new user (possibly newly created) + - create a new session + - associate new session with user + - if old session had a history and it was not associated with a user, associate it with the new session. + """ + prev_galaxy_session = self.galaxy_session + prev_galaxy_session.is_valid = False + self.galaxy_session = self.__create_new_session( prev_galaxy_session, user ) + if prev_galaxy_session.current_history: + history = prev_galaxy_session.current_history + if history.user is None: + self.galaxy_session.add_history( history ) + self.galaxy_session.current_history = history + history.user = user + self.sa_session.flush( [ prev_galaxy_session, self.galaxy_session, history ] ) + else: + self.sa_session.flush( [ prev_galaxy_session, self.galaxy_session ] ) + self.__update_session_cookie() + def handle_user_logout( self ): + """ + Logout the current user: + - invalidate the current session + - create a new session with no user associated + """ + prev_galaxy_session = self.galaxy_session + prev_galaxy_session.is_valid = False + self.galaxy_session = self.__create_new_session( prev_galaxy_session, None ) + self.sa_session.flush( [ prev_galaxy_session, self.galaxy_session ] ) + self.__update_session_cookie() + + def get_galaxy_session( self ): + """ + Return the current galaxy session + """ + return self.galaxy_session + def get_history( self, create=False ): - """Load the current history""" - if self.__history is NOT_SET: - self.__history = None - # See if we have a galaxysession cookie - secure_id = self.get_cookie( name='galaxysession' ) - if secure_id: - session_key = self.security.decode_session_key( secure_id ) - try: - galaxy_session = self.app.model.GalaxySession.filter_by( session_key=session_key ).first() - if galaxy_session and galaxy_session.is_valid and galaxy_session.current_history_id: - history = self.app.model.History.get( galaxy_session.current_history_id ) - if history and not history.deleted: - self.__history = history - except Exception, e: - # This should only occur in development if the cookie is not synced with the db - pass - else: - # See if we have a deprecated universe cookie - # TODO: this should be eliminated some time after October 1, 2008 - # We'll keep it until then because the old universe cookies are valid for 90 days - history_id = self.get_cookie( name='universe' ) - if history_id: - history = self.app.model.History.get( int( history_id ) ) - if history and not history.deleted: - self.__history = history - # Expire the universe cookie since it is deprecated - self.set_cookie( name='universe', value=id, age=0 ) - if self.__history is None: - return self.new_history() - if create is True and self.__history is None: - return self.new_history() - return self.__history + """ + Load the current history. + + NOTE: It looks like create was being ignored for a long time, so this + will currently *always* create a new history. This is wasteful + though, and we should verify that callers are using the create + flag correctly and fix. + """ + history = self.galaxy_session.current_history + if history is None: + history = self.new_history() + return history + def set_history( self, history ): + if history and not history.deleted: + self.galaxy_session.current_history = history + self.sa_session.flush( [ self.galaxy_session ] ) + history = property( get_history, set_history ) def new_history( self ): + """ + Create a new history and associate it with the current session and + its associated user (if set). + """ + # Create new history history = self.app.model.History() - # Make sure we have an id - history.flush() - # Immediately associate the new history with self - self.__history = history - # Make sure we have a valid session to associate with the new history - if self.galaxy_session_is_valid(): - galaxy_session = self.get_galaxy_session() - else: - galaxy_session = self.new_galaxy_session() - # We are associating the last used genome_build with histories, so we will always - # initialize a new history with the first dbkey in util.dbnames which is currently - # ? unspecified (?) + # Associate with session + history.add_galaxy_session( self.galaxy_session ) + # Make it the session's current history + self.galaxy_session.current_history = history + # Associate with user + if self.galaxy_session.user: + history.user = self.galaxy_session.user + # Track genome_build with history history.genome_build = util.dbnames.default_value - if self.user: - history.user_id = self.user.id - galaxy_session.user_id = self.user.id - try: - # See if we have already associated the history with the session - association = self.app.model.GalaxySessionToHistoryAssociation.filter_by( session_id=galaxy_session.id, history_id=history.id ).first() - except: - association = None - history.add_galaxy_session( galaxy_session, association=association ) - history.flush() - galaxy_session.current_history_id = history.id - galaxy_session.flush() - self.__history = history - return self.__history - def set_history( self, history ): - if history and not history.deleted and self.galaxy_session_is_valid(): - galaxy_session = self.get_galaxy_session() - galaxy_session.current_history_id = history.id - galaxy_session.flush() - self.__history = history - history = property( get_history, set_history ) + # Save + self.sa_session.flush( [ self.galaxy_session, history ] ) + return history + def get_user( self ): """Return the current user if logged in or None.""" - if self.__user is NOT_SET: - self.__user = None - user = self.get_cookie_user() - if self.app.config.use_remote_user: - remote_user = self.get_remote_user() - if user is not None and user.email != remote_user.email: - # The user has a cookie for a different user than the one - # they've authed as. - log.warning( "User logged in as '%s' externally, but has a cookie as '%s'," - % ( remote_user.email, user.email ) + " invalidating session" ) - self.logout_galaxy_session() - if user is None or ( user is not None and user.email != remote_user.email ): - # The user has no cookie, was not logged in, or the above. - self.set_user( remote_user ) - self.make_associations() - self.__user = remote_user - else: - if user is not None and user.external: - # use_remote_user is off, but the user still has a cookie - # from when it was on. Force the user to get a new - # unauthenticated session. - log.warning( "User '%s' is an external user with an existing " % user.email - + "session, invalidating session since external auth is disabled" ) - self.logout_galaxy_session() - else: - self.__user = user - return self.__user - def get_remote_user( self ): - """Return the user in $HTTP_REMOTE_USER and create if necessary""" - # remote_user middleware ensures HTTP_REMOTE_USER exists - try: - user = self.app.model.User.filter_by( email=self.environ[ 'HTTP_REMOTE_USER' ] ).first() - except: - user = self.app.model.User( email=self.environ[ 'HTTP_REMOTE_USER' ] ) - user.set_password_cleartext( 'external' ) - user.external = True - user.flush() - self.log_event( "Automatically created account '%s'" % user.email ) - return user - def get_cookie_user( self ): - """Return the user in the galaxysession cookie""" - __user = None - # See if we have a galaxysession cookie - secure_id = self.get_cookie( name='galaxysession' ) - if secure_id: - session_key = self.security.decode_session_key( secure_id ) - try: - galaxy_session = self.app.model.GalaxySession.filter_by( session_key=session_key ).first() - if galaxy_session and galaxy_session.is_valid and galaxy_session.user_id: - user = self.app.model.User.get( galaxy_session.user_id ) - if user: - __user = user - except: - # This should only occur in development if the cookie is not synced with the db - pass - else: - # See if we have a deprecated universe_user cookie - # TODO: this should be eliminated some time after October 1, 2008 - # We'll keep it until then because the old universe cookies are valid for 90 days - user_id = self.get_cookie( name='universe_user' ) - if user_id: - user = self.app.model.User.get( int( user_id ) ) - if user: - __user = user - # Expire the universe_user cookie since it is deprecated - self.set_cookie( name='universe_user', value='', age=0 ) - return __user + return self.galaxy_session.user def set_user( self, user ): - """Set the current user if logged in.""" - if user is not None and self.galaxy_session_is_valid(): - galaxy_session = self.get_galaxy_session() - if galaxy_session.user_id != user.id: - galaxy_session.user_id = user.id - galaxy_session.flush() - self.__user = user + """Set the current user.""" + self.galaxy_session.user = user + self.sa_session.flush( [ self.galaxy_session ] ) user = property( get_user, set_user ) - def get_galaxy_session( self, create=False ): - """Return the current user's GalaxySession""" - if self.__galaxy_session is NOT_SET: - self.__galaxy_session = None - # See if we have a galaxysession cookie - secure_id = self.get_cookie( name='galaxysession' ) - if secure_id: - # Decode the cookie value to get the session_key - session_key = self.security.decode_session_key( secure_id ) - try: - # Retrive the galaxy_session id via the unique session_key - galaxy_session = self.app.model.GalaxySession.filter_by( session_key=session_key ).first() - if galaxy_session and galaxy_session.is_valid: - self.__galaxy_session = galaxy_session - except: - # This should only occur in development if the cookie is not synced with the db - pass - else: - # See if we have a deprecated universe_session cookie - # TODO: this should be eliminated some time after October 1, 2008 - # We'll keep it until then because the old universe cookies are valid for 90 days - session_id = self.get_cookie( name='universe_session' ) - if session_id: - galaxy_session = self.app.model.GalaxySession.get( int( session_id ) ) - # NOTE: We can't test for is_valid here since the old session records did not include this flag - if galaxy_session: - # Set the new galaxysession cookie value, old session records did not have a session_key or is_valid flag - session_key = self.security.get_new_session_key() - galaxy_session.session_key = session_key - galaxy_session.is_valid = True - galaxy_session.flush() - secure_id = self.security.encode_session_key( session_key ) - self.set_cookie( name='galaxysession', value=secure_id ) - # Expire the universe_user cookie since it is deprecated - self.set_cookie( name='universe_session', value='', age=0 ) - self.__galaxy_session = galaxy_session - if create is True and self.__galaxy_session is None: - return self.new_galaxy_session() - return self.__galaxy_session - def new_galaxy_session( self, prev_session_id=None ): - """Create a new secure galaxy_session""" - session_key = self.security.get_new_session_key() - galaxy_session = self.app.model.GalaxySession( session_key=session_key, is_valid=True, prev_session_id=prev_session_id ) - # Make sure we have an id - galaxy_session.flush() - # Immediately associate the new session with self - self.__galaxy_session = galaxy_session - if prev_session_id is not None: - # User logged out, so we need to create a new history for this session - self.history = self.new_history() - galaxy_session.current_history_id = self.history.id - elif self.user is not None: - galaxy_session.user_id = self.user.id - # Set this session's current_history_id to the user's last updated history - h = self.app.model.History - ht = h.table - where = ( ht.c.user_id==self.user.id ) & ( ht.c.deleted=='f' ) - history = h.query().filter( where ).order_by( desc( ht.c.update_time ) ).first() - if history: - self.history = history - galaxy_session.current_history_id = self.history.id - elif self.history: - galaxy_session.current_history_id = self.history.id - galaxy_session.remote_host = self.request.remote_host - galaxy_session.remote_addr = self.request.remote_addr - try: - galaxy_session.referer = self.request.headers['Referer'] - except: - galaxy_session.referer = None - if self.history is not None: - # See if we have already associated the session with the history - try: - association = self.app.model.GalaxySessionToHistoryAssociation.filter_by( session_id=galaxy_session.id, history_id=self.history.id ).first() - except: - association = None - galaxy_session.add_history( self.history, association=association ) - galaxy_session.flush() - # Set the cookie value to the encrypted session_key - self.set_cookie( name='galaxysession', value=self.security.encode_session_key( session_key ) ) - self.__galaxy_session = galaxy_session - return self.__galaxy_session - def set_galaxy_session( self, galaxy_session ): - """Set the current galaxy_session""" - self.__galaxy_session = galaxy_session - galaxy_session = property( get_galaxy_session, set_galaxy_session ) - def galaxy_session_is_valid( self ): - try: - return self.galaxy_session.is_valid - except: - return False - def ensure_valid_galaxy_session( self ): - """Make sure we have a valid galaxy session, create a new one if necessary.""" - if not self.galaxy_session_is_valid(): - galaxy_session = self.new_galaxy_session() - def logout_galaxy_session( self ): - """ - Logout the current user by setting user to None and galaxy_session.is_valid to False - in the db. A new galaxy_session is automatically created with prev_session_id is set - to save a reference to the current one as a way of chaining them together - """ - if self.galaxy_session_is_valid(): - galaxy_session = self.get_galaxy_session() - old_session_id = galaxy_session.id - galaxy_session.is_valid = False - galaxy_session.flush() - self.set_user( None ) - return self.new_galaxy_session( prev_session_id=old_session_id ) - else: - error( "Attempted to logout an invalid galaxy_session" ) - def make_associations( self ): - history = self.get_history() - user = self.get_user() - if self.galaxy_session_is_valid(): - galaxy_session = self.get_galaxy_session() - if galaxy_session.user_id is None and user is not None: - galaxy_session.user_id = user.id - if history is not None: - galaxy_session.current_history_id = history.id - galaxy_session.flush() - self.__galaxy_session = galaxy_session - if history is not None and user is not None: - history.user_id = user.id - history.flush() - self.__history = history def get_toolbox(self): """Returns the application toolbox""" diff -r 13cbdd1bbd19 -r 7ae75e6d9d6a lib/galaxy/web/security/__init__.py --- a/lib/galaxy/web/security/__init__.py Mon Nov 03 15:21:03 2008 -0500 +++ b/lib/galaxy/web/security/__init__.py Mon Nov 03 15:31:38 2008 -0500 @@ -1,9 +1,35 @@ +import os, os.path, logging + import pkg_resources pkg_resources.require( "pycrypto" ) from Crypto.Cipher import Blowfish from Crypto.Util.randpool import RandomPool from Crypto.Util import number + +log = logging.getLogger( __name__ ) + +if os.path.exists( "/dev/urandom" ): + log.debug("###using /dev/urandom....") + # We have urandom, use it as the source of random data + random_fd = os.open( "/dev/urandom", os.O_RDONLY ) + def get_random_bytes( nbytes ): + value = os.read( random_fd, nbytes ) + # Normally we should get as much as we need + if len( value ) == nbytes: + return value + # If we don't, keep reading (this is slow and should never happen) + while len( value ) < nbytes: + value += os.read( random_fd, nbytes - len( value ) ) + return value +else: + def get_random_bytes( nbytes ): + nbits = nbytes * 8 + random_pool = RandomPool( 1064 ) + while random_pool.entropy < nbits: + random_pool.add_event() + random_pool.stir() + return( str( number.getRandomNumber( nbits, random_pool.get_bytes ) ) ) class SecurityHelper( object ): # TODO: checking if histories/datasets are owned by the current user) will be moved here. @@ -30,11 +56,5 @@ return self.id_cipher.decrypt( session_key.decode( 'hex' ) ).lstrip( "!" ) def get_new_session_key( self ): # Generate a unique, high entropy 128 bit random number - random_pool = RandomPool( 1064 ) - while random_pool.entropy < 128: - random_pool.add_event() - random_pool.stir() - rn = number.getRandomNumber( 128, random_pool.get_bytes ) - # session_key must be a string - return str( rn ) + return get_random_bytes( 16 ) \ No newline at end of file diff -r 13cbdd1bbd19 -r 7ae75e6d9d6a templates/root/history.mako --- a/templates/root/history.mako Mon Nov 03 15:21:03 2008 -0500 +++ b/templates/root/history.mako Mon Nov 03 15:31:38 2008 -0500 @@ -231,6 +231,7 @@ <body class="historyPage"> + <div id="top-links" class="historyLinks"> <a href="${h.url_for('history', show_deleted=show_deleted)}">refresh</a> %if show_deleted:

1 0

[hg] galaxy 1599: A fix for the metadata reverting issue seen wi...
by Greg Von Kuster 04 Nov '08

04 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/13cbdd1bbd19 changeset: 1599:13cbdd1bbd19 user: Dan Blankenberg <dan(a)bx.psu.edu> date: Mon Nov 03 15:21:03 2008 -0500 description: A fix for the metadata reverting issue seen with PBS job runner. 1 file(s) affected in this change: lib/galaxy/jobs/__init__.py diffs (18 lines): diff -r ca640ce1abcf -r 13cbdd1bbd19 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py Mon Nov 03 15:15:22 2008 -0500 +++ b/lib/galaxy/jobs/__init__.py Mon Nov 03 15:21:03 2008 -0500 @@ -274,6 +274,7 @@ Prepare the job to run by creating the working directory and the config files. """ + mapping.context.current.clear() #this prevents the metadata reverting that has been seen in conjunction with the PBS job runner # Create the working directory self.working_directory = \ os.path.join( self.app.config.job_working_directory, str( self.job_id ) ) @@ -643,4 +644,4 @@ def put( self, *args ): return def shutdown( self ): - return \ No newline at end of file + return

1 0

[hg] galaxy 1598: Fix a bug: errored jobs weren't actually being...
by Nate Coraor 03 Nov '08

03 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/ca640ce1abcf changeset: 1598:ca640ce1abcf user: Nate Coraor <nate(a)bx.psu.edu> date: Mon Nov 03 15:15:22 2008 -0500 description: Fix a bug: errored jobs weren't actually being deleted. 1 file(s) affected in this change: lib/galaxy/web/controllers/root.py diffs (19 lines): diff -r 21f98b638868 -r ca640ce1abcf lib/galaxy/web/controllers/root.py --- a/lib/galaxy/web/controllers/root.py Mon Nov 03 15:07:22 2008 -0500 +++ b/lib/galaxy/web/controllers/root.py Mon Nov 03 15:15:22 2008 -0500 @@ -281,11 +281,10 @@ if data.parent_id is None and len( data.creating_job_associations ) > 0: # Mark associated job for deletion job = data.creating_job_associations[0].job - if job.state not in [ model.Job.states.QUEUED, model.Job.states.RUNNING, model.Job.states.NEW ]: - return - # Are *all* of the job's other output datasets deleted? - if job.check_if_output_datasets_deleted(): - job.mark_deleted() + if job.state in [ model.Job.states.QUEUED, model.Job.states.RUNNING, model.Job.states.NEW ]: + # Are *all* of the job's other output datasets deleted? + if job.check_if_output_datasets_deleted(): + job.mark_deleted() self.app.model.flush() @web.expose

1 0

[hg] galaxy 1596: A fix for the metadata reverting issue seen wi...
by Nate Coraor 03 Nov '08

03 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/36a297157114 changeset: 1596:36a297157114 user: Dan Blankenberg <dan(a)bx.psu.edu> date: Mon Nov 03 14:48:25 2008 -0500 description: A fix for the metadata reverting issue seen with PBS job runner. 1 file(s) affected in this change: lib/galaxy/jobs/__init__.py diffs (18 lines): diff -r 56bc2f789894 -r 36a297157114 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py Sat Nov 01 15:30:13 2008 -0400 +++ b/lib/galaxy/jobs/__init__.py Mon Nov 03 14:48:25 2008 -0500 @@ -274,6 +274,7 @@ Prepare the job to run by creating the working directory and the config files. """ + mapping.context.current.clear() #this prevents the metadata reverting that has been seen in conjunction with the PBS job runner # Create the working directory self.working_directory = \ os.path.join( self.app.config.job_working_directory, str( self.job_id ) ) @@ -643,4 +644,4 @@ def put( self, *args ): return def shutdown( self ): - return \ No newline at end of file + return

1 0