2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ea8d92515d42/ Changeset: ea8d92515d42 Branch: next-stable User: carlfeberhard Date: 2013-05-21 21:19:33 Summary: History: add simple profiling method Affected #: 3 files diff -r 6b59809541ce4766efd1d044fbdee86a954746c4 -r ea8d92515d42321592e77842387290dcc1190a2b lib/galaxy/util/debugging.py --- a/lib/galaxy/util/debugging.py +++ b/lib/galaxy/util/debugging.py @@ -1,6 +1,7 @@ import inspect import pprint +import time import logging log = logging.getLogger( __name__ ) @@ -27,3 +28,39 @@ stack_list.append( line_format.format( **caller_data ) ) return '\n'.join( stack_list ) + + +class SimpleProfiler( object ): + """ + Simple profiler that captures the duration between calls to `report` + and stores the results in a list. + """ + REPORT_FORMAT = '%20f: %s' + + def __init__( self, log=None ): + self.log = log + self.start_time = 0 + self.reports = [] + + def start( self, msg=None ): + msg = msg or 'Start' + self.start_time = time.time() + report = self.REPORT_FORMAT %( self.start_time, msg ) + self.reports.append( report ) + if self.log: + self.log( report ) + + def report( self, msg ): + if not self.start_time: + self.start() + duration = time.time() - self.start_time + report = self.REPORT_FORMAT %( duration, msg ) + self.reports.append( report ) + if self.log: + self.log( report ) + + def get_reports( self ): + return self.reports + + def __str__( self ): + return '\n'.join( self.reports ) diff -r 6b59809541ce4766efd1d044fbdee86a954746c4 -r ea8d92515d42321592e77842387290dcc1190a2b lib/galaxy/webapps/galaxy/controllers/root.py --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -2,7 +2,6 @@ Contains the main interface in the Universe class """ import cgi -import logging import os import urllib @@ -15,6 +14,9 @@ from galaxy.web.base.controller import BaseUIController, UsesHistoryDatasetAssociationMixin, UsesHistoryMixin from galaxy.util.json import to_json_string +from galaxy.util.debugging import SimpleProfiler + +import logging log = logging.getLogger( __name__ ) class RootController( BaseUIController, UsesHistoryMixin, UsesHistoryDatasetAssociationMixin, UsesAnnotations ): @@ -176,6 +178,73 @@ history_json = to_json_string( history_dictionary ), hda_json = to_json_string( hda_dictionaries ), show_deleted=show_deleted, show_hidden=show_hidden, hda_id=hda_id, log=log, message=message, status=status ) + @web.expose + def profile_history( self, trans, as_xml=False, show_deleted=None, show_hidden=None, hda_id=None, **kwd ): + """ + Same as above but adds SimpleProfiler to get some + profiling times for the operations done. + """ + if as_xml: + return self.history_as_xml( trans, + show_deleted=string_as_bool( show_deleted ), show_hidden=string_as_bool( show_hidden ) ) + + # get all datasets server-side, client-side will get flags and render appropriately + show_deleted = string_as_bool_or_none( show_deleted ) + show_purged = show_deleted + show_hidden = string_as_bool_or_none( show_hidden ) + params = Params( kwd ) + message = params.get( 'message', '' ) + #TODO: ugh... + message = message if message != 'None' else '' + status = params.get( 'status', 'done' ) + + if trans.app.config.require_login and not trans.user: + return trans.fill_template( '/no_access.mako', message = 'Please log in to access Galaxy histories.' ) + + def err_msg( where=None ): + where = where if where else 'getting the history data from the server' + err_msg = ( 'An error occurred %s. ' + + 'Please contact a Galaxy administrator if the problem persists.' ) %( where ) + return err_msg, 'error' + + profiler = SimpleProfiler() + profiler.start() + + history_dictionary = {} + hda_dictionaries = [] + try: + history = trans.get_history( create=True ) + profiler.report( 'trans.get_history' ) + hdas = self.get_history_datasets( trans, history, + show_deleted=True, show_hidden=True, show_purged=True ) + profiler.report( 'get_history_datasets' ) + + for hda in hdas: + try: + hda_dictionaries.append( self.get_hda_dict( trans, hda ) ) + profiler.report( '\t hda -> dictionary (%s)' %( hda.name ) ) + + except Exception, exc: + # don't fail entire list if hda err's, record and move on + log.error( 'Error bootstrapping hda %d: %s', hda.id, str( exc ), exc_info=True ) + hda_dictionaries.append( self.get_hda_dict_with_error( trans, hda, str( exc ) ) ) + profiler.report( 'hdas -> dictionaries' ) + + # re-use the hdas above to get the history data... + history_dictionary = self.get_history_dict( trans, history, hda_dictionaries=hda_dictionaries ) + profiler.report( 'history -> dictionary' ) + + except Exception, exc: + user_id = str( trans.user.id ) if trans.user else '(anonymous)' + log.error( 'Error bootstrapping history for user %s: %s', user_id, str( exc ), exc_info=True ) + message, status = err_msg() + history_dictionary[ 'error' ] = message + + return trans.stream_template_mako( "root/history.mako", + history_json = to_json_string( history_dictionary ), hda_json = to_json_string( hda_dictionaries ), + show_deleted=show_deleted, show_hidden=show_hidden, hda_id=hda_id, log=log, message=message, status=status, + profiling=profiler.get_reports() ) + ## ---- Dataset display / editing ---------------------------------------- @web.expose def display( self, trans, id=None, hid=None, tofile=None, toext=".txt", **kwd ): diff -r 6b59809541ce4766efd1d044fbdee86a954746c4 -r ea8d92515d42321592e77842387290dcc1190a2b templates/webapps/galaxy/root/history.mako --- a/templates/webapps/galaxy/root/history.mako +++ b/templates/webapps/galaxy/root/history.mako @@ -213,6 +213,10 @@ galaxyPageSetUp(); // ostensibly, this is the App + %if 'profiling' in self.context.kwargs: + Galaxy.profiling = ${h.to_json_string( profiling )}.join( '\n' ); + %endif + //NOTE: for debugging on non-local instances (main/test) // 1. load history panel in own tab // 2. from console: new PersistantStorage( '__history_panel' ).set( 'debugging', true ) https://bitbucket.org/galaxy/galaxy-central/commits/342617fcad86/ Changeset: 342617fcad86 User: carlfeberhard Date: 2013-05-21 21:20:03 Summary: merge next-stable Affected #: 3 files diff -r cbc1808b261b662f8dc9682b31f86c9b6c429bbd -r 342617fcad86f6948dabaf136fe0c04a83212aad lib/galaxy/util/debugging.py --- a/lib/galaxy/util/debugging.py +++ b/lib/galaxy/util/debugging.py @@ -1,6 +1,7 @@ import inspect import pprint +import time import logging log = logging.getLogger( __name__ ) @@ -27,3 +28,39 @@ stack_list.append( line_format.format( **caller_data ) ) return '\n'.join( stack_list ) + + +class SimpleProfiler( object ): + """ + Simple profiler that captures the duration between calls to `report` + and stores the results in a list. + """ + REPORT_FORMAT = '%20f: %s' + + def __init__( self, log=None ): + self.log = log + self.start_time = 0 + self.reports = [] + + def start( self, msg=None ): + msg = msg or 'Start' + self.start_time = time.time() + report = self.REPORT_FORMAT %( self.start_time, msg ) + self.reports.append( report ) + if self.log: + self.log( report ) + + def report( self, msg ): + if not self.start_time: + self.start() + duration = time.time() - self.start_time + report = self.REPORT_FORMAT %( duration, msg ) + self.reports.append( report ) + if self.log: + self.log( report ) + + def get_reports( self ): + return self.reports + + def __str__( self ): + return '\n'.join( self.reports ) diff -r cbc1808b261b662f8dc9682b31f86c9b6c429bbd -r 342617fcad86f6948dabaf136fe0c04a83212aad lib/galaxy/webapps/galaxy/controllers/root.py --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -2,7 +2,6 @@ Contains the main interface in the Universe class """ import cgi -import logging import os import urllib @@ -15,6 +14,9 @@ from galaxy.web.base.controller import BaseUIController, UsesHistoryDatasetAssociationMixin, UsesHistoryMixin from galaxy.util.json import to_json_string +from galaxy.util.debugging import SimpleProfiler + +import logging log = logging.getLogger( __name__ ) class RootController( BaseUIController, UsesHistoryMixin, UsesHistoryDatasetAssociationMixin, UsesAnnotations ): @@ -176,6 +178,73 @@ history_json = to_json_string( history_dictionary ), hda_json = to_json_string( hda_dictionaries ), show_deleted=show_deleted, show_hidden=show_hidden, hda_id=hda_id, log=log, message=message, status=status ) + @web.expose + def profile_history( self, trans, as_xml=False, show_deleted=None, show_hidden=None, hda_id=None, **kwd ): + """ + Same as above but adds SimpleProfiler to get some + profiling times for the operations done. + """ + if as_xml: + return self.history_as_xml( trans, + show_deleted=string_as_bool( show_deleted ), show_hidden=string_as_bool( show_hidden ) ) + + # get all datasets server-side, client-side will get flags and render appropriately + show_deleted = string_as_bool_or_none( show_deleted ) + show_purged = show_deleted + show_hidden = string_as_bool_or_none( show_hidden ) + params = Params( kwd ) + message = params.get( 'message', '' ) + #TODO: ugh... + message = message if message != 'None' else '' + status = params.get( 'status', 'done' ) + + if trans.app.config.require_login and not trans.user: + return trans.fill_template( '/no_access.mako', message = 'Please log in to access Galaxy histories.' ) + + def err_msg( where=None ): + where = where if where else 'getting the history data from the server' + err_msg = ( 'An error occurred %s. ' + + 'Please contact a Galaxy administrator if the problem persists.' ) %( where ) + return err_msg, 'error' + + profiler = SimpleProfiler() + profiler.start() + + history_dictionary = {} + hda_dictionaries = [] + try: + history = trans.get_history( create=True ) + profiler.report( 'trans.get_history' ) + hdas = self.get_history_datasets( trans, history, + show_deleted=True, show_hidden=True, show_purged=True ) + profiler.report( 'get_history_datasets' ) + + for hda in hdas: + try: + hda_dictionaries.append( self.get_hda_dict( trans, hda ) ) + profiler.report( '\t hda -> dictionary (%s)' %( hda.name ) ) + + except Exception, exc: + # don't fail entire list if hda err's, record and move on + log.error( 'Error bootstrapping hda %d: %s', hda.id, str( exc ), exc_info=True ) + hda_dictionaries.append( self.get_hda_dict_with_error( trans, hda, str( exc ) ) ) + profiler.report( 'hdas -> dictionaries' ) + + # re-use the hdas above to get the history data... + history_dictionary = self.get_history_dict( trans, history, hda_dictionaries=hda_dictionaries ) + profiler.report( 'history -> dictionary' ) + + except Exception, exc: + user_id = str( trans.user.id ) if trans.user else '(anonymous)' + log.error( 'Error bootstrapping history for user %s: %s', user_id, str( exc ), exc_info=True ) + message, status = err_msg() + history_dictionary[ 'error' ] = message + + return trans.stream_template_mako( "root/history.mako", + history_json = to_json_string( history_dictionary ), hda_json = to_json_string( hda_dictionaries ), + show_deleted=show_deleted, show_hidden=show_hidden, hda_id=hda_id, log=log, message=message, status=status, + profiling=profiler.get_reports() ) + ## ---- Dataset display / editing ---------------------------------------- @web.expose def display( self, trans, id=None, hid=None, tofile=None, toext=".txt", **kwd ): diff -r cbc1808b261b662f8dc9682b31f86c9b6c429bbd -r 342617fcad86f6948dabaf136fe0c04a83212aad templates/webapps/galaxy/root/history.mako --- a/templates/webapps/galaxy/root/history.mako +++ b/templates/webapps/galaxy/root/history.mako @@ -213,6 +213,10 @@ galaxyPageSetUp(); // ostensibly, this is the App + %if 'profiling' in self.context.kwargs: + Galaxy.profiling = ${h.to_json_string( profiling )}.join( '\n' ); + %endif + //NOTE: for debugging on non-local instances (main/test) // 1. load history panel in own tab // 2. from console: new PersistantStorage( '__history_panel' ).set( 'debugging', true ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.