details: http://www.bx.psu.edu/hg/galaxy/rev/d1bc3b0535d0 changeset: 3285:d1bc3b0535d0 user: jeremy goecks <jeremy.goecks@emory.edu> date: Wed Jan 27 15:50:21 2010 -0500 description: Merge. diffstat: dist-eggs.ini | 24 +++--- lib/galaxy/config.py | 1 + lib/galaxy/security/__init__.py | 6 +- lib/galaxy/util/heartbeat.py | 86 +++++++++++++++++++++++++++- lib/galaxy/visualization/tracks/data/bam.py | 29 +++++++- lib/galaxy/web/controllers/library_common.py | 14 +++- lib/galaxy/web/framework/base.py | 4 + 7 files changed, 141 insertions(+), 23 deletions(-) diffs (280 lines): diff -r acf985b23928 -r d1bc3b0535d0 dist-eggs.ini --- a/dist-eggs.ini Wed Jan 27 15:49:50 2010 -0500 +++ b/dist-eggs.ini Wed Jan 27 15:50:21 2010 -0500 @@ -7,18 +7,18 @@ ; [hosts] -py2.4-linux-i686-ucs2 = basie.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs2/bin/python2.4 -py2.4-linux-i686-ucs4 = basie.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs4/bin/python2.4 -py2.5-linux-i686-ucs2 = basie.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs2/bin/python2.5 -py2.5-linux-i686-ucs4 = basie.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs4/bin/python2.5 -py2.6-linux-i686-ucs2 = basie.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs2/bin/python2.6 -py2.6-linux-i686-ucs4 = basie.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs4/bin/python2.6 -py2.4-linux-x86_64-ucs2 = scofield.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs2/bin/python2.4 -py2.4-linux-x86_64-ucs4 = scofield.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.4 -py2.5-linux-x86_64-ucs2 = scofield.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs2/bin/python2.5 -py2.5-linux-x86_64-ucs4 = scofield.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.5 -py2.6-linux-x86_64-ucs2 = scofield.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs2/bin/python2.6 -py2.6-linux-x86_64-ucs4 = scofield.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.6 +py2.4-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs2/bin/python2.4 +py2.4-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs4/bin/python2.4 +py2.5-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs2/bin/python2.5 +py2.5-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs4/bin/python2.5 +py2.6-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs2/bin/python2.6 +py2.6-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs4/bin/python2.6 +py2.4-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs2/bin/python2.4 +py2.4-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.4 +py2.5-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs2/bin/python2.5 +py2.5-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.5 +py2.6-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs2/bin/python2.6 +py2.6-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.6 py2.4-macosx-10.3-fat-ucs2 = medeski.bx.psu.edu /usr/local/bin/python2.4 py2.5-macosx-10.3-fat-ucs2 = medeski.bx.psu.edu /usr/local/bin/python2.5 py2.6-macosx-10.3-fat-ucs2 = medeski.bx.psu.edu /usr/local/bin/python2.6 diff -r acf985b23928 -r d1bc3b0535d0 lib/galaxy/config.py --- a/lib/galaxy/config.py Wed Jan 27 15:49:50 2010 -0500 +++ b/lib/galaxy/config.py Wed Jan 27 15:50:21 2010 -0500 @@ -97,6 +97,7 @@ raise ConfigurationError( "user_library_import_dir specified in config (%s) does not exist" % self.user_library_import_dir ) self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False ) # Configuration options for taking advantage of nginx features + self.apache_xsendfile = kwargs.get( 'apache_xsendfile', False ) self.nginx_x_accel_redirect_base = kwargs.get( 'nginx_x_accel_redirect_base', False ) self.nginx_upload_store = kwargs.get( 'nginx_upload_store', False ) self.nginx_upload_path = kwargs.get( 'nginx_upload_path', False ) diff -r acf985b23928 -r d1bc3b0535d0 lib/galaxy/security/__init__.py --- a/lib/galaxy/security/__init__.py Wed Jan 27 15:49:50 2010 -0500 +++ b/lib/galaxy/security/__init__.py Wed Jan 27 15:50:21 2010 -0500 @@ -138,13 +138,15 @@ intermed.sort() return map( operator.getitem, intermed, ( -1, ) * len( intermed ) ) roles = set() - # If a library has roles associated with the LIBRARY_ACCESS permission, we need to start with them. + # If item has roles associated with the access permission, we need to start with them. access_roles = item.get_access_roles( trans ) for role in access_roles: roles.add( role ) # Each role potentially has users. We need to find all roles that each of those users have. for ura in role.users: - roles.add( ura.role ) + user = ura.user + for ura2 in user.roles: + roles.add( ura2.role ) # Each role also potentially has groups which, in turn, have members ( users ). We need to # find all roles that each group's members have. for gra in role.groups: diff -r acf985b23928 -r d1bc3b0535d0 lib/galaxy/util/heartbeat.py --- a/lib/galaxy/util/heartbeat.py Wed Jan 27 15:49:50 2010 -0500 +++ b/lib/galaxy/util/heartbeat.py Wed Jan 27 15:50:21 2010 -0500 @@ -54,6 +54,9 @@ self.period = period self.fname = fname self.file = None + self.fname_nonsleeping = fname + ".nonsleeping" + self.file_nonsleeping = None + self.nonsleeping_heartbeats = { } # Save process id self.pid = os.getpid() # Event to wait on when sleeping, allows us to interrupt for shutdown @@ -62,6 +65,9 @@ self.file = open( self.fname, "a" ) print >> self.file, "Heartbeat for pid %d thread started at %s" % ( self.pid, time.asctime() ) print >> self.file + self.file_nonsleeping = open ( self.fname_nonsleeping, "a" ) + print >> self.file_nonsleeping, "Non-Sleeping-threads for pid %d thread started at %s" % ( self.pid, time.asctime() ) + print >> self.file_nonsleeping try: while not self.should_stop: # Print separator with timestamp @@ -81,6 +87,7 @@ print >> self.file, "End dump" print >> self.file self.file.flush() + self.print_nonsleeping(threads) # Sleep for a bit self.wait_event.wait( self.period ) finally: @@ -88,9 +95,86 @@ print >> self.file # Cleanup self.file.close() + self.file_nonsleeping.close() def shutdown( self ): self.should_stop = True self.wait_event.set() self.join() - \ No newline at end of file + + def thread_is_sleeping ( self, last_stack_frame ): + """ + Returns True if the given stack-frame represents a known + sleeper function (at least in python 2.5) + """ + _filename = last_stack_frame[0] + _line = last_stack_frame[1] + _funcname = last_stack_frame[2] + _text = last_stack_frame[3] + ### Ugly hack to tell if a thread is supposedly sleeping or not + ### These are the most common sleeping functions I've found. + ### Is there a better way? (python interpreter internals?) + ### Tested only with python 2.5 + if _funcname=="wait" and _text=="waiter.acquire()": + return True + if _funcname=="wait" and _text=="_sleep(delay)": + return True + if _funcname=="accept" and _text[-14:]=="_sock.accept()": + return True + #Ugly hack: always skip the heartbeat thread + #TODO: get the current thread-id in python + # skip heartbeat thread by thread-id, not by filename + if _filename.find("/lib/galaxy/util/heartbeat.py")!=-1: + return True + ## By default, assume the thread is not sleeping + return False + + def get_interesting_stack_frame ( self, stack_frames ): + """ + Scans a given backtrace stack frames, returns a single + quadraple of [filename, line, function-name, text] of + the single, deepest, most interesting frame. + Interesting being: + inside the galaxy source code ("/lib/galaxy"), + prefreably not an egg. + """ + for _filename, _line, _funcname, _text in reversed(stack_frames): + idx = _filename.find("/lib/galaxy/") + if idx!=-1: + relative_filename = _filename[idx:] + return ( relative_filename, _line, _funcname, _text ) + # no "/lib/galaxy" code found, return the innermost frame + return stack_frames[-1] + + def print_nonsleeping( self, threads_object_dict ): + print >> self.file_nonsleeping, "Non-Sleeping threads at %s:" % time.asctime() + print >> self.file_nonsleeping + all_threads_are_sleeping = True + threads = get_current_thread_object_dict() + for thread_id, frame in threadframe.dict().iteritems(): + if thread_id in threads: + object = repr( threads[thread_id] ) + else: + object = "<No Thread object>" + tb = traceback.extract_stack(frame) + if self.thread_is_sleeping(tb[-1]): + if thread_id in self.nonsleeping_heartbeats: + del self.nonsleeping_heartbeats[thread_id] + continue + + # Count non-sleeping thread heartbeats + if thread_id in self.nonsleeping_heartbeats: + self.nonsleeping_heartbeats[thread_id] += 1 + else: + self.nonsleeping_heartbeats[thread_id]=1 + + good_frame = self.get_interesting_stack_frame(tb) + print >> self.file_nonsleeping, "Thread %s\t%s\tnon-sleeping for %d heartbeat(s)\n File %s:%d\n Function \"%s\"\n %s" % \ + ( thread_id, object, self.nonsleeping_heartbeats[thread_id], good_frame[0], good_frame[1], good_frame[2], good_frame[3] ) + all_threads_are_sleeping = False + + if all_threads_are_sleeping: + print >> self.file_nonsleeping, "All threads are sleeping." + print >> self.file_nonsleeping + self.file_nonsleeping.flush() + diff -r acf985b23928 -r d1bc3b0535d0 lib/galaxy/visualization/tracks/data/bam.py --- a/lib/galaxy/visualization/tracks/data/bam.py Wed Jan 27 15:49:50 2010 -0500 +++ b/lib/galaxy/visualization/tracks/data/bam.py Wed Jan 27 15:50:21 2010 -0500 @@ -1,23 +1,42 @@ +""" +Visualization data provider for BAM format. +""" + import pkg_resources; pkg_resources.require( "pysam" ) + from pysam import csamtools from math import floor, ceil, log import logging class BamDataProvider( object ): + """ + Provides access to intervals from a sorted indexed BAM file. + """ def __init__( self, index, original_dataset ): - self.log = logging.getLogger(__name__) + # self.log = logging.getLogger(__name__) self.index = index self.original_dataset = original_dataset def get_data( self, chrom, start, end, **kwargs ): + """ + Fetch intervals in the region + """ start, end = int(start), int(end) - bamfile = csamtools.Samfile(filename=self.original_dataset.file_name, mode='rb', index_filename=self.index.file_name) - - data = bamfile.fetch(start=start, end=end, reference=chrom) + # Attempt to open the BAM file with index + bamfile = csamtools.Samfile( filename=self.original_dataset.file_name, mode='rb', index_filename=self.index.file_name ) + try: + data = bamfile.fetch(start=start, end=end, reference=chrom) + except ValueError, e: + # Some BAM files do not prefix chromosome names with chr, try + # without + if chrom.startswith( 'chr' ): + data = bamfile.fetch( start=start, end=end, reference=chrom[3:] ) + else: + raise + # Encode reads as list of dictionaries results = [] for read in data: payload = { 'uid': str(read.pos) + str(read.seq), 'start': read.pos, 'end': read.pos + read.rlen, 'name': read.seq } - results.append(payload) bamfile.close() return results diff -r acf985b23928 -r d1bc3b0535d0 lib/galaxy/web/controllers/library_common.py --- a/lib/galaxy/web/controllers/library_common.py Wed Jan 27 15:49:50 2010 -0500 +++ b/lib/galaxy/web/controllers/library_common.py Wed Jan 27 15:50:21 2010 -0500 @@ -489,10 +489,18 @@ msg=util.sanitize_text( msg ), messagetype='error' ) ) lddas.append( ldda ) - # If the library is public all roles are legitimate, but if the library is restricted, only those - # roles associated with the LIBRARY_ACCESS permission are legitimate. library = trans.sa_session.query( trans.app.model.Library ).get( trans.security.decode_id( library_id ) ) - roles = trans.app.security_agent.get_legitimate_roles( trans, library ) + # If access to the dataset is restricted, then use the roles associated with the DATASET_ACCESS permission to + # determine the legitimate roles. If the dataset is public, see if access to the library is restricted. If + # it is, use the roles associated with the LIBRARY_ACCESS permission to determine the legitimate roles. If both + # the dataset and the library are public, all roles are legitimate. All of the datasets will have the same + # permissions at this point. + ldda = lddas[0] + if trans.app.security_agent.dataset_is_public( ldda.dataset ): + # The dataset is public, so check access to the library + roles = trans.app.security_agent.get_legitimate_roles( trans, library ) + else: + roles = trans.app.security_agent.get_legitimate_roles( trans, ldda.dataset ) if params.get( 'update_roles_button', False ): current_user_roles = trans.get_current_user_roles() if cntrller=='library_admin' or ( trans.app.security_agent.can_manage_library_item( current_user_roles, ldda ) and \ diff -r acf985b23928 -r d1bc3b0535d0 lib/galaxy/web/framework/base.py --- a/lib/galaxy/web/framework/base.py Wed Jan 27 15:49:50 2010 -0500 +++ b/lib/galaxy/web/framework/base.py Wed Jan 27 15:50:21 2010 -0500 @@ -327,10 +327,14 @@ def send_file( start_response, trans, body ): # If configured use X-Accel-Redirect header for nginx base = trans.app.config.nginx_x_accel_redirect_base + apache_xsendfile = trans.app.config.apache_xsendfile if base: trans.response.headers['X-Accel-Redirect'] = \ base + os.path.abspath( body.name ) body = [ "" ] + elif apache_xsendfile: + trans.response.headers['X-Sendfile'] = os.path.abspath( body.name ) + body = [ "" ] # Fall back on sending the file in chunks else: body = iterate_file( body )