galaxy-dev
Threads by month
- ----- 2025 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2024 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2023 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2022 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2021 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2020 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2019 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2018 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2017 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2016 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2015 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2014 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2013 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2012 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2011 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2010 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2009 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2008 -----
- December
- November
- October
- September
- August
- 1 participants
- 10009 discussions
05 Nov '08
details: http://www.bx.psu.edu/hg/galaxy/rev/53aa8d34c20a
changeset: 1608:53aa8d34c20a
user: Dan Blankenberg <dan(a)bx.psu.edu>
date: Wed Nov 05 11:43:22 2008 -0500
description:
Possible fix for the infinite metadata recursion issue.
1 file(s) affected in this change:
lib/galaxy/model/__init__.py
diffs (12 lines):
diff -r 4e2ad77de095 -r 53aa8d34c20a lib/galaxy/model/__init__.py
--- a/lib/galaxy/model/__init__.py Wed Nov 05 10:35:06 2008 -0500
+++ b/lib/galaxy/model/__init__.py Wed Nov 05 11:43:22 2008 -0500
@@ -188,7 +188,7 @@
return datatypes_registry.get_datatype_by_extension( self.extension )
def get_metadata( self ):
- if not hasattr( self, '_metadata_collection' ):
+ if not hasattr( self, '_metadata_collection' ) or self._metadata_collection.parent is None: #using weakref to store parent (to prevent circ ref), does a Session.clear() cause parent to be invalidated, while still copying over this non-database attribute?
self._metadata_collection = MetadataCollection( self )
return self._metadata_collection
def set_metadata( self, bunch ):
1
0
05 Nov '08
details: http://www.bx.psu.edu/hg/galaxy/rev/4e2ad77de095
changeset: 1607:4e2ad77de095
user: Greg Von Kuster <greg(a)bx.psu.edu>
date: Wed Nov 05 10:35:06 2008 -0500
description:
Fix jobs report to display jobs in deleted and waiting states.
10 file(s) affected in this change:
lib/galaxy/jobs/__init__.py
lib/galaxy/webapps/reports/controllers/jobs.py
lib/galaxy/webapps/reports/static/january_2008_style/blue/base.css
lib/galaxy/webapps/reports/templates/jobs_all_unfinished.mako
lib/galaxy/webapps/reports/templates/jobs_per_month_in_error.mako
lib/galaxy/webapps/reports/templates/jobs_specified_date_all.mako
lib/galaxy/webapps/reports/templates/jobs_specified_date_in_error.mako
lib/galaxy/webapps/reports/templates/jobs_specified_month_in_error.mako
lib/galaxy/webapps/reports/templates/jobs_tool_for_month.mako
lib/galaxy/webapps/reports/templates/jobs_user_for_month.mako
diffs (181 lines):
diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/jobs/__init__.py
--- a/lib/galaxy/jobs/__init__.py Wed Nov 05 09:28:48 2008 -0500
+++ b/lib/galaxy/jobs/__init__.py Wed Nov 05 10:35:06 2008 -0500
@@ -403,11 +403,11 @@
idata.dataset.refresh() #we need to refresh the base Dataset, since that is where 'state' is stored
# don't run jobs for which the input dataset was deleted
if idata.deleted:
- self.fail( "input data %d was deleted before this job started" % idata.hid )
+ self.fail( "input data %d (file: %s) was deleted before the job started" % ( idata.hid, idata.file_name ) )
return JOB_INPUT_DELETED
# an error in the input data causes us to bail immediately
elif idata.state == idata.states.ERROR:
- self.fail( "input data %d is in an error state" % idata.hid )
+ self.fail( "input data %d (file: %s) is in an error state" % ( idata.hid, idata.file_name ) )
return JOB_INPUT_ERROR
elif idata.state != idata.states.OK:
# need to requeue
diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/webapps/reports/controllers/jobs.py
--- a/lib/galaxy/webapps/reports/controllers/jobs.py Wed Nov 05 09:28:48 2008 -0500
+++ b/lib/galaxy/webapps/reports/controllers/jobs.py Wed Nov 05 10:35:06 2008 -0500
@@ -91,7 +91,8 @@
year_label = start_date.strftime( "%Y" )
q = sa.select( ( sa.func.date( galaxy.model.Job.table.c.create_time ).label( 'date' ),
sa.func.count( galaxy.model.Job.table.c.id ).label( 'total_jobs' ) ),
- whereclause = sa.and_( galaxy.model.Job.table.c.state == 'error',
+ whereclause = sa.and_( sa.or_( galaxy.model.Job.table.c.state == 'error',
+ galaxy.model.Job.table.c.state == 'deleted' ),
galaxy.model.Job.table.c.create_time >= start_date,
galaxy.model.Job.table.c.create_time < end_date ),
from_obj = [ sa.outerjoin( galaxy.model.Job.table,
@@ -133,7 +134,8 @@
galaxy.model.Job.table.c.info,
( galaxy.model.User.table.c.email ).label( 'user_email' ),
galaxy.model.GalaxySession.table.c.remote_addr ),
- whereclause = sa.and_( galaxy.model.Job.table.c.state == 'error',
+ whereclause = sa.and_( sa.or_( galaxy.model.Job.table.c.state == 'error',
+ galaxy.model.Job.table.c.state == 'deleted' ),
galaxy.model.Job.table.c.create_time >= start_date,
galaxy.model.Job.table.c.create_time < end_date ),
from_obj = [ sa.outerjoin( galaxy.model.Job.table,
@@ -245,6 +247,7 @@
galaxy.model.GalaxySession.table.c.remote_addr ),
whereclause = sa.or_( galaxy.model.Job.table.c.state == 'running',
galaxy.model.Job.table.c.state == 'queued',
+ galaxy.model.Job.table.c.state == 'waiting',
galaxy.model.Job.table.c.state == 'new' ),
from_obj = [ sa.outerjoin( galaxy.model.Job.table,
galaxy.model.History.table ).outerjoin( galaxy.model.User.table ).outerjoin( galaxy.model.GalaxySession.table,
@@ -297,7 +300,8 @@
msg = ''
q = sa.select( ( sa.func.date_trunc( 'month', sa.func.date( galaxy.model.Job.table.c.create_time ) ).label( 'date' ),
sa.func.count( galaxy.model.Job.table.c.id ).label( 'total_jobs' ) ),
- whereclause = galaxy.model.Job.table.c.state == 'error',
+ whereclause = sa.or_( galaxy.model.Job.table.c.state == 'error',
+ galaxy.model.Job.table.c.state == 'deleted' ),
from_obj = [ galaxy.model.Job.table ],
group_by = [ sa.func.date_trunc( 'month', sa.func.date( galaxy.model.Job.table.c.create_time ) ) ],
order_by = [ sa.desc( 'date' ) ] )
diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/webapps/reports/static/january_2008_style/blue/base.css
--- a/lib/galaxy/webapps/reports/static/january_2008_style/blue/base.css Wed Nov 05 09:28:48 2008 -0500
+++ b/lib/galaxy/webapps/reports/static/january_2008_style/blue/base.css Wed Nov 05 10:35:06 2008 -0500
@@ -267,6 +267,14 @@
border-bottom: solid #d8b365 1px;
font-weight: bold;
}
+table.colored tr.headerwaiting
+{
+ background: #ebd9b2;
+ background-repeat: repeat-x;
+ background-position: top;
+ border-bottom: solid #d8b365 1px;
+ font-weight: bold;
+}
table.colored tr.headerqueued
{
background: #ee82ee;
diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/webapps/reports/templates/jobs_all_unfinished.mako
--- a/lib/galaxy/webapps/reports/templates/jobs_all_unfinished.mako Wed Nov 05 09:28:48 2008 -0500
+++ b/lib/galaxy/webapps/reports/templates/jobs_all_unfinished.mako Wed Nov 05 10:35:06 2008 -0500
@@ -17,6 +17,8 @@
state = job[0]
if state == 'new':
rowdef = '<tr class="headernew">'
+ elif state == 'waiting':
+ rowdef = '<tr class="headerwaiting">'
elif state == 'queued':
rowdef = '<tr class="headerqueued">'
elif state == 'running':
diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/webapps/reports/templates/jobs_per_month_in_error.mako
--- a/lib/galaxy/webapps/reports/templates/jobs_per_month_in_error.mako Wed Nov 05 09:28:48 2008 -0500
+++ b/lib/galaxy/webapps/reports/templates/jobs_per_month_in_error.mako Wed Nov 05 10:35:06 2008 -0500
@@ -2,7 +2,7 @@
<%def name="main_body()">
<div class="reportBody">
- <h3 align="center">Jobs In Error Per Month</h3>
+ <h3 align="center">Jobs In Error Per Month ( includes jobs deleted before finishing )</h3>
<h4 align="center">Click Month to view the number of jobs in error for each day of that month</h4>
%if msg:
<table align="center" width="70%" class="border" cellpadding="5" cellspacing="5">
diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/webapps/reports/templates/jobs_specified_date_all.mako
--- a/lib/galaxy/webapps/reports/templates/jobs_specified_date_all.mako Wed Nov 05 09:28:48 2008 -0500
+++ b/lib/galaxy/webapps/reports/templates/jobs_specified_date_all.mako Wed Nov 05 10:35:06 2008 -0500
@@ -17,11 +17,15 @@
state = job[0]
if state == 'new':
rowdef = '<tr class="headernew">'
+ elif state == 'waiting':
+ rowdef = '<tr class="headerwaiting">'
elif state == 'queued':
rowdef = '<tr class="headerqueued">'
elif state == 'running':
rowdef = '<tr class="headerrunning">'
elif state == 'error':
+ rowdef = '<tr class="headererror">'
+ elif state == 'deleted':
rowdef = '<tr class="headererror">'
elif state == 'ok':
rowdef = '<tr class="headerok">'
diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/webapps/reports/templates/jobs_specified_date_in_error.mako
--- a/lib/galaxy/webapps/reports/templates/jobs_specified_date_in_error.mako Wed Nov 05 09:28:48 2008 -0500
+++ b/lib/galaxy/webapps/reports/templates/jobs_specified_date_in_error.mako Wed Nov 05 10:35:06 2008 -0500
@@ -2,7 +2,7 @@
<%def name="main_body()">
<div class="reportBody">
- <h3 align="center">All Jobs in Error for ${day_label}, ${month_label} ${day_of_month}, ${year_label}</h3>
+ <h3 align="center">All Jobs in Error for ${day_label}, ${month_label} ${day_of_month}, ${year_label} ( includes jobs deleted before finishing )</h3>
%if msg:
<table align="center" width="70%" class="border" cellpadding="5" cellspacing="5">
<tr><td class="ok_bgr">${msg}</td></tr>
diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/webapps/reports/templates/jobs_specified_month_in_error.mako
--- a/lib/galaxy/webapps/reports/templates/jobs_specified_month_in_error.mako Wed Nov 05 09:28:48 2008 -0500
+++ b/lib/galaxy/webapps/reports/templates/jobs_specified_month_in_error.mako Wed Nov 05 10:35:06 2008 -0500
@@ -2,7 +2,7 @@
<%def name="main_body()">
<div class="reportBody">
- <h3 align="center">Jobs in Error for ${month_label} ${year_label}</h3>
+ <h3 align="center">Jobs in Error for ${month_label} ${year_label} ( includes jobs deleted before finishing )</h3>
<h4 align="center">Click Jobs in Error to view jobs in error for that day</h4>
%if msg:
<table align="center" width="70%" class="border" cellpadding="5" cellspacing="5">
diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/webapps/reports/templates/jobs_tool_for_month.mako
--- a/lib/galaxy/webapps/reports/templates/jobs_tool_for_month.mako Wed Nov 05 09:28:48 2008 -0500
+++ b/lib/galaxy/webapps/reports/templates/jobs_tool_for_month.mako Wed Nov 05 10:35:06 2008 -0500
@@ -17,11 +17,15 @@
state = job[0]
if state == 'new':
rowdef = '<tr class="headernew">'
+ elif state == 'waiting':
+ rowdef = '<tr class="headerwaiting">'
elif state == 'queued':
rowdef = '<tr class="headerqueued">'
elif state == 'running':
rowdef = '<tr class="headerrunning">'
elif state == 'error':
+ rowdef = '<tr class="headererror">'
+ elif state == 'deleted':
rowdef = '<tr class="headererror">'
elif state == 'ok':
rowdef = '<tr class="headerok">'
diff -r 72252f33edd1 -r 4e2ad77de095 lib/galaxy/webapps/reports/templates/jobs_user_for_month.mako
--- a/lib/galaxy/webapps/reports/templates/jobs_user_for_month.mako Wed Nov 05 09:28:48 2008 -0500
+++ b/lib/galaxy/webapps/reports/templates/jobs_user_for_month.mako Wed Nov 05 10:35:06 2008 -0500
@@ -17,11 +17,15 @@
state = job[0]
if state == 'new':
rowdef = '<tr class="headernew">'
+ elif state == 'waiting':
+ rowdef = '<tr class="headerwaiting">'
elif state == 'queued':
rowdef = '<tr class="headerqueued">'
elif state == 'running':
rowdef = '<tr class="headerrunning">'
elif state == 'error':
+ rowdef = '<tr class="headererror">'
+ elif state == 'deleted':
rowdef = '<tr class="headererror">'
elif state == 'ok':
rowdef = '<tr class="headerok">'
1
0
details: http://www.bx.psu.edu/hg/galaxy/rev/72252f33edd1
changeset: 1606:72252f33edd1
user: Greg Von Kuster <greg(a)bx.psu.edu>
date: Wed Nov 05 09:28:48 2008 -0500
description:
Make logging events a config setting.
3 file(s) affected in this change:
lib/galaxy/config.py
lib/galaxy/web/framework/__init__.py
universe_wsgi.ini.sample
diffs (68 lines):
diff -r f2e583f7cab9 -r 72252f33edd1 lib/galaxy/config.py
--- a/lib/galaxy/config.py Tue Nov 04 16:04:49 2008 -0500
+++ b/lib/galaxy/config.py Wed Nov 05 09:28:48 2008 -0500
@@ -58,6 +58,7 @@
self.use_heartbeat = string_as_bool( kwargs.get( 'use_heartbeat', False ) )
self.use_memdump = string_as_bool( kwargs.get( 'use_memdump', False ) )
self.log_memory_usage = string_as_bool( kwargs.get( 'log_memory_usage', False ) )
+ self.log_events = string_as_bool( kwargs.get( 'log_events', False ) )
self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea" ).lower().split(",")
self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "wormbase,flybase,elegans" ).lower().split(",")
self.brand = kwargs.get( 'brand', None )
diff -r f2e583f7cab9 -r 72252f33edd1 lib/galaxy/web/framework/__init__.py
--- a/lib/galaxy/web/framework/__init__.py Tue Nov 04 16:04:49 2008 -0500
+++ b/lib/galaxy/web/framework/__init__.py Wed Nov 05 09:28:48 2008 -0500
@@ -127,23 +127,24 @@
return self.app.model.context.current
def log_event( self, message, tool_id=None, **kwargs ):
"""
- Application level logging. Still needs fleshing out (log levels and
- such)
+ Application level logging. Still needs fleshing out (log levels and such)
+ Logging events is a config setting - if False, do not log.
"""
- event = self.app.model.Event()
- event.tool_id = tool_id
- try:
- event.message = message % kwargs
- except:
- event.message = message
- event.history = self.history
- try:
- event.history_id = self.history.id
- except:
- event.history_id = None
- event.user = self.user
- event.session_id = self.galaxy_session.id
- event.flush()
+ if self.app.config.log_events:
+ event = self.app.model.Event()
+ event.tool_id = tool_id
+ try:
+ event.message = message % kwargs
+ except:
+ event.message = message
+ event.history = self.history
+ try:
+ event.history_id = self.history.id
+ except:
+ event.history_id = None
+ event.user = self.user
+ event.session_id = self.galaxy_session.id
+ event.flush()
def get_cookie( self, name='galaxysession' ):
"""Convienience method for getting the galaxysession cookie"""
try:
diff -r f2e583f7cab9 -r 72252f33edd1 universe_wsgi.ini.sample
--- a/universe_wsgi.ini.sample Tue Nov 04 16:04:49 2008 -0500
+++ b/universe_wsgi.ini.sample Wed Nov 05 09:28:48 2008 -0500
@@ -18,6 +18,9 @@
# Log memory usage
log_memory_usage = False
+
+# Log events
+log_events = True
# Should jobs be tracked through the database, rather than in memory
## track_jobs_in_database = true
1
0
05 Nov '08
details: http://www.bx.psu.edu/hg/galaxy/rev/f2e583f7cab9
changeset: 1605:f2e583f7cab9
user: Greg Von Kuster <greg(a)bx.psu.edu>
date: Tue Nov 04 16:04:49 2008 -0500
description:
Add ability to track memory usage in code secments - requires config addition:
# Log memory usage
log_memory_usage = False
Added logging statements in tool execution methods and most methods in the user and root controllers.
Also fixes for checking config settings that should be boolean ( e.g., memdump, heartbeat ).
9 file(s) affected in this change:
lib/galaxy/app.py
lib/galaxy/config.py
lib/galaxy/eggs/__init__.py
lib/galaxy/tools/actions/__init__.py
lib/galaxy/tools/actions/upload.py
lib/galaxy/util/memory_usage.py
lib/galaxy/web/controllers/root.py
lib/galaxy/web/controllers/user.py
universe_wsgi.ini.sample
diffs (469 lines):
diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/app.py
--- a/lib/galaxy/app.py Tue Nov 04 14:00:30 2008 -0500
+++ b/lib/galaxy/app.py Tue Nov 04 16:04:49 2008 -0500
@@ -38,6 +38,7 @@
# Heartbeat and memdump for thread / heap profiling
self.heartbeat = None
self.memdump = None
+ self.memory_usage = None
# Start the heartbeat process if configured and available
if self.config.use_heartbeat:
from galaxy.util import heartbeat
@@ -49,6 +50,10 @@
from galaxy.util import memdump
if memdump.Memdump:
self.memdump = memdump.Memdump()
+ # Enable memory_usage logging if configured
+ if self.config.log_memory_usage:
+ from galaxy.util import memory_usage
+ self.memory_usage = memory_usage
def shutdown( self ):
self.job_manager.shutdown()
if self.heartbeat:
diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/config.py
--- a/lib/galaxy/config.py Tue Nov 04 14:00:30 2008 -0500
+++ b/lib/galaxy/config.py Tue Nov 04 16:04:49 2008 -0500
@@ -55,8 +55,9 @@
self.pbs_dataset_server = kwargs.get('pbs_dataset_server', "" )
self.pbs_dataset_path = kwargs.get('pbs_dataset_path', "" )
self.pbs_stage_path = kwargs.get('pbs_stage_path', "" )
- self.use_heartbeat = string_as_bool( kwargs.get( 'use_heartbeat', "False" ) )
- self.use_memdump = kwargs.get( 'use_memdump', False )
+ self.use_heartbeat = string_as_bool( kwargs.get( 'use_heartbeat', False ) )
+ self.use_memdump = string_as_bool( kwargs.get( 'use_memdump', False ) )
+ self.log_memory_usage = string_as_bool( kwargs.get( 'log_memory_usage', False ) )
self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea" ).lower().split(",")
self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "wormbase,flybase,elegans" ).lower().split(",")
self.brand = kwargs.get( 'brand', None )
diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/eggs/__init__.py
--- a/lib/galaxy/eggs/__init__.py Tue Nov 04 14:00:30 2008 -0500
+++ b/lib/galaxy/eggs/__init__.py Tue Nov 04 16:04:49 2008 -0500
@@ -484,7 +484,7 @@
return False
elif egg_name == "threadframe":
try:
- if self.config.get( "app:main", "use_heartbeat" ) == "True":
+ if self.config.get( "app:main", "use_heartbeat" ):
return True
else:
return False
@@ -492,7 +492,7 @@
return False
elif egg_name == "guppy":
try:
- if self.config.get( "app:main", "use_memdump" ) == "True":
+ if self.config.get( "app:main", "use_memdump" ):
return True
else:
return False
diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/tools/actions/__init__.py
--- a/lib/galaxy/tools/actions/__init__.py Tue Nov 04 14:00:30 2008 -0500
+++ b/lib/galaxy/tools/actions/__init__.py Tue Nov 04 16:04:49 2008 -0500
@@ -65,6 +65,9 @@
return input_datasets
def execute(self, tool, trans, incoming={}, set_output_hid=True ):
+ if trans.app.memory_usage:
+ # Keep track of memory usage
+ m0 = trans.app.memory_usage.memory()
out_data = {}
# Collect any input datasets from the incoming parameters
inp_data = self.collect_input_datasets( tool, incoming, trans )
@@ -198,6 +201,9 @@
for name, dataset in out_data.iteritems():
job.add_output_dataset( name, dataset )
trans.app.model.flush()
+ if trans.app.memory_usage:
+ m1 = trans.app.memory_usage.memory( m0, pretty=True )
+ log.info("End of tool %s execution for job id %d, memory used increased by %s" % ( tool.id, job.id, m1 ) )
# Some tools are not really executable, but jobs are still created for them ( for record keeping ).
# Examples include tools that redirect to other applications ( epigraph ). These special tools must
# include something that can be retrieved from the params ( e.g., REDIRECT_URL ) to keep the job
diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/tools/actions/upload.py
--- a/lib/galaxy/tools/actions/upload.py Tue Nov 04 14:00:30 2008 -0500
+++ b/lib/galaxy/tools/actions/upload.py Tue Nov 04 16:04:49 2008 -0500
@@ -15,6 +15,9 @@
self.line_count = None
def execute( self, tool, trans, incoming={}, set_output_hid = True ):
+ if trans.app.memory_usage:
+ # Keep track of memory usage
+ m0 = trans.app.memory_usage.memory()
data_file = incoming['file_data']
file_type = incoming['file_type']
dbkey = incoming['dbkey']
@@ -93,6 +96,9 @@
job.flush()
log.info( 'job id %d ended ok, file size: %s' % ( job.id, file_size_str ) )
trans.log_event( 'job id %d ended ok, file size: %s' % ( job.id, file_size_str ), tool_id=tool.id )
+ if trans.app.memory_usage:
+ m1 = trans.app.memory_usage.memory( m0, pretty=True )
+ log.info("End of tool %s execution for job id %d, memory used increased by %s" % ( tool.id, job.id, m1 ) )
return dict( output=hda )
def upload_empty(self, trans, job, err_code, err_msg):
diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/util/memory_usage.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/lib/galaxy/util/memory_usage.py Tue Nov 04 16:04:49 2008 -0500
@@ -0,0 +1,44 @@
+import os, platform, logging
+from galaxy.datatypes.data import nice_size
+
+log = logging.getLogger( __name__ )
+
+_proc_status = '/proc/%d/status' % os.getpid()
+
+_scale = { 'kB': 1024.0, 'mB': 1024.0*1024.0, 'KB': 1024.0, 'MB': 1024.0*1024.0 }
+
+def _VmB( VmKey ):
+ '''Private.
+ '''
+ global _proc_status, _scale
+ # get pseudo file /proc/<pid>/status
+ try:
+ t = open( _proc_status )
+ v = t.read()
+ t.close()
+ except:
+ log.debug("memory_usage is currently supported only on Linux, your platform is %s %s" % ( platform.system(), platform.release() ) )
+ return 0.0 # non-Linux?
+ # get VmKey line e.g. 'VmRSS: 9999 kB\n ...'
+ i = v.index( VmKey )
+ v = v[ i: ].split( None, 3 ) # whitespace
+ if len( v ) < 3:
+ return 0.0 # invalid format?
+ # convert Vm value to bytes
+ return float( v[ 1 ] ) * _scale[ v[ 2 ] ]
+def memory( since=0.0, pretty=False ):
+ '''Return memory usage in bytes.
+ '''
+ size = _VmB( 'VmSize:' ) - since
+ if pretty:
+ return nice_size( size )
+ else:
+ return size
+def resident( since=0.0 ):
+ '''Return resident memory usage in bytes.
+ '''
+ return _VmB( 'VmRSS:' ) - since
+def stacksize( since=0.0 ):
+ '''Return stack size in bytes.
+ '''
+ return _VmB( 'VmStk:' ) - since
diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/web/controllers/root.py
--- a/lib/galaxy/web/controllers/root.py Tue Nov 04 14:00:30 2008 -0500
+++ b/lib/galaxy/web/controllers/root.py Tue Nov 04 16:04:49 2008 -0500
@@ -116,6 +116,9 @@
Returns data directly into the browser.
Sets the mime-type according to the extension
"""
+ if trans.app.memory_usage:
+ # Keep track of memory usage
+ m0 = self.app.memory_usage.memory()
if hid is not None:
try:
hid = int( hid )
@@ -146,6 +149,9 @@
fname = ''.join(c in valid_chars and c or '_' for c in fname)[0:150]
trans.response.headers["Content-Disposition"] = "attachment; filename=GalaxyHistoryItem-%s-[%s]%s" % (data.hid, fname, toext)
trans.log_event( "Display dataset id: %s" % str(id) )
+ if self.app.memory_usage:
+ m1 = trans.app.memory_usage.memory( m0, pretty=True )
+ log.info( "End of root/display, memory used increased by %s" % m1 )
try:
return open( data.file_name )
except:
@@ -321,6 +327,9 @@
@web.expose
def history_delete( self, trans, id=None, **kwd):
"""Deletes a list of histories, ensures that histories are owned by current user"""
+ if trans.app.memory_usage:
+ # Keep track of memory usage
+ m0 = self.app.memory_usage.memory()
history_names = []
if id:
if isinstance( id, list ):
@@ -348,12 +357,18 @@
trans.log_event( "History id %s marked as deleted" % str(hid) )
else:
return trans.show_message( "You must select at least one history to delete." )
+ if self.app.memory_usage:
+ m1 = trans.app.memory_usage.memory( m0, pretty=True )
+ log.info( "End of root/history_delete, memory used increased by %s" % m1 )
return trans.show_message( "History deleted: %s" % ",".join(history_names),
refresh_frames=['history'])
@web.expose
def history_undelete( self, trans, id=[], **kwd):
"""Undeletes a list of histories, ensures that histories are owned by current user"""
+ if trans.app.memory_usage:
+ # Keep track of memory usage
+ m0 = self.app.memory_usage.memory()
history_names = []
errors = []
ok_msg = ""
@@ -385,22 +400,34 @@
ok_msg = "Histories (%s) have been undeleted." % ", ".join( history_names )
else:
errors.append( "You must select at least one history to undelete." )
+ if self.app.memory_usage:
+ m1 = trans.app.memory_usage.memory( m0, pretty=True )
+ log.info( "End of root/history_undelete, memory used increased by %s" % m1 )
return self.history_available( trans, id=','.join( id ), show_deleted=True, ok_msg = ok_msg, error_msg = " ".join( errors ) )
@web.expose
def clear_history( self, trans ):
"""Clears the history for a user"""
+ if trans.app.memory_usage:
+ # Keep track of memory usage
+ m0 = self.app.memory_usage.memory()
history = trans.get_history()
for dataset in history.datasets:
dataset.deleted = True
dataset.clear_associated_files()
self.app.model.flush()
trans.log_event( "History id %s cleared" % (str(history.id)) )
+ if self.app.memory_usage:
+ m1 = trans.app.memory_usage.memory( m0, pretty=True )
+ log.info( "End of root/clear_history, memory used increased by %s" % m1 )
trans.response.send_redirect( url_for("/index" ) )
@web.expose
@web.require_login( "share histories with other users" )
def history_share( self, trans, id=None, email="", **kwd ):
+ if trans.app.memory_usage:
+ # Keep track of memory usage
+ m0 = self.app.memory_usage.memory()
send_to_err = ""
if not id:
id = trans.get_history().id
@@ -427,6 +454,9 @@
trans.log_event( "History share, id: %s, name: '%s': to new id: %s" % (str(history.id), history.name, str(new_history.id)) )
self.app.model.flush()
return trans.show_message( "History (%s) has been shared with: %s" % (",".join(history_names),email) )
+ if self.app.memory_usage:
+ m1 = trans.app.memory_usage.memory( m0, pretty=True )
+ log.info( "End of root/history_share, memory used increased by %s" % m1 )
return trans.fill_template( "/history/share.mako", histories=histories, email=email, send_to_err=send_to_err)
@web.expose
@@ -435,18 +465,21 @@
"""
List all available histories
"""
+ if trans.app.memory_usage:
+ # Keep track of memory usage
+ m0 = self.app.memory_usage.memory()
if as_xml:
trans.response.set_content_type('text/xml')
return trans.fill_template( "/history/list_as_xml.mako" )
if not isinstance( id, list ):
id = id.split( "," )
trans.log_event( "History id %s available" % str( id ) )
-
history_operations = dict( share=self.history_share, rename=self.history_rename, delete=self.history_delete, undelete=self.history_undelete )
-
+ if self.app.memory_usage:
+ m1 = trans.app.memory_usage.memory( m0, pretty=True )
+ log.info( "End of root/history_available, memory used increased by %s" % m1 )
if do_operation in history_operations:
return history_operations[do_operation]( trans, id=id, show_deleted=show_deleted, ok_msg=ok_msg, error_msg=error_msg, **kwd )
-
return trans.fill_template( "/history/list.mako", ids=id,
user=trans.get_user(),
current_history=trans.get_history(),
@@ -457,6 +490,9 @@
@web.expose
def history_import( self, trans, id=None, confirm=False, **kwd ):
+ if trans.app.memory_usage:
+ # Keep track of memory usage
+ m0 = self.app.memory_usage.memory()
msg = ""
user = trans.get_user()
user_history = trans.get_history()
@@ -497,6 +533,9 @@
new_history.flush()
trans.set_history( new_history )
trans.log_event( "History imported, id: %s, name: '%s': " % (str(new_history.id) , new_history.name ) )
+ if self.app.memory_usage:
+ m1 = trans.app.memory_usage.memory( m0, pretty=True )
+ log.info( "End of root/history_import, memory used increased by %s" % m1 )
return trans.show_ok_message( """
History "%s" has been imported. Click <a href="%s">here</a>
to begin.""" % ( new_history.name, web.url_for( '/' ) ) )
@@ -511,6 +550,9 @@
if not id:
return trans.response.send_redirect( web.url_for( action='history_available' ) )
else:
+ if trans.app.memory_usage:
+ # Keep track of memory usage
+ m0 = self.app.memory_usage.memory()
new_history = trans.app.model.History.get( id )
if new_history:
galaxy_session = trans.get_galaxy_session()
@@ -522,6 +564,9 @@
new_history.flush()
trans.set_history( new_history )
trans.log_event( "History switched to id: %s, name: '%s'" % (str(new_history.id), new_history.name ) )
+ if self.app.memory_usage:
+ m1 = trans.app.memory_usage.memory( m0, pretty=True )
+ log.info( "End of root/history_switch, memory used increased by %s" % m1 )
return trans.show_message( "History switched to: %s" % new_history.name,
refresh_frames=['history'])
else:
@@ -529,13 +574,22 @@
@web.expose
def history_new( self, trans ):
+ if trans.app.memory_usage:
+ # Keep track of memory usage
+ m0 = self.app.memory_usage.memory()
trans.new_history()
trans.log_event( "Created new History, id: %s." % str(trans.get_history().id) )
+ if self.app.memory_usage:
+ m1 = trans.app.memory_usage.memory( m0, pretty=True )
+ log.info( "End of root/history_new, memory used increased by %s" % m1 )
return trans.show_message( "New history created", refresh_frames = ['history'] )
@web.expose
@web.require_login( "renames histories" )
def history_rename( self, trans, id=None, name=None, **kwd ):
+ if trans.app.memory_usage:
+ # Keep track of memory usage
+ m0 = self.app.memory_usage.memory()
user = trans.get_user()
if not isinstance( id, list ):
@@ -572,11 +626,17 @@
change_msg = change_msg + "<p>You must specify a valid name for History: "+cur_names[i]+"</p>"
else:
change_msg = change_msg + "<p>History: "+cur_names[i]+" does not appear to belong to you.</p>"
+ if self.app.memory_usage:
+ m1 = trans.app.memory_usage.memory( m0, pretty=True )
+ log.info( "End of root/history_rename, memory used increased by %s" % m1 )
return trans.show_message( "<p>%s" % change_msg, refresh_frames=['history'] )
@web.expose
def history_add_to( self, trans, history_id=None, file_data=None, name="Data Added to History",info=None,ext="txt",dbkey="?",**kwd ):
"""Adds a POSTed file to a History"""
+ if trans.app.memory_usage:
+ # Keep track of memory usage
+ m0 = self.app.memory_usage.memory()
try:
history = trans.app.model.History.get( history_id )
data = trans.app.model.HistoryDatasetAssociation( name = name, info = info, extension = ext, dbkey = dbkey, create_dataset = True )
@@ -595,6 +655,9 @@
data.set_size()
data.flush()
trans.log_event("Added dataset %d to history %d" %(data.id, trans.history.id))
+ if self.app.memory_usage:
+ m1 = trans.app.memory_usage.memory( m0, pretty=True )
+ log.info( "End of root/history_add_to, memory used increased by %s" % m1 )
return trans.show_ok_message("Dataset "+str(data.hid)+" added to history "+str(history_id)+".")
except:
return trans.show_error_message("Adding File to History has Failed")
@@ -602,6 +665,9 @@
@web.expose
def dataset_make_primary( self, trans, id=None):
"""Copies a dataset and makes primary"""
+ if trans.app.memory_usage:
+ # Keep track of memory usage
+ m0 = self.app.memory_usage.memory()
try:
old_data = self.app.model.HistoryDatasetAssociation.get( id )
new_data = old_data.copy()
@@ -610,6 +676,9 @@
history = trans.get_history()
history.add_dataset(new_data)
new_data.flush()
+ if self.app.memory_usage:
+ m1 = trans.app.memory_usage.memory( m0, pretty=True )
+ log.info( "End of root/dataset_make_primary, memory used increased by %s" % m1 )
return trans.show_message( "<p>Secondary dataset has been made primary.</p>", refresh_frames=['history'] )
except:
return trans.show_error_message( "<p>Failed to make secondary dataset primary.</p>" )
diff -r 7a5d9759c702 -r f2e583f7cab9 lib/galaxy/web/controllers/user.py
--- a/lib/galaxy/web/controllers/user.py Tue Nov 04 14:00:30 2008 -0500
+++ b/lib/galaxy/web/controllers/user.py Tue Nov 04 16:04:49 2008 -0500
@@ -70,6 +70,9 @@
@web.expose
def login( self, trans, email='', password='' ):
+ if trans.app.memory_usage:
+ # Keep track of memory usage
+ m0 = trans.app.memory_usage.memory()
email_error = password_error = None
# Attempt login
if email or password:
@@ -85,6 +88,9 @@
trans.handle_user_login( user )
trans.log_event( "User logged in" )
return trans.show_ok_message( "Now logged in as " + user.email, refresh_frames=['masthead', 'history'] )
+ if trans.app.memory_usage:
+ m1 = trans.app.memory_usage.memory( m0, pretty=True )
+ log.info( "End of user/login, memory used increased by %s" % m1 )
return trans.show_form(
web.FormBuilder( web.url_for(), "Login", submit_text="Login" )
.add_text( "email", "Email address", value=email, error=email_error )
@@ -92,13 +98,22 @@
help="<a href='%s'>Forgot password? Reset here</a>" % web.url_for( action='reset_password' ) ) )
@web.expose
def logout( self, trans ):
+ if trans.app.memory_usage:
+ # Keep track of memory usage
+ m0 = trans.app.memory_usage.memory()
# Since logging an event requires a session, we'll log prior to ending the session
trans.log_event( "User logged out" )
trans.handle_user_logout()
+ if trans.app.memory_usage:
+ m1 = trans.app.memory_usage.memory( m0, pretty=True )
+ log.info( "End of user/logout, memory used increased by %s" % m1 )
return trans.show_ok_message( "You are no longer logged in", refresh_frames=['masthead', 'history'] )
@web.expose
def create( self, trans, email='', password='', confirm='',subscribe=False ):
+ if trans.app.memory_usage:
+ # Keep track of memory usage
+ m0 = trans.app.memory_usage.memory()
email_error = password_error = confirm_error = None
if email:
if len( email ) == 0 or "@" not in email or "." not in email:
@@ -120,10 +135,13 @@
trans.log_event( "User logged in" )
#subscribe user to email list
if subscribe:
- mail = os.popen("%s -t" % self.app.config.sendmail_path, 'w')
- mail.write("To: %s\nFrom: %s\nSubject: Join Mailing List\n\nJoin Mailing list." % (self.app.config.mailing_join_addr,email) )
+ mail = os.popen("%s -t" % trans.app.config.sendmail_path, 'w')
+ mail.write("To: %s\nFrom: %s\nSubject: Join Mailing List\n\nJoin Mailing list." % (trans.app.config.mailing_join_addr,email) )
if mail.close():
return trans.show_warn_message( "Now logged in as " + user.email+". However, subscribing to the mailing list has failed.", refresh_frames=['masthead', 'history'] )
+ if trans.app.memory_usage:
+ m1 = trans.app.memory_usage.memory( m0, pretty=True )
+ log.info( "End of user/create, memory used increased by %s" % m1 )
return trans.show_ok_message( "Now logged in as " + user.email, refresh_frames=['masthead', 'history'] )
return trans.show_form(
web.FormBuilder( web.url_for(), "Create account", submit_text="Create" )
@@ -145,7 +163,7 @@
new_pass = ""
for i in range(15):
new_pass = new_pass + choice(chars)
- mail = os.popen("%s -t" % self.app.config.sendmail_path, 'w')
+ mail = os.popen("%s -t" % trans.app.config.sendmail_path, 'w')
mail.write("To: %s\nFrom: no-reply@%s\nSubject: Galaxy Password Reset\n\nYour password has been reset to \"%s\" (no quotes)." % (email, trans.request.remote_addr, new_pass) )
if mail.close():
return trans.show_ok_message( "Failed to reset password! If this problem persist, submit a bug report.")
diff -r 7a5d9759c702 -r f2e583f7cab9 universe_wsgi.ini.sample
--- a/universe_wsgi.ini.sample Tue Nov 04 14:00:30 2008 -0500
+++ b/universe_wsgi.ini.sample Tue Nov 04 16:04:49 2008 -0500
@@ -15,6 +15,9 @@
# Specifies the factory for the universe WSGI application
paste.app_factory = galaxy.web.buildapp:app_factory
log_level = DEBUG
+
+# Log memory usage
+log_memory_usage = False
# Should jobs be tracked through the database, rather than in memory
## track_jobs_in_database = true
1
0
details: http://www.bx.psu.edu/hg/galaxy/rev/7a5d9759c702
changeset: 1604:7a5d9759c702
user: Nate Coraor <nate(a)bx.psu.edu>
date: Tue Nov 04 14:00:30 2008 -0500
description:
Fix get_bool in config
1 file(s) affected in this change:
lib/galaxy/config.py
diffs (12 lines):
diff -r 85347b21a1b8 -r 7a5d9759c702 lib/galaxy/config.py
--- a/lib/galaxy/config.py Tue Nov 04 13:20:39 2008 -0500
+++ b/lib/galaxy/config.py Tue Nov 04 14:00:30 2008 -0500
@@ -85,7 +85,7 @@
return self.config_dict.get( key, default )
def get_bool( self, key, default ):
if key in self.config_dict:
- return string_as_bool( key )
+ return string_as_bool( self.config_dict[key] )
else:
return default
def check( self ):
1
0
05 Nov '08
details: http://www.bx.psu.edu/hg/galaxy/rev/52e504c91cbc
changeset: 1601:52e504c91cbc
user: Greg Von Kuster <greg(a)bx.psu.edu>
date: Tue Nov 04 09:27:49 2008 -0500
description:
Apply James' faster_random patch from Nov 1.
1 file(s) affected in this change:
lib/galaxy/web/security/__init__.py
diffs (34 lines):
diff -r 7ae75e6d9d6a -r 52e504c91cbc lib/galaxy/web/security/__init__.py
--- a/lib/galaxy/web/security/__init__.py Mon Nov 03 15:31:38 2008 -0500
+++ b/lib/galaxy/web/security/__init__.py Tue Nov 04 09:27:49 2008 -0500
@@ -10,18 +10,17 @@
log = logging.getLogger( __name__ )
if os.path.exists( "/dev/urandom" ):
- log.debug("###using /dev/urandom....")
# We have urandom, use it as the source of random data
random_fd = os.open( "/dev/urandom", os.O_RDONLY )
def get_random_bytes( nbytes ):
value = os.read( random_fd, nbytes )
# Normally we should get as much as we need
if len( value ) == nbytes:
- return value
+ return value.encode( "hex" )
# If we don't, keep reading (this is slow and should never happen)
while len( value ) < nbytes:
value += os.read( random_fd, nbytes - len( value ) )
- return value
+ return value.encode( "hex" )
else:
def get_random_bytes( nbytes ):
nbits = nbytes * 8
@@ -29,7 +28,8 @@
while random_pool.entropy < nbits:
random_pool.add_event()
random_pool.stir()
- return( str( number.getRandomNumber( nbits, random_pool.get_bytes ) ) )
+ return str( number.getRandomNumber( nbits, random_pool.get_bytes ) )
+
class SecurityHelper( object ):
# TODO: checking if histories/datasets are owned by the current user) will be moved here.
1
0
details: http://www.bx.psu.edu/hg/galaxy/rev/85347b21a1b8
changeset: 1603:85347b21a1b8
user: Greg Von Kuster <greg(a)bx.psu.edu>
date: Tue Nov 04 13:20:39 2008 -0500
description:
Fixes for upload tool job creation.
1 file(s) affected in this change:
lib/galaxy/tools/actions/upload.py
diffs (58 lines):
diff -r 531be9ea2486 -r 85347b21a1b8 lib/galaxy/tools/actions/upload.py
--- a/lib/galaxy/tools/actions/upload.py Tue Nov 04 11:56:09 2008 -0500
+++ b/lib/galaxy/tools/actions/upload.py Tue Nov 04 13:20:39 2008 -0500
@@ -36,7 +36,7 @@
except:
job.tool_version = "1.0.0"
job.flush()
- log.debug( 'tool %s created job id %d' % ( tool.id, job.id ) )
+ log.info( 'tool %s created job id %d' % ( tool.id, job.id ) )
trans.log_event( 'created job id %d' % job.id, tool_id=tool.id )
if 'local_filename' in dir( data_file ):
# Use the existing file
@@ -88,11 +88,11 @@
return self.upload_empty( trans, job, "No data error:", "either you pasted no data, the url you specified is invalid, or you have not specified a file." )
hda = data_list[0]
job.state = trans.app.model.Job.states.OK
- file_size_str = nice_size( hda.dataset.file_size )
- job.info = "Uploaded file '%s', size: %s" % ( file_name, file_size_str )
+ file_size_str = datatypes.data.nice_size( hda.dataset.file_size )
+ job.info = "%s, size: %s" % ( hda.info, file_size_str )
job.flush()
- log.debug( 'tool %s job id %d ended ok, file: %s, size: %s' % ( tool.id, job.id, file_name, file_size_str ) )
- trans.log_event( 'job id %d ended ok, file: %s, size: %s' % ( job.id, file_name, file_size_str ), tool_id=tool.id )
+ log.info( 'job id %d ended ok, file size: %s' % ( job.id, file_size_str ) )
+ trans.log_event( 'job id %d ended ok, file size: %s' % ( job.id, file_size_str ), tool_id=tool.id )
return dict( output=hda )
def upload_empty(self, trans, job, err_code, err_msg):
@@ -111,7 +111,7 @@
job.info = err_msg
# If the failure is due to a Galaxy framework exception, save the traceback
job.flush()
- log.debug( 'tool %s job id %d ended with errors' % ( job.tool_id, job.id ) )
+ log.info( 'job id %d ended with errors' % job.id )
trans.log_event( 'job id %d ended with errors' % job.id, tool_id=job.tool_id )
return dict( output=data )
@@ -301,20 +301,5 @@
temp.close()
return False
-def nice_size( size ):
- """Returns a readably formatted string with the size"""
- words = [ 'bytes', 'Kb', 'Mb', 'Gb' ]
- try:
- size = float( size )
- except:
- return '??? bytes'
- for ind, word in enumerate( words ):
- step = 1024 ** ( ind + 1 )
- if step > size:
- size = size / float( 1024 ** ind )
- out = "%.1f %s" % ( size, word )
- return out
- return '??? bytes'
-
class BadFileException( Exception ):
pass
1
0
05 Nov '08
details: http://www.bx.psu.edu/hg/galaxy/rev/531be9ea2486
changeset: 1602:531be9ea2486
user: Greg Von Kuster <greg(a)bx.psu.edu>
date: Tue Nov 04 11:56:09 2008 -0500
description:
Upload tool will now create a job record, no job queued though.
1 file(s) affected in this change:
lib/galaxy/tools/actions/upload.py
diffs (120 lines):
diff -r 52e504c91cbc -r 531be9ea2486 lib/galaxy/tools/actions/upload.py
--- a/lib/galaxy/tools/actions/upload.py Tue Nov 04 09:27:49 2008 -0500
+++ b/lib/galaxy/tools/actions/upload.py Tue Nov 04 11:56:09 2008 -0500
@@ -25,7 +25,19 @@
space_to_tab = True
temp_name = ""
data_list = []
-
+ # Create the job object
+ job = trans.app.model.Job()
+ job.session_id = trans.get_galaxy_session().id
+ job.history_id = trans.history.id
+ job.tool_id = tool.id
+ try:
+ # For backward compatibility, some tools may not have versions yet.
+ job.tool_version = tool.version
+ except:
+ job.tool_version = "1.0.0"
+ job.flush()
+ log.debug( 'tool %s created job id %d' % ( tool.id, job.id ) )
+ trans.log_event( 'created job id %d' % job.id, tool_id=tool.id )
if 'local_filename' in dir( data_file ):
# Use the existing file
try:
@@ -34,7 +46,7 @@
file_name = file_name.split( '/' )[-1]
data_list.append( self.add_file( trans, data_file.local_filename, file_name, file_type, dbkey, space_to_tab=space_to_tab ) )
except Exception, e:
- return self.upload_empty( trans, "Error:", str( e ) )
+ return self.upload_empty( trans, job, "Error:", str( e ) )
elif 'filename' in dir( data_file ):
try:
file_name = data_file.filename
@@ -43,7 +55,7 @@
temp_name = sniff.stream_to_file( data_file.file )
data_list.append( self.add_file( trans, temp_name, file_name, file_type, dbkey, space_to_tab=space_to_tab ) )
except Exception, e:
- return self.upload_empty( trans, "Error:", str( e ) )
+ return self.upload_empty( trans, job, "Error:", str( e ) )
if url_paste not in [ None, "" ]:
if url_paste.lower().find( 'http://' ) >= 0 or url_paste.lower().find( 'ftp://' ) >= 0:
url_paste = url_paste.replace( '\r', '' ).split( '\n' )
@@ -54,7 +66,7 @@
temp_name = sniff.stream_to_file( urllib.urlopen( line ) )
data_list.append( self.add_file( trans, temp_name, line, file_type, dbkey, info="uploaded url", space_to_tab=space_to_tab ) )
except Exception, e:
- return self.upload_empty( trans, "Error:", str( e ) )
+ return self.upload_empty( trans, job, "Error:", str( e ) )
else:
is_valid = False
for line in url_paste:
@@ -67,26 +79,40 @@
temp_name = sniff.stream_to_file( StringIO.StringIO( url_paste ) )
data_list.append( self.add_file( trans, temp_name, 'Pasted Entry', file_type, dbkey, info="pasted entry", space_to_tab=space_to_tab ) )
except Exception, e:
- return self.upload_empty( trans, "Error:", str( e ) )
+ return self.upload_empty( trans, job, "Error:", str( e ) )
else:
- return self.upload_empty( trans, "No data error:", "you pasted no data." )
+ return self.upload_empty( trans, job, "No data error:", "you pasted no data." )
if self.empty:
- return self.upload_empty( trans, "Empty file error:", "you attempted to upload an empty file." )
+ return self.upload_empty( trans, job, "Empty file error:", "you attempted to upload an empty file." )
elif len( data_list ) < 1:
- return self.upload_empty( trans, "No data error:", "either you pasted no data, the url you specified is invalid, or you have not specified a file." )
- return dict( output=data_list[0] )
+ return self.upload_empty( trans, job, "No data error:", "either you pasted no data, the url you specified is invalid, or you have not specified a file." )
+ hda = data_list[0]
+ job.state = trans.app.model.Job.states.OK
+ file_size_str = nice_size( hda.dataset.file_size )
+ job.info = "Uploaded file '%s', size: %s" % ( file_name, file_size_str )
+ job.flush()
+ log.debug( 'tool %s job id %d ended ok, file: %s, size: %s' % ( tool.id, job.id, file_name, file_size_str ) )
+ trans.log_event( 'job id %d ended ok, file: %s, size: %s' % ( job.id, file_name, file_size_str ), tool_id=tool.id )
+ return dict( output=hda )
- def upload_empty(self, trans, err_code, err_msg):
+ def upload_empty(self, trans, job, err_code, err_msg):
data = trans.app.model.HistoryDatasetAssociation( create_dataset = True )
data.name = err_code
data.extension = "txt"
data.dbkey = "?"
data.info = err_msg
data.file_size = 0
+ data.state = data.states.EMPTY
data.flush()
- data.state = data.states.EMPTY
trans.history.add_dataset( data )
trans.app.model.flush()
+ # Indicate job failure by setting state and message
+ job.state = trans.app.model.Job.states.ERROR
+ job.info = err_msg
+ # If the failure is due to a Galaxy framework exception, save the traceback
+ job.flush()
+ log.debug( 'tool %s job id %d ended with errors' % ( job.tool_id, job.id ) )
+ trans.log_event( 'job id %d ended with errors' % job.id, tool_id=job.tool_id )
return dict( output=data )
def add_file( self, trans, temp_name, file_name, file_type, dbkey, info=None, space_to_tab=False ):
@@ -275,5 +301,20 @@
temp.close()
return False
+def nice_size( size ):
+ """Returns a readably formatted string with the size"""
+ words = [ 'bytes', 'Kb', 'Mb', 'Gb' ]
+ try:
+ size = float( size )
+ except:
+ return '??? bytes'
+ for ind, word in enumerate( words ):
+ step = 1024 ** ( ind + 1 )
+ if step > size:
+ size = size / float( 1024 ** ind )
+ out = "%.1f %s" % ( size, word )
+ return out
+ return '??? bytes'
+
class BadFileException( Exception ):
pass
1
0
05 Nov '08
details: http://www.bx.psu.edu/hg/galaxy/rev/7ae75e6d9d6a
changeset: 1600:7ae75e6d9d6a
user: Greg Von Kuster <greg(a)bx.psu.edu>
date: Mon Nov 03 15:31:38 2008 -0500
description:
Integrate "session_create_speed" and "faster_random" patches from James.
8 file(s) affected in this change:
lib/galaxy/model/mapping.py
lib/galaxy/tools/actions/__init__.py
lib/galaxy/web/controllers/root.py
lib/galaxy/web/controllers/tool_runner.py
lib/galaxy/web/controllers/user.py
lib/galaxy/web/framework/__init__.py
lib/galaxy/web/security/__init__.py
templates/root/history.mako
diffs (650 lines):
diff -r 13cbdd1bbd19 -r 7ae75e6d9d6a lib/galaxy/model/mapping.py
--- a/lib/galaxy/model/mapping.py Mon Nov 03 15:21:03 2008 -0500
+++ b/lib/galaxy/model/mapping.py Mon Nov 03 15:31:38 2008 -0500
@@ -326,6 +326,7 @@
assign_mapper( context, GalaxySession, GalaxySession.table,
properties=dict( histories=relation( GalaxySessionToHistoryAssociation ),
+ current_history=relation( History ),
user=relation( User.mapper ) ) )
assign_mapper( context, GalaxySessionToHistoryAssociation, GalaxySessionToHistoryAssociation.table,
diff -r 13cbdd1bbd19 -r 7ae75e6d9d6a lib/galaxy/tools/actions/__init__.py
--- a/lib/galaxy/tools/actions/__init__.py Mon Nov 03 15:21:03 2008 -0500
+++ b/lib/galaxy/tools/actions/__init__.py Mon Nov 03 15:31:38 2008 -0500
@@ -177,7 +177,7 @@
trans.app.model.flush()
# Create the job object
job = trans.app.model.Job()
- job.session_id = trans.get_galaxy_session( create=True ).id
+ job.session_id = trans.get_galaxy_session().id
job.history_id = trans.history.id
job.tool_id = tool.id
try:
diff -r 13cbdd1bbd19 -r 7ae75e6d9d6a lib/galaxy/web/controllers/root.py
--- a/lib/galaxy/web/controllers/root.py Mon Nov 03 15:21:03 2008 -0500
+++ b/lib/galaxy/web/controllers/root.py Mon Nov 03 15:31:38 2008 -0500
@@ -54,10 +54,7 @@
NOTE: No longer accepts "id" or "template" options for security reasons.
"""
- try:
- history = trans.get_history()
- except:
- return self.history_new(trans)
+ history = trans.get_history()
if as_xml:
trans.response.set_content_type('text/xml')
return trans.fill_template_mako( "root/history_as_xml.mako", history=history )
diff -r 13cbdd1bbd19 -r 7ae75e6d9d6a lib/galaxy/web/controllers/tool_runner.py
--- a/lib/galaxy/web/controllers/tool_runner.py Mon Nov 03 15:21:03 2008 -0500
+++ b/lib/galaxy/web/controllers/tool_runner.py Mon Nov 03 15:31:38 2008 -0500
@@ -41,7 +41,6 @@
return "Tool '%s' does not exist, kwd=%s " % (tool_id, kwd)
params = util.Params( kwd, sanitize=tool.options.sanitize, tool=tool )
history = trans.get_history()
- trans.ensure_valid_galaxy_session()
template, vars = tool.handle_input( trans, params.__dict__ )
if len(params) > 0:
trans.log_event( "Tool params: %s" % (str(params)), tool_id=tool_id )
diff -r 13cbdd1bbd19 -r 7ae75e6d9d6a lib/galaxy/web/controllers/user.py
--- a/lib/galaxy/web/controllers/user.py Mon Nov 03 15:21:03 2008 -0500
+++ b/lib/galaxy/web/controllers/user.py Mon Nov 03 15:31:38 2008 -0500
@@ -82,10 +82,7 @@
elif not user.check_password( password ):
password_error = "Invalid password"
else:
- trans.set_user( user )
- trans.ensure_valid_galaxy_session()
- # Associate user with galaxy_session and history
- trans.make_associations()
+ trans.handle_user_login( user )
trans.log_event( "User logged in" )
return trans.show_ok_message( "Now logged in as " + user.email, refresh_frames=['masthead', 'history'] )
return trans.show_form(
@@ -97,7 +94,7 @@
def logout( self, trans ):
# Since logging an event requires a session, we'll log prior to ending the session
trans.log_event( "User logged out" )
- new_galaxy_session = trans.logout_galaxy_session()
+ trans.handle_user_logout()
return trans.show_ok_message( "You are no longer logged in", refresh_frames=['masthead', 'history'] )
@web.expose
@@ -118,12 +115,7 @@
user = trans.app.model.User( email=email )
user.set_password_cleartext( password )
user.flush()
- trans.set_user( user )
- trans.ensure_valid_galaxy_session()
- """
- Associate user with galaxy_session and history
- """
- trans.make_associations()
+ trans.handle_user_login( user )
trans.log_event( "User created a new account" )
trans.log_event( "User logged in" )
#subscribe user to email list
diff -r 13cbdd1bbd19 -r 7ae75e6d9d6a lib/galaxy/web/framework/__init__.py
--- a/lib/galaxy/web/framework/__init__.py Mon Nov 03 15:21:03 2008 -0500
+++ b/lib/galaxy/web/framework/__init__.py Mon Nov 03 15:31:38 2008 -0500
@@ -109,12 +109,14 @@
self.__history = NOT_SET
self.__galaxy_session = NOT_SET
base.DefaultWebTransaction.__init__( self, environ )
- self.app.model.context.current.clear()
+ self.sa_session.clear()
self.debug = asbool( self.app.config.get( 'debug', False ) )
# Flag indicating whether we are in workflow building mode (means
# that the current history should not be used for parameter values
# and such).
self.workflow_building_mode = False
+ # Always have a valid galaxy session
+ self.__ensure_valid_session()
@property
def sa_session( self ):
"""
@@ -140,7 +142,6 @@
except:
event.history_id = None
event.user = self.user
- self.ensure_valid_galaxy_session()
event.session_id = self.galaxy_session.id
event.flush()
def get_cookie( self, name='galaxysession' ):
@@ -163,277 +164,208 @@
tstamp = time.localtime ( time.time() + 3600 * 24 * age )
self.response.cookies[name]['expires'] = time.strftime( '%a, %d-%b-%Y %H:%M:%S GMT', tstamp )
self.response.cookies[name]['version'] = version
+ #@property
+ #def galaxy_session( self ):
+ # if not self.__galaxy_session:
+ # self.__ensure_valid_session()
+ # return self.__galaxy_session
+ def __ensure_valid_session( self ):
+ """
+ Ensure that a valid Galaxy session exists and is available as
+ trans.session (part of initialization)
+
+ Support for universe_session and universe_user cookies has been
+ removed as of 31 Oct 2008.
+ """
+ sa_session = self.sa_session
+ # Try to load an existing session
+ secure_id = self.get_cookie( name='galaxysession' )
+ galaxy_session = None
+ prev_galaxy_session = None
+ user_for_new_session = None
+ invalidate_existing_session = False
+ # Track whether the session has changed so we can avoid calling flush
+ # in the most common case (session exists and is valid).
+ galaxy_session_requires_flush = False
+ if secure_id:
+ # Decode the cookie value to get the session_key
+ session_key = self.security.decode_session_key( secure_id )
+ # Retrive the galaxy_session id via the unique session_key
+ galaxy_session = sa_session.query( self.app.model.GalaxySession ).filter_by( session_key=session_key, is_valid=True ).first()
+ # If remote user is in use it can invalidate the session, so we need to
+ # to check some things now.
+ if self.app.config.use_remote_user:
+ assert "HTTP_REMOTE_USER" in self.environ, \
+ "use_remote_user is set but no HTTP_REMOTE_USER variable"
+ remote_user_email = self.environ[ 'HTTP_REMOTE_USER' ]
+ if galaxy_session:
+ # An existing session, make sure correct association exists
+ if galaxy_session.user is None:
+ # No user, associate
+ galaxy_session.user = self.__get_or_create_remote_user( remote_user_email )
+ galaxy_session_requires_flush = True
+ elif galaxy_session.user.email != remote_user_email:
+ # Session exists but is not associated with the correct
+ # remote user
+ invalidate_existing_session = True
+ user_for_new_session = self.__get_or_create_remote_user( remote_user_email )
+ log.warning( "User logged in as '%s' externally, but has a cookie as '%s' invalidating session",
+ remote_user_email, prev_galaxy_session.user.email )
+ else:
+ if galaxy_session is not None and galaxy_session.user and galaxy_session.user.external:
+ # Remote user support is not enabled, but there is an existing
+ # session with an external user, invalidate
+ invalidate_existing_session = True
+ log.warning( "User '%s' is an external user with an existing session, invalidating session since external auth is disabled",
+ galaxy_session.user.email )
+ # Do we need to invalidate the session for some reason?
+ if invalidate_existing_session:
+ prev_galaxy_session = galaxy_session
+ prev_galaxy_session.is_valid = False
+ galaxy_session = None
+ # No relevant cookies, or couldn't find, or invalid, so create a new session
+ if galaxy_session is None:
+ galaxy_session = self.__create_new_session( prev_galaxy_session, user_for_new_session )
+ galaxy_session_requires_flush = True
+ self.galaxy_session = galaxy_session
+ self.__update_session_cookie()
+ else:
+ self.galaxy_session = galaxy_session
+ # Do we need to flush the session?
+ if galaxy_session_requires_flush:
+ objects_to_flush = [ galaxy_session ]
+ # FIXME: If prev_session is a proper relation this would not
+ # be needed.
+ if prev_galaxy_session:
+ objects_to_flush.append( prev_galaxy_session )
+ sa_session.flush( objects_to_flush )
+ def __create_new_session( self, prev_galaxy_session=None, user_for_new_session=None ):
+ """
+ Create a new GalaxySession for this request, possibly with a connection
+ to a previous session (in `prev_galaxy_session`) and an existing user
+ (in `user_for_new_session`).
+
+ Caller is responsible for flushing the returned session.
+ """
+ session_key = self.security.get_new_session_key()
+ galaxy_session = self.app.model.GalaxySession(
+ session_key=session_key,
+ is_valid=True,
+ remote_host = self.request.remote_host,
+ remote_addr = self.request.remote_addr,
+ referer = self.request.headers.get( 'Referer', None ) )
+ # Invalidated an existing sesssion for some reason, keep track
+ if prev_galaxy_session:
+ galaxy_session.prev_session_id = prev_galaxy_session.id
+ # The new session should be immediately associated with a user
+ if user_for_new_session:
+ galaxy_session.user = user_for_new_session
+ return galaxy_session
+ def __get_or_create_remote_user( self, remote_user_email ):
+ """
+ Return the user in $HTTP_REMOTE_USER and create if necessary
+
+ Caller is responsible for flushing the returned user.
+ """
+ # remote_user middleware ensures HTTP_REMOTE_USER exists
+ user = self.app.model.User.filter_by( email=remote_user_email ).first()
+ if user is None:
+ user = self.app.model.User( email=remote_user_email )
+ user.set_password_cleartext( 'external' )
+ user.external = True
+ self.log_event( "Automatically created account '%s'", user.email )
+ return user
+ def __update_session_cookie( self ):
+ """
+ Update the 'galaxysession' cookie to match the current session.
+ """
+ self.set_cookie( name='galaxysession',
+ value=self.security.encode_session_key( self.galaxy_session.session_key ) )
+
+ def handle_user_login( self, user ):
+ """
+ Login a new user (possibly newly created)
+ - create a new session
+ - associate new session with user
+ - if old session had a history and it was not associated with a user, associate it with the new session.
+ """
+ prev_galaxy_session = self.galaxy_session
+ prev_galaxy_session.is_valid = False
+ self.galaxy_session = self.__create_new_session( prev_galaxy_session, user )
+ if prev_galaxy_session.current_history:
+ history = prev_galaxy_session.current_history
+ if history.user is None:
+ self.galaxy_session.add_history( history )
+ self.galaxy_session.current_history = history
+ history.user = user
+ self.sa_session.flush( [ prev_galaxy_session, self.galaxy_session, history ] )
+ else:
+ self.sa_session.flush( [ prev_galaxy_session, self.galaxy_session ] )
+ self.__update_session_cookie()
+ def handle_user_logout( self ):
+ """
+ Logout the current user:
+ - invalidate the current session
+ - create a new session with no user associated
+ """
+ prev_galaxy_session = self.galaxy_session
+ prev_galaxy_session.is_valid = False
+ self.galaxy_session = self.__create_new_session( prev_galaxy_session, None )
+ self.sa_session.flush( [ prev_galaxy_session, self.galaxy_session ] )
+ self.__update_session_cookie()
+
+ def get_galaxy_session( self ):
+ """
+ Return the current galaxy session
+ """
+ return self.galaxy_session
+
def get_history( self, create=False ):
- """Load the current history"""
- if self.__history is NOT_SET:
- self.__history = None
- # See if we have a galaxysession cookie
- secure_id = self.get_cookie( name='galaxysession' )
- if secure_id:
- session_key = self.security.decode_session_key( secure_id )
- try:
- galaxy_session = self.app.model.GalaxySession.filter_by( session_key=session_key ).first()
- if galaxy_session and galaxy_session.is_valid and galaxy_session.current_history_id:
- history = self.app.model.History.get( galaxy_session.current_history_id )
- if history and not history.deleted:
- self.__history = history
- except Exception, e:
- # This should only occur in development if the cookie is not synced with the db
- pass
- else:
- # See if we have a deprecated universe cookie
- # TODO: this should be eliminated some time after October 1, 2008
- # We'll keep it until then because the old universe cookies are valid for 90 days
- history_id = self.get_cookie( name='universe' )
- if history_id:
- history = self.app.model.History.get( int( history_id ) )
- if history and not history.deleted:
- self.__history = history
- # Expire the universe cookie since it is deprecated
- self.set_cookie( name='universe', value=id, age=0 )
- if self.__history is None:
- return self.new_history()
- if create is True and self.__history is None:
- return self.new_history()
- return self.__history
+ """
+ Load the current history.
+
+ NOTE: It looks like create was being ignored for a long time, so this
+ will currently *always* create a new history. This is wasteful
+ though, and we should verify that callers are using the create
+ flag correctly and fix.
+ """
+ history = self.galaxy_session.current_history
+ if history is None:
+ history = self.new_history()
+ return history
+ def set_history( self, history ):
+ if history and not history.deleted:
+ self.galaxy_session.current_history = history
+ self.sa_session.flush( [ self.galaxy_session ] )
+ history = property( get_history, set_history )
def new_history( self ):
+ """
+ Create a new history and associate it with the current session and
+ its associated user (if set).
+ """
+ # Create new history
history = self.app.model.History()
- # Make sure we have an id
- history.flush()
- # Immediately associate the new history with self
- self.__history = history
- # Make sure we have a valid session to associate with the new history
- if self.galaxy_session_is_valid():
- galaxy_session = self.get_galaxy_session()
- else:
- galaxy_session = self.new_galaxy_session()
- # We are associating the last used genome_build with histories, so we will always
- # initialize a new history with the first dbkey in util.dbnames which is currently
- # ? unspecified (?)
+ # Associate with session
+ history.add_galaxy_session( self.galaxy_session )
+ # Make it the session's current history
+ self.galaxy_session.current_history = history
+ # Associate with user
+ if self.galaxy_session.user:
+ history.user = self.galaxy_session.user
+ # Track genome_build with history
history.genome_build = util.dbnames.default_value
- if self.user:
- history.user_id = self.user.id
- galaxy_session.user_id = self.user.id
- try:
- # See if we have already associated the history with the session
- association = self.app.model.GalaxySessionToHistoryAssociation.filter_by( session_id=galaxy_session.id, history_id=history.id ).first()
- except:
- association = None
- history.add_galaxy_session( galaxy_session, association=association )
- history.flush()
- galaxy_session.current_history_id = history.id
- galaxy_session.flush()
- self.__history = history
- return self.__history
- def set_history( self, history ):
- if history and not history.deleted and self.galaxy_session_is_valid():
- galaxy_session = self.get_galaxy_session()
- galaxy_session.current_history_id = history.id
- galaxy_session.flush()
- self.__history = history
- history = property( get_history, set_history )
+ # Save
+ self.sa_session.flush( [ self.galaxy_session, history ] )
+ return history
+
def get_user( self ):
"""Return the current user if logged in or None."""
- if self.__user is NOT_SET:
- self.__user = None
- user = self.get_cookie_user()
- if self.app.config.use_remote_user:
- remote_user = self.get_remote_user()
- if user is not None and user.email != remote_user.email:
- # The user has a cookie for a different user than the one
- # they've authed as.
- log.warning( "User logged in as '%s' externally, but has a cookie as '%s',"
- % ( remote_user.email, user.email ) + " invalidating session" )
- self.logout_galaxy_session()
- if user is None or ( user is not None and user.email != remote_user.email ):
- # The user has no cookie, was not logged in, or the above.
- self.set_user( remote_user )
- self.make_associations()
- self.__user = remote_user
- else:
- if user is not None and user.external:
- # use_remote_user is off, but the user still has a cookie
- # from when it was on. Force the user to get a new
- # unauthenticated session.
- log.warning( "User '%s' is an external user with an existing " % user.email
- + "session, invalidating session since external auth is disabled" )
- self.logout_galaxy_session()
- else:
- self.__user = user
- return self.__user
- def get_remote_user( self ):
- """Return the user in $HTTP_REMOTE_USER and create if necessary"""
- # remote_user middleware ensures HTTP_REMOTE_USER exists
- try:
- user = self.app.model.User.filter_by( email=self.environ[ 'HTTP_REMOTE_USER' ] ).first()
- except:
- user = self.app.model.User( email=self.environ[ 'HTTP_REMOTE_USER' ] )
- user.set_password_cleartext( 'external' )
- user.external = True
- user.flush()
- self.log_event( "Automatically created account '%s'" % user.email )
- return user
- def get_cookie_user( self ):
- """Return the user in the galaxysession cookie"""
- __user = None
- # See if we have a galaxysession cookie
- secure_id = self.get_cookie( name='galaxysession' )
- if secure_id:
- session_key = self.security.decode_session_key( secure_id )
- try:
- galaxy_session = self.app.model.GalaxySession.filter_by( session_key=session_key ).first()
- if galaxy_session and galaxy_session.is_valid and galaxy_session.user_id:
- user = self.app.model.User.get( galaxy_session.user_id )
- if user:
- __user = user
- except:
- # This should only occur in development if the cookie is not synced with the db
- pass
- else:
- # See if we have a deprecated universe_user cookie
- # TODO: this should be eliminated some time after October 1, 2008
- # We'll keep it until then because the old universe cookies are valid for 90 days
- user_id = self.get_cookie( name='universe_user' )
- if user_id:
- user = self.app.model.User.get( int( user_id ) )
- if user:
- __user = user
- # Expire the universe_user cookie since it is deprecated
- self.set_cookie( name='universe_user', value='', age=0 )
- return __user
+ return self.galaxy_session.user
def set_user( self, user ):
- """Set the current user if logged in."""
- if user is not None and self.galaxy_session_is_valid():
- galaxy_session = self.get_galaxy_session()
- if galaxy_session.user_id != user.id:
- galaxy_session.user_id = user.id
- galaxy_session.flush()
- self.__user = user
+ """Set the current user."""
+ self.galaxy_session.user = user
+ self.sa_session.flush( [ self.galaxy_session ] )
user = property( get_user, set_user )
- def get_galaxy_session( self, create=False ):
- """Return the current user's GalaxySession"""
- if self.__galaxy_session is NOT_SET:
- self.__galaxy_session = None
- # See if we have a galaxysession cookie
- secure_id = self.get_cookie( name='galaxysession' )
- if secure_id:
- # Decode the cookie value to get the session_key
- session_key = self.security.decode_session_key( secure_id )
- try:
- # Retrive the galaxy_session id via the unique session_key
- galaxy_session = self.app.model.GalaxySession.filter_by( session_key=session_key ).first()
- if galaxy_session and galaxy_session.is_valid:
- self.__galaxy_session = galaxy_session
- except:
- # This should only occur in development if the cookie is not synced with the db
- pass
- else:
- # See if we have a deprecated universe_session cookie
- # TODO: this should be eliminated some time after October 1, 2008
- # We'll keep it until then because the old universe cookies are valid for 90 days
- session_id = self.get_cookie( name='universe_session' )
- if session_id:
- galaxy_session = self.app.model.GalaxySession.get( int( session_id ) )
- # NOTE: We can't test for is_valid here since the old session records did not include this flag
- if galaxy_session:
- # Set the new galaxysession cookie value, old session records did not have a session_key or is_valid flag
- session_key = self.security.get_new_session_key()
- galaxy_session.session_key = session_key
- galaxy_session.is_valid = True
- galaxy_session.flush()
- secure_id = self.security.encode_session_key( session_key )
- self.set_cookie( name='galaxysession', value=secure_id )
- # Expire the universe_user cookie since it is deprecated
- self.set_cookie( name='universe_session', value='', age=0 )
- self.__galaxy_session = galaxy_session
- if create is True and self.__galaxy_session is None:
- return self.new_galaxy_session()
- return self.__galaxy_session
- def new_galaxy_session( self, prev_session_id=None ):
- """Create a new secure galaxy_session"""
- session_key = self.security.get_new_session_key()
- galaxy_session = self.app.model.GalaxySession( session_key=session_key, is_valid=True, prev_session_id=prev_session_id )
- # Make sure we have an id
- galaxy_session.flush()
- # Immediately associate the new session with self
- self.__galaxy_session = galaxy_session
- if prev_session_id is not None:
- # User logged out, so we need to create a new history for this session
- self.history = self.new_history()
- galaxy_session.current_history_id = self.history.id
- elif self.user is not None:
- galaxy_session.user_id = self.user.id
- # Set this session's current_history_id to the user's last updated history
- h = self.app.model.History
- ht = h.table
- where = ( ht.c.user_id==self.user.id ) & ( ht.c.deleted=='f' )
- history = h.query().filter( where ).order_by( desc( ht.c.update_time ) ).first()
- if history:
- self.history = history
- galaxy_session.current_history_id = self.history.id
- elif self.history:
- galaxy_session.current_history_id = self.history.id
- galaxy_session.remote_host = self.request.remote_host
- galaxy_session.remote_addr = self.request.remote_addr
- try:
- galaxy_session.referer = self.request.headers['Referer']
- except:
- galaxy_session.referer = None
- if self.history is not None:
- # See if we have already associated the session with the history
- try:
- association = self.app.model.GalaxySessionToHistoryAssociation.filter_by( session_id=galaxy_session.id, history_id=self.history.id ).first()
- except:
- association = None
- galaxy_session.add_history( self.history, association=association )
- galaxy_session.flush()
- # Set the cookie value to the encrypted session_key
- self.set_cookie( name='galaxysession', value=self.security.encode_session_key( session_key ) )
- self.__galaxy_session = galaxy_session
- return self.__galaxy_session
- def set_galaxy_session( self, galaxy_session ):
- """Set the current galaxy_session"""
- self.__galaxy_session = galaxy_session
- galaxy_session = property( get_galaxy_session, set_galaxy_session )
- def galaxy_session_is_valid( self ):
- try:
- return self.galaxy_session.is_valid
- except:
- return False
- def ensure_valid_galaxy_session( self ):
- """Make sure we have a valid galaxy session, create a new one if necessary."""
- if not self.galaxy_session_is_valid():
- galaxy_session = self.new_galaxy_session()
- def logout_galaxy_session( self ):
- """
- Logout the current user by setting user to None and galaxy_session.is_valid to False
- in the db. A new galaxy_session is automatically created with prev_session_id is set
- to save a reference to the current one as a way of chaining them together
- """
- if self.galaxy_session_is_valid():
- galaxy_session = self.get_galaxy_session()
- old_session_id = galaxy_session.id
- galaxy_session.is_valid = False
- galaxy_session.flush()
- self.set_user( None )
- return self.new_galaxy_session( prev_session_id=old_session_id )
- else:
- error( "Attempted to logout an invalid galaxy_session" )
- def make_associations( self ):
- history = self.get_history()
- user = self.get_user()
- if self.galaxy_session_is_valid():
- galaxy_session = self.get_galaxy_session()
- if galaxy_session.user_id is None and user is not None:
- galaxy_session.user_id = user.id
- if history is not None:
- galaxy_session.current_history_id = history.id
- galaxy_session.flush()
- self.__galaxy_session = galaxy_session
- if history is not None and user is not None:
- history.user_id = user.id
- history.flush()
- self.__history = history
def get_toolbox(self):
"""Returns the application toolbox"""
diff -r 13cbdd1bbd19 -r 7ae75e6d9d6a lib/galaxy/web/security/__init__.py
--- a/lib/galaxy/web/security/__init__.py Mon Nov 03 15:21:03 2008 -0500
+++ b/lib/galaxy/web/security/__init__.py Mon Nov 03 15:31:38 2008 -0500
@@ -1,9 +1,35 @@
+import os, os.path, logging
+
import pkg_resources
pkg_resources.require( "pycrypto" )
from Crypto.Cipher import Blowfish
from Crypto.Util.randpool import RandomPool
from Crypto.Util import number
+
+log = logging.getLogger( __name__ )
+
+if os.path.exists( "/dev/urandom" ):
+ log.debug("###using /dev/urandom....")
+ # We have urandom, use it as the source of random data
+ random_fd = os.open( "/dev/urandom", os.O_RDONLY )
+ def get_random_bytes( nbytes ):
+ value = os.read( random_fd, nbytes )
+ # Normally we should get as much as we need
+ if len( value ) == nbytes:
+ return value
+ # If we don't, keep reading (this is slow and should never happen)
+ while len( value ) < nbytes:
+ value += os.read( random_fd, nbytes - len( value ) )
+ return value
+else:
+ def get_random_bytes( nbytes ):
+ nbits = nbytes * 8
+ random_pool = RandomPool( 1064 )
+ while random_pool.entropy < nbits:
+ random_pool.add_event()
+ random_pool.stir()
+ return( str( number.getRandomNumber( nbits, random_pool.get_bytes ) ) )
class SecurityHelper( object ):
# TODO: checking if histories/datasets are owned by the current user) will be moved here.
@@ -30,11 +56,5 @@
return self.id_cipher.decrypt( session_key.decode( 'hex' ) ).lstrip( "!" )
def get_new_session_key( self ):
# Generate a unique, high entropy 128 bit random number
- random_pool = RandomPool( 1064 )
- while random_pool.entropy < 128:
- random_pool.add_event()
- random_pool.stir()
- rn = number.getRandomNumber( 128, random_pool.get_bytes )
- # session_key must be a string
- return str( rn )
+ return get_random_bytes( 16 )
\ No newline at end of file
diff -r 13cbdd1bbd19 -r 7ae75e6d9d6a templates/root/history.mako
--- a/templates/root/history.mako Mon Nov 03 15:21:03 2008 -0500
+++ b/templates/root/history.mako Mon Nov 03 15:31:38 2008 -0500
@@ -231,6 +231,7 @@
<body class="historyPage">
+
<div id="top-links" class="historyLinks">
<a href="${h.url_for('history', show_deleted=show_deleted)}">refresh</a>
%if show_deleted:
1
0
05 Nov '08
details: http://www.bx.psu.edu/hg/galaxy/rev/13cbdd1bbd19
changeset: 1599:13cbdd1bbd19
user: Dan Blankenberg <dan(a)bx.psu.edu>
date: Mon Nov 03 15:21:03 2008 -0500
description:
A fix for the metadata reverting issue seen with PBS job runner.
1 file(s) affected in this change:
lib/galaxy/jobs/__init__.py
diffs (18 lines):
diff -r ca640ce1abcf -r 13cbdd1bbd19 lib/galaxy/jobs/__init__.py
--- a/lib/galaxy/jobs/__init__.py Mon Nov 03 15:15:22 2008 -0500
+++ b/lib/galaxy/jobs/__init__.py Mon Nov 03 15:21:03 2008 -0500
@@ -274,6 +274,7 @@
Prepare the job to run by creating the working directory and the
config files.
"""
+ mapping.context.current.clear() #this prevents the metadata reverting that has been seen in conjunction with the PBS job runner
# Create the working directory
self.working_directory = \
os.path.join( self.app.config.job_working_directory, str( self.job_id ) )
@@ -643,4 +644,4 @@
def put( self, *args ):
return
def shutdown( self ):
- return
\ No newline at end of file
+ return
1
0