# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Greg Von Kuster <greg@bx.psu.edu> # Date 1275672554 14400 # Node ID a76d05bb41e1c1f1eeaa93a18289eeed1213199b # Parent dd69c382eff1d3c0c160bc3ec392b06671317fd0 Fixes and enhancements to the reports webapp: 0) The report that is initially displayed is now the number of jobs per day for the current month rather than the grid displaying all jobs for today. 1) Eliminate eagerloading when querying jobs so that queries are much faster 2) Fix the disk space ( system ) report, and add a new feature that displays all active ( undeleted ) hda and ldda pointers to a specific dataset. 3) Standardize request parameters so that queries can more easily be built using parameters from the request. 4) A few other miscellaneous bug fixes in the reports. --- a/templates/webapps/reports/registered_users.mako +++ b/templates/webapps/reports/registered_users.mako @@ -2,7 +2,7 @@ <%namespace file="/message.mako" import="render_msg" /> %if message: - ${render_msg( message, status )} + ${render_msg( message, 'done' )} %endif <div class="toolForm"> --- a/templates/webapps/reports/jobs_user_per_month.mako +++ b/templates/webapps/reports/jobs_user_per_month.mako @@ -4,7 +4,7 @@ <% from galaxy import util %> %if message: - ${render_msg( message, status )} + ${render_msg( message, 'done' )} %endif <div class="toolForm"> @@ -27,7 +27,7 @@ <tr class="tr"> %endif <td>${job[2]} ${job[3]}</td> - <td><a href="${h.url_for( controller='jobs', action='user_for_month', email=email, month=job[0] )}">${job[1]}</a></td> + <td><a href="${h.url_for( controller='jobs', action='user_for_month', email=email, specified_date=job[0]+'-01' )}">${job[1]}</a></td></tr><% ctr += 1 %> %endfor --- a/lib/galaxy/webapps/reports/controllers/jobs.py +++ b/lib/galaxy/webapps/reports/controllers/jobs.py @@ -63,7 +63,7 @@ class SpecifiedDateListGrid( grids.Grid model_class=model.User, link=( lambda item: dict( operation="user_per_month", id=item.id, webapp="reports" ) ), attach_popup=False ), - grids.StateColumn( "All", key="state", model_class=model.Job, visible=False, filterable="advanced" ) + grids.StateColumn( "state", key="state", model_class=model.Job, visible=False, filterable="advanced" ) ] columns.append( grids.MulticolFilterColumn( "Search", cols_to_filter=[ columns[1], columns[2] ], @@ -78,45 +78,59 @@ class SpecifiedDateListGrid( grids.Grid def build_initial_query( self, trans, **kwd ): specified_date = kwd.get( 'specified_date', 'All' ) if specified_date == 'All': - return trans.sa_session.query( self.model_class ) + return trans.sa_session.query( self.model_class ) \ + .enable_eagerloads( False ) year, month, day = map( int, specified_date.split( "-" ) ) start_date = date( year, month, day ) end_date = start_date + timedelta( days=1 ) - return trans.sa_session.query( self.model_class ).filter( and_( self.model_class.table.c.create_time >= start_date, - self.model_class.table.c.create_time < end_date ) ) + return trans.sa_session.query( self.model_class ) \ + .filter( and_( self.model_class.table.c.create_time >= start_date, + self.model_class.table.c.create_time < end_date ) ) \ + .enable_eagerloads( False ) class SpecifiedDateInErrorListGrid( SpecifiedDateListGrid ): def build_initial_query( self, trans, **kwd ): specified_date = kwd.get( 'specified_date', 'All' ) if specified_date == 'All': - return trans.sa_session.query( self.model_class ).filter( self.model_class.table.c.state == model.Job.states.ERROR ) + return trans.sa_session.query( self.model_class ) \ + .filter( self.model_class.table.c.state == model.Job.states.ERROR ) \ + .enable_eagerloads( False ) year, month, day = map( int, specified_date.split( "-" ) ) start_date = date( year, month, day ) end_date = start_date + timedelta( days=1 ) - return trans.sa_session.query( self.model_class ).filter( and_( self.model_class.table.c.state == model.Job.states.ERROR, - self.model_class.table.c.create_time >= start_date, - self.model_class.table.c.create_time < end_date ) ) + return trans.sa_session.query( self.model_class ) \ + .filter( and_( self.model_class.table.c.state == model.Job.states.ERROR, + self.model_class.table.c.create_time >= start_date, + self.model_class.table.c.create_time < end_date ) ) \ + .enable_eagerloads( False ) class AllUnfinishedListGrid( SpecifiedDateListGrid ): def build_initial_query( self, trans, **kwd ): specified_date = kwd.get( 'specified_date', 'All' ) if specified_date == 'All': - return trans.sa_session.query( self.model_class ).filter( not_( or_( model.Job.table.c.state == model.Job.states.OK, - model.Job.table.c.state == model.Job.states.ERROR, - model.Job.table.c.state == model.Job.states.DELETED ) ) ) + return trans.sa_session.query( self.model_class ) \ + .filter( not_( or_( model.Job.table.c.state == model.Job.states.OK, + model.Job.table.c.state == model.Job.states.ERROR, + model.Job.table.c.state == model.Job.states.DELETED ) ) ) \ + .enable_eagerloads( False ) year, month, day = map( int, specified_date.split( "-" ) ) start_date = date( year, month, day ) end_date = start_date + timedelta( days=1 ) - return trans.sa_session.query( self.model_class ).filter( and_( not_( or_( model.Job.table.c.state == model.Job.states.OK, - model.Job.table.c.state == model.Job.states.ERROR, - model.Job.table.c.state == model.Job.states.DELETED ) ), - self.model_class.table.c.create_time >= start_date, - self.model_class.table.c.create_time < end_date ) ) + return trans.sa_session.query( self.model_class ) \ + .filter( and_( not_( or_( model.Job.table.c.state == model.Job.states.OK, + model.Job.table.c.state == model.Job.states.ERROR, + model.Job.table.c.state == model.Job.states.DELETED ) ), + self.model_class.table.c.create_time >= start_date, + self.model_class.table.c.create_time < end_date ) ) \ + .enable_eagerloads( False ) class UserForMonthListGrid( SpecifiedDateListGrid ): def build_initial_query( self, trans, **kwd ): email = util.restore_text( kwd.get( 'email', '' ) ) - year, month = map( int, kwd.get( 'month', datetime.utcnow().strftime( "%Y-%m" ) ).split( "-" ) ) + # If specified_date is not received, we'll default to the current month + specified_date = kwd.get( 'specified_date', datetime.utcnow().strftime( "%Y-%m-%d" ) ) + specified_month = specified_date[ :7 ] + year, month = map( int, specified_month.split( "-" ) ) start_date = date( year, month, 1 ) end_date = start_date + timedelta( days=calendar.monthrange( year, month )[1] ) return trans.sa_session.query( model.Job ) \ @@ -127,19 +141,22 @@ class UserForMonthListGrid( SpecifiedDat model.User.table.c.email == email, model.Job.table.c.create_time >= start_date, model.Job.table.c.create_time < end_date ) ) \ - .order_by( desc( model.Job.table.c.create_time ) ) + .enable_eagerloads( False ) class ToolForMonthListGrid( SpecifiedDateListGrid ): def build_initial_query( self, trans, **kwd ): + # If specified_date is not received, we'll default to the current month + specified_date = kwd.get( 'specified_date', datetime.utcnow().strftime( "%Y-%m-%d" ) ) + specified_month = specified_date[ :7 ] tool_id = util.restore_text( kwd.get( 'tool_id', '' ) ) - year, month = map( int, kwd.get( 'month', datetime.utcnow().strftime( "%Y-%m" ) ).split( "-" ) ) + year, month = map( int, specified_month.split( "-" ) ) start_date = date( year, month, 1 ) end_date = start_date + timedelta( days=calendar.monthrange( year, month )[1] ) - return trans.sa_session.query( model.Job ) \ - .filter( and_( model.Job.table.c.tool_id == tool_id, - model.Job.table.c.create_time >= start_date, - model.Job.table.c.create_time < end_date ) ) \ - .order_by( desc( model.Job.table.c.create_time ) ) + return trans.sa_session.query( self.model_class ) \ + .filter( and_( self.model_class.table.c.tool_id == tool_id, + self.model_class.table.c.create_time >= start_date, + self.model_class.table.c.create_time < end_date ) ) \ + .enable_eagerloads( False ) class Jobs( BaseController ): @@ -158,8 +175,7 @@ class Jobs( BaseController ): action='job_info', **kwd ) ) if operation == "tool_per_month": - # The received id is the job id, so we need to get the id of the user - # that submitted the job. + # The received id is the job id, so we need to get the jobs tool_id. job_id = kwd.get( 'id', None ) job = get_job( trans, job_id ) kwd[ 'tool_id' ] = job.tool_id @@ -194,8 +210,7 @@ class Jobs( BaseController ): action='job_info', **kwd ) ) if operation == "tool_per_month": - # The received id is the job id, so we need to get the id of the user - # that submitted the job. + # The received id is the job id, so we need to get the jobs tool_id. job_id = kwd.get( 'id', None ) job = get_job( trans, job_id ) kwd[ 'tool_id' ] = job.tool_id @@ -222,7 +237,10 @@ class Jobs( BaseController ): params = util.Params( kwd ) message = '' monitor_email = params.get( 'monitor_email', 'monitor@bx.psu.edu' ) - year, month = map( int, params.get( 'month', datetime.utcnow().strftime( "%Y-%m" ) ).split( "-" ) ) + # If specified_date is not received, we'll default to the current month + specified_date = kwd.get( 'specified_date', datetime.utcnow().strftime( "%Y-%m-%d" ) ) + specified_month = specified_date[ :7 ] + year, month = map( int, specified_month.split( "-" ) ) start_date = date( year, month, 1 ) end_date = start_date + timedelta( days=calendar.monthrange( year, month )[1] ) month_label = start_date.strftime( "%B" ) @@ -254,7 +272,10 @@ class Jobs( BaseController ): def specified_month_in_error( self, trans, **kwd ): params = util.Params( kwd ) message = '' - year, month = map( int, params.get( 'month', datetime.utcnow().strftime( "%Y-%m" ) ).split( "-" ) ) + # If specified_date is not received, we'll default to the current month + specified_date = kwd.get( 'specified_date', datetime.utcnow().strftime( "%Y-%m-%d" ) ) + specified_month = specified_date[ :7 ] + year, month = map( int, specified_month.split( "-" ) ) start_date = date( year, month, 1 ) end_date = start_date + timedelta( days=calendar.monthrange( year, month )[1] ) month_label = start_date.strftime( "%B" ) @@ -289,8 +310,7 @@ class Jobs( BaseController ): action='job_info', **kwd ) ) if operation == "tool_per_month": - # The received id is the job id, so we need to get the id of the user - # that submitted the job. + # The received id is the job id, so we need to get the jobs tool_id. job_id = kwd.get( 'id', None ) job = get_job( trans, job_id ) kwd[ 'tool_id' ] = job.tool_id @@ -332,7 +352,9 @@ class Jobs( BaseController ): row.total_jobs, row.date.strftime( "%B" ), row.date.strftime( "%Y" ) ) ) - return trans.fill_template( '/webapps/reports/jobs_per_month_all.mako', jobs=jobs, message=message ) + return trans.fill_template( '/webapps/reports/jobs_per_month_all.mako', + jobs=jobs, + message=message ) @web.expose def per_month_in_error( self, trans, **kwd ): params = util.Params( kwd ) @@ -349,7 +371,9 @@ class Jobs( BaseController ): row.total_jobs, row.date.strftime( "%B" ), row.date.strftime( "%Y" ) ) ) - return trans.fill_template( '/webapps/reports/jobs_per_month_in_error.mako', jobs=jobs, message=message ) + return trans.fill_template( '/webapps/reports/jobs_per_month_in_error.mako', + jobs=jobs, + message=message ) @web.expose def per_user( self, trans, **kwd ): params = util.Params( kwd ) @@ -398,8 +422,7 @@ class Jobs( BaseController ): action='job_info', **kwd ) ) if operation == "tool_per_month": - # The received id is the job id, so we need to get the id of the user - # that submitted the job. + # The received id is the job id, so we need to get the jobs tool_id. job_id = kwd.get( 'id', None ) job = get_job( trans, job_id ) kwd[ 'tool_id' ] = job.tool_id @@ -434,12 +457,15 @@ class Jobs( BaseController ): for row in q.execute(): jobs.append( ( row.tool_id, row.total_jobs ) ) - return trans.fill_template( '/webapps/reports/jobs_per_tool.mako', jobs=jobs, message=message ) + return trans.fill_template( '/webapps/reports/jobs_per_tool.mako', + jobs=jobs, + message=message ) @web.expose def tool_per_month( self, trans, **kwd ): params = util.Params( kwd ) message = '' tool_id = params.get( 'tool_id', 'Add a column1' ) + specified_date = params.get( 'specified_date', datetime.utcnow().strftime( "%Y-%m-%d" ) ) q = sa.select( ( sa.func.date_trunc( 'month', sa.func.date( model.Job.table.c.create_time ) ).label( 'date' ), sa.func.count( model.Job.table.c.id ).label( 'total_jobs' ) ), whereclause = model.Job.table.c.tool_id == tool_id, @@ -452,7 +478,11 @@ class Jobs( BaseController ): row.total_jobs, row.date.strftime( "%B" ), row.date.strftime( "%Y" ) ) ) - return trans.fill_template( '/webapps/reports/jobs_tool_per_month.mako', tool_id=tool_id, jobs=jobs, message=message ) + return trans.fill_template( '/webapps/reports/jobs_tool_per_month.mako', + specified_date=specified_date, + tool_id=tool_id, + jobs=jobs, + message=message ) @web.expose def tool_for_month( self, trans, **kwd ): if 'operation' in kwd: @@ -462,8 +492,7 @@ class Jobs( BaseController ): action='job_info', **kwd ) ) if operation == "tool_per_month": - # The received id is the job id, so we need to get the id of the user - # that submitted the job. + # The received id is the job id, so we need to get the jobs tool_id. job_id = kwd.get( 'id', None ) job = get_job( trans, job_id ) kwd[ 'tool_id' ] = job.tool_id @@ -497,7 +526,10 @@ class Jobs( BaseController ): .one() # TODO: for some reason the job_info.id is not the same as job_id in the template, so we need to pass job_id # This needs to be fixed ASAP! - return trans.fill_template( '/webapps/reports/job_info.mako', job_id=job_id, job_info=job_info, message=message ) + return trans.fill_template( '/webapps/reports/job_info.mako', + job_id=job_id, + job_info=job_info, + message=message ) @web.expose def per_domain( self, trans, **kwd ): # TODO: rewrite using alchemy --- a/templates/webapps/reports/system.mako +++ b/templates/webapps/reports/system.mako @@ -1,13 +1,17 @@ <%inherit file="/base.mako"/><%namespace file="/message.mako" import="render_msg" /> +<% + from galaxy.web.framework.helpers import time_ago +%> + %if message: ${render_msg( message, 'done' )} %endif <div class="toolForm"><h3 align="center">Old Histories and Datasets</h3> - <table align="center" width="70%" class="border" cellpadding="5" cellspacing="5"> + <table align="center" width="90%" class="border" cellpadding="5" cellspacing="5"><tr><td><form method="post" action="system"> @@ -33,10 +37,14 @@ </td></tr></table> - <br clear="left" /><br /> + <br clear="left" /><h3 align="center">Current Disk Space Where Datasets are Stored</h3> - <table align="center" width="60%" class="colored"> - <tr><td colspan="5"><div class="toolFormTitle">Disk Usage for ${file_path}</div></td></tr> + <table align="center" width="90%" class="colored"> + <tr> + <td colspan="5"> + <b>Disk Usage for ${file_path}</b> + </td> + </tr><tr class="header"><td>File System</td><td>Disk Size</td> @@ -51,16 +59,16 @@ <td>${disk_usage[3]}</td><td>${disk_usage[4]}</td></tr> - %if len( datasets ) == 0: - <tr class="header"><td colspan="5">There are no datasets larger than ${file_size_str}</td></tr> - %else: - <tr><td colspan="5"><div class="toolFormTitle">${len( datasets )} largest datasets over ${file_size_str}</div></td></tr> + </table> + <br clear="left" /> + %if datasets.count() > 0: + <h3 align="center">${datasets.count()} largest unpurged data files over ${file_size_str}</h3> + <table align="center" width="90%" class="colored"><tr class="header"><td>File</td><td>Last Updated</td> - <td>History ID</td><td>Deleted</td> - <td>Size on Disk</td> + <td>File Size</td></tr><% ctr = 0 %> %for dataset in datasets: @@ -69,14 +77,19 @@ %else: <tr class="tr"> %endif - <td>dataset_${dataset[0]}.dat</td> - <td>${dataset[1]}</td> - <td>${dataset[2]}</td> - <td>${dataset[3]}</td> - <td>${dataset[4]}</td> + <td> + <% dataset_label = 'dataset%d_.dat' % dataset.id %> + <a href="${h.url_for( controller='system', action='dataset_info', id=trans.security.encode_id( dataset.id ) )}">${dataset_label}</a> + </td> + <td>${time_ago( dataset.update_time )}</td> + <td>${dataset.deleted}</td> + <td>${dataset.file_size}</td></tr><% ctr += 1 %> %endfor - %endif - </table> + </table> + <br clear="left" /> + %else: + <h3 align="center">There are no unpurged data files larger than ${file_size_str}</h3> + %endif </div> --- /dev/null +++ b/templates/webapps/reports/dataset_info.mako @@ -0,0 +1,119 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + +<% + from galaxy.web.framework.helpers import time_ago + from galaxy.web.controllers.library_common import get_containing_library_from_library_dataset +%> + +%if message: + ${render_msg( message, 'done' )} +%endif + +<div class="toolForm"> + <h3 align="center">Dataset Information</h3> + <div class="toolFormBody"> + <div class="form-row"> + <label>Date uploaded:</label> + ${time_ago( dataset.create_time )} + <div style="clear: both"></div> + </div> + <div class="form-row"> + <label>Last updated:</label> + ${time_ago( dataset.update_time )} + <div style="clear: both"></div> + </div> + <div class="form-row"> + <label>File size:</label> + ${dataset.get_size( nice_size=True )} + <div style="clear: both"></div> + </div> + <div class="form-row"> + <label>State:</label> + ${dataset.state} + <div style="clear: both"></div> + </div> + </div> +</div> +%if associated_hdas: + <p/> + <b>Active (undeleted) history items that use this library dataset's disk file</b> + <div class="toolForm"> + <table class="grid"> + <thead> + <tr> + <th>History</th> + <th>History Item</th> + <th>Last Updated</th> + <th>User</th> + </tr> + </thead> + %for hda in associated_hdas: + <tr> + <td>${hda.history.get_display_name()}</td> + <td>${hda.get_display_name()}</td> + <td>${time_ago( hda.update_time )}</td> + <td> + %if hda.history.user: + ${hda.history.user.email} + %else: + anonymous + %endif + </td> + </tr> + %endfor + </table> + </div> + <p/> +%endif +%if associated_lddas: + <p/> + <b>Other active (undeleted) library datasets that use this library dataset's disk file</b> + <div class="toolForm"> + <table class="grid"> + <thead> + <tr> + <th>Library</th> + <th>Folder</th> + <th>Library Dataset</th> + <th>Last Updated</th> + <th>Uploaded By</th> + </tr> + </thead> + %for ldda in associated_lddas: + <% containing_library = get_containing_library_from_library_dataset( trans, ldda.library_dataset ) %> + <tr> + <td> + <% + if containing_library: + library_display_name = containing_library.get_display_name() + else: + library_display_name = 'error finding library' + %> + ${library_display_name} + </td> + <td> + <% + library_dataset = ldda.library_dataset + folder = library_dataset.folder + folder_display_name = folder.get_display_name() + if folder_display_name == library_display_name: + folder_display_name = 'library root' + %> + ${folder_display_name} + </td> + <td>${ldda.get_display_name()}</td> + <td>${time_ago( ldda.update_time )}</td> + <td> + %if ldda.user: + ${ldda.user.email} + %else: + anonymous + %endif + </td> + </tr> + %endfor + </table> + </div> + <p/> +%endif --- a/templates/webapps/reports/jobs_per_user.mako +++ b/templates/webapps/reports/jobs_per_user.mako @@ -2,7 +2,7 @@ <%namespace file="/message.mako" import="render_msg" /> %if message: - ${render_msg( message, status )} + ${render_msg( message, 'done' )} %endif <div class="toolForm"> --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -432,13 +432,19 @@ class Dataset( object ): path = os.path.join( os.path.join( self.file_path, *directory_hash_id( self.id ) ), "dataset_%d_files" % self.id ) # Make path absolute return os.path.abspath( path ) - def get_size( self ): + def get_size( self, nice_size=False ): """Returns the size of the data on disk""" if self.file_size: - return self.file_size + if nice_size: + return galaxy.datatypes.data.nice_size( self.file_size ) + else: + return self.file_size else: try: - return os.path.getsize( self.file_name ) + if nice_size: + return galaxy.datatypes.data.nice_size( os.path.getsize( self.file_name ) ) + else: + return os.path.getsize( self.file_name ) except OSError: return 0 def set_size( self ): --- a/templates/webapps/reports/jobs_specified_month_in_error.mako +++ b/templates/webapps/reports/jobs_specified_month_in_error.mako @@ -2,7 +2,7 @@ <%namespace file="/message.mako" import="render_msg" /> %if message: - ${render_msg( message, status )} + ${render_msg( message, 'done' )} %endif <div class="toolForm"> --- a/templates/webapps/reports/jobs_per_tool.mako +++ b/templates/webapps/reports/jobs_per_tool.mako @@ -2,7 +2,7 @@ <%namespace file="/message.mako" import="render_msg" /> %if message: - ${render_msg( message, status )} + ${render_msg( message, 'done' )} %endif <div class="toolForm"> --- a/templates/webapps/reports/users_last_access_date.mako +++ b/templates/webapps/reports/users_last_access_date.mako @@ -2,7 +2,7 @@ <%namespace file="/message.mako" import="render_msg" /> %if message: - ${render_msg( message, status )} + ${render_msg( message, 'done' )} %endif <div class="toolForm"> --- a/templates/webapps/reports/jobs_per_month_in_error.mako +++ b/templates/webapps/reports/jobs_per_month_in_error.mako @@ -2,7 +2,7 @@ <%namespace file="/message.mako" import="render_msg" /> %if message: - ${render_msg( message, status )} + ${render_msg( message, 'done' )} %endif <div class="toolForm"> @@ -24,7 +24,7 @@ %else: <tr class="tr"> %endif - <td><a href="${h.url_for( controller='jobs', action='specified_month_in_error', month=job[0] )}">${job[2]} ${job[3]}</a></td> + <td><a href="${h.url_for( controller='jobs', action='specified_month_in_error', specified_date=job[0]+'-01' )}">${job[2]} ${job[3]}</a></td><td>${job[1]}</td></tr><% ctr += 1 %> --- a/lib/galaxy/webapps/reports/controllers/system.py +++ b/lib/galaxy/webapps/reports/controllers/system.py @@ -1,10 +1,6 @@ import operator, os from datetime import datetime, timedelta from galaxy.web.base.controller import * -#import pkg_resources -#pkg_resources.require( "SQLAlchemy >= 0.4" ) -#from sqlalchemy.orm import eagerload -#from sqlalchemy import desc from galaxy import model from galaxy.model.orm import * import logging @@ -44,7 +40,6 @@ class System( BaseController ): deleted_histories_days=deleted_histories_days, deleted_datasets_days=deleted_datasets_days, message=message ) - def userless_histories( self, trans, **kwd ): """The number of userless histories and associated datasets that have not been updated for the specified number of days.""" params = util.Params( kwd ) @@ -54,11 +49,6 @@ class System( BaseController ): cutoff_time = datetime.utcnow() - timedelta( days=userless_histories_days ) history_count = 0 dataset_count = 0 - #h = self.app.model.History - #where = ( h.table.c.user_id==None ) & ( h.table.c.deleted=='f' ) & ( h.table.c.update_time < cutoff_time ) - #histories = h.query().filter( where ).options( eagerload( 'datasets' ) ) - - #for history in histories: for history in trans.sa_session.query( model.History ) \ .filter( and_( model.History.table.c.user_id == None, model.History.table.c.deleted == True, @@ -71,7 +61,6 @@ class System( BaseController ): else: message = "Enter the number of days." return str( userless_histories_days ), message - def deleted_histories( self, trans, **kwd ): """ The number of histories that were deleted more than the specified number of days ago, but have not yet been purged. @@ -104,7 +93,6 @@ class System( BaseController ): else: message = "Enter the number of days." return str( deleted_histories_days ), message - def deleted_datasets( self, trans, **kwd ): """The number of datasets that were deleted more than the specified number of days ago, but have not yet been purged.""" params = util.Params( kwd ) @@ -128,6 +116,25 @@ class System( BaseController ): else: message = "Enter the number of days." return str( deleted_datasets_days ), message + @web.expose + def dataset_info( self, trans, **kwd ): + params = util.Params( kwd ) + message = '' + dataset = trans.sa_session.query( model.Dataset ).get( trans.security.decode_id( kwd.get( 'id', '' ) ) ) + # Get all associated hdas and lddas that use the same disk file. + associated_hdas = trans.sa_session.query( trans.model.HistoryDatasetAssociation ) \ + .filter( and_( trans.model.HistoryDatasetAssociation.deleted == False, + trans.model.HistoryDatasetAssociation.dataset_id == dataset.id ) ) \ + .all() + associated_lddas = trans.sa_session.query( trans.model.LibraryDatasetDatasetAssociation ) \ + .filter( and_( trans.model.LibraryDatasetDatasetAssociation.deleted == False, + trans.model.LibraryDatasetDatasetAssociation.dataset_id == dataset.id ) ) \ + .all() + return trans.fill_template( '/webapps/reports/dataset_info.mako', + dataset=dataset, + associated_hdas=associated_hdas, + associated_lddas=associated_lddas, + message=message ) def get_disk_usage( self, file_path ): df_cmd = 'df -h ' + file_path is_sym_link = os.path.islink( file_path ) @@ -163,13 +170,13 @@ class System( BaseController ): def disk_usage( self, trans, **kwd ): file_path = trans.app.config.file_path disk_usage = self.get_disk_usage( file_path ) - min_file_size = 2**32 # 4 Gb + #min_file_size = 2**32 # 4 Gb + min_file_size = 0 file_size_str = nice_size( min_file_size ) - datasets = [] - for dataset in trans.sa_session.query( model.Dataset ) \ - .filter( model.Dataset.table.c.file_size > min_file_size ) \ - .order_by( desc( model.Dataset.table.c.file_size ) ): - datasets.append( ( dataset.id, str( dataset.update_time )[0:10], dataset.history.history_id, dataset.deleted, dataset.file_size ) ) + datasets = trans.sa_session.query( model.Dataset ) \ + .filter( and_( model.Dataset.table.c.purged == False, + model.Dataset.table.c.file_size > min_file_size ) ) \ + .order_by( desc( model.Dataset.table.c.file_size ) ) return file_path, disk_usage, datasets, file_size_str def nice_size( size ): --- a/lib/galaxy/webapps/reports/controllers/users.py +++ b/lib/galaxy/webapps/reports/controllers/users.py @@ -13,14 +13,12 @@ log = logging.getLogger( __name__ ) class Users( BaseController ): @web.expose def registered_users( self, trans, **kwd ): - params = util.Params( kwd ) - message = params.get( 'message', '' ) + message = util.restore_text( kwd.get( 'message', '' ) ) num_users = trans.sa_session.query( galaxy.model.User ).count() return trans.fill_template( '/webapps/reports/registered_users.mako', num_users=num_users, message=message ) @web.expose def registered_users_per_month( self, trans, **kwd ): - params = util.Params( kwd ) - message = params.get( 'message', '' ) + message = util.restore_text( kwd.get( 'message', '' ) ) q = sa.select( ( sa.func.date_trunc( 'month', sa.func.date( galaxy.model.User.table.c.create_time ) ).label( 'date' ), sa.func.count( galaxy.model.User.table.c.id ).label( 'num_users' ) ), from_obj = [ galaxy.model.User.table ], @@ -29,15 +27,19 @@ class Users( BaseController ): users = [] for row in q.execute(): users.append( ( row.date.strftime( "%Y-%m" ), - row.num_users, - row.date.strftime( "%B" ), - row.date.strftime( "%Y" ) ) ) - return trans.fill_template( '/webapps/reports/registered_users_per_month.mako', users=users, message=message ) + row.num_users, + row.date.strftime( "%B" ), + row.date.strftime( "%Y" ) ) ) + return trans.fill_template( '/webapps/reports/registered_users_per_month.mako', + users=users, + message=message ) @web.expose def specified_month( self, trans, **kwd ): - params = util.Params( kwd ) - message = params.get( 'message', '' ) - year, month = map( int, params.get( 'month', datetime.utcnow().strftime( "%Y-%m" ) ).split( "-" ) ) + message = util.restore_text( kwd.get( 'message', '' ) ) + # If specified_date is not received, we'll default to the current month + specified_date = kwd.get( 'specified_date', datetime.utcnow().strftime( "%Y-%m-%d" ) ) + specified_month = specified_date[ :7 ] + year, month = map( int, specified_month.split( "-" ) ) start_date = date( year, month, 1 ) end_date = start_date + timedelta( days=calendar.monthrange( year, month )[1] ) month_label = start_date.strftime( "%B" ) @@ -63,9 +65,10 @@ class Users( BaseController ): message=message ) @web.expose def specified_date( self, trans, **kwd ): - params = util.Params( kwd ) - message = params.get( 'message', '' ) - year, month, day = map( int, params.get( 'specified_date', datetime.utcnow().strftime( "%Y-%m-%d" ) ).split( "-" ) ) + message = util.restore_text( kwd.get( 'message', '' ) ) + # If specified_date is not received, we'll default to the current month + specified_date = kwd.get( 'specified_date', datetime.utcnow().strftime( "%Y-%m-%d" ) ) + year, month, day = map( int, specified_date.split( "-" ) ) start_date = date( year, month, day ) end_date = start_date + timedelta( days=1 ) day_of_month = start_date.strftime( "%d" ) @@ -91,9 +94,8 @@ class Users( BaseController ): message=message ) @web.expose def last_access_date( self, trans, **kwd ): - params = util.Params( kwd ) - message = params.get( 'message', '' ) - not_logged_in_for_days = params.get( 'not_logged_in_for_days', 90 ) + message = util.restore_text( kwd.get( 'message', '' ) ) + not_logged_in_for_days = kwd.get( 'not_logged_in_for_days', 90 ) if not not_logged_in_for_days: not_logged_in_for_days = 0 cutoff_time = datetime.utcnow() - timedelta( days=int( not_logged_in_for_days ) ) --- a/templates/webapps/reports/registered_users_specified_month.mako +++ b/templates/webapps/reports/registered_users_specified_month.mako @@ -24,7 +24,7 @@ %else: <tr class="tr"> %endif - <td><a href="${h.url_for( controller='users', action='specified_date', specified_date=user[0], day_label=user[3], month_label=month_label, year_label=year_label, day_of_month=user[1] )}">${user[3]}, ${month_label} ${user[1]}, ${year_label}</a></td> + <td><a href="${h.url_for( controller='users', action='specified_date', specified_date=user[0] )}">${user[3]}, ${month_label} ${user[1]}, ${year_label}</a></td><td>${user[2]}</td></tr><% ctr += 1 %> --- a/templates/webapps/reports/index.mako +++ b/templates/webapps/reports/index.mako @@ -73,14 +73,14 @@ </div><div class="toolSectionBody"><div class="toolSectionBg"> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='jobs', action='today_all' )}">Number of jobs today</a></div> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='jobs', action='specified_month_all' )}">Number of jobs per day for current month</a></div> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='jobs', action='specified_month_in_error' )}">Number of jobs in error per day for current month</a></div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='jobs', action='today_all' )}">Today's jobs</a></div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='jobs', action='specified_month_all' )}">Jobs per day this month</a></div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='jobs', action='specified_month_in_error' )}">Jobs in error per day this month</a></div><div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='jobs', action='all_unfinished' )}">All unfinished jobs</a></div> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='jobs', action='per_month_all' )}">Number of jobs per month</a></div> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='jobs', action='per_month_in_error' )}">Number of jobs in error per month</a></div> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='jobs', action='per_user' )}">Number of jobs per user</a></div> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='jobs', action='per_tool' )}">Number of jobs per tool</a></div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='jobs', action='per_month_all' )}">Jobs per month</a></div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='jobs', action='per_month_in_error' )}">Jobs in error per month</a></div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='jobs', action='per_user' )}">Jobs per user</a></div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='jobs', action='per_tool' )}">Jobs per tool</a></div></div></div><div class="toolSectionPad"></div> @@ -101,7 +101,7 @@ </div><div class="toolSectionBody"><div class="toolSectionBg"> - <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='system', action='index' )}">Disk space, old histories and datasets</a></div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='system', action='index' )}">Disk space maintenance</a></div></div></div></div> @@ -110,6 +110,6 @@ </%def><%def name="center_panel()"> - <% center_url = h.url_for( controller='jobs', action='today_all' ) %> + <% center_url = h.url_for( controller='jobs', action='specified_month_all' ) %><iframe name="galaxy_main" id="galaxy_main" frameborder="0" style="position: absolute; width: 100%; height: 100%;" src="${center_url}"></iframe></%def> --- a/templates/webapps/reports/jobs_per_domain.mako +++ b/templates/webapps/reports/jobs_per_domain.mako @@ -2,7 +2,7 @@ <%namespace file="/message.mako" import="render_msg" /> %if message: - ${render_msg( message, status )} + ${render_msg( message, 'done' )} %endif <div class="toolForm"> --- a/lib/galaxy/web/controllers/library_common.py +++ b/lib/galaxy/web/controllers/library_common.py @@ -1904,15 +1904,14 @@ def branch_deleted( folder ): def get_containing_library_from_library_dataset( trans, library_dataset ): """Given a library_dataset, get the containing library""" folder = library_dataset.folder - parent = folder while folder.parent: - parent = folder.parent - # We have parent set to the library's root folder, which has the - # same name as the library + folder = folder.parent + # We have folder set to the library's root folder, which has the same name as the library for library in trans.sa_session.query( trans.model.Library ) \ .filter( and_( trans.model.Library.table.c.deleted == False, - trans.model.Library.table.c.name == parent.name ) ): - if library.root_folder == parent: + trans.model.Library.table.c.name == folder.name ) ): + # Just to double-check + if library.root_folder == folder: return library return None --- a/templates/webapps/reports/jobs_tool_per_month.mako +++ b/templates/webapps/reports/jobs_tool_per_month.mako @@ -2,7 +2,7 @@ <%namespace file="/message.mako" import="render_msg" /> %if message: - ${render_msg( message, status )} + ${render_msg( message, 'done' )} %endif <div class="toolForm"> @@ -25,7 +25,7 @@ <tr class="tr"> %endif <td>${job[2]} ${job[3]}</td> - <td><a href="${h.url_for( controller='jobs', action='tool_for_month', tool_id=tool_id, month=job[0] )}">${job[1]}</a></td> + <td><a href="${h.url_for( controller='jobs', action='tool_for_month', tool_id=tool_id, specified_date=job[0]+'-01' )}">${job[1]}</a></td></tr><% ctr += 1 %> %endfor --- a/templates/webapps/reports/registered_users_per_month.mako +++ b/templates/webapps/reports/registered_users_per_month.mako @@ -24,7 +24,7 @@ %else: <tr class="tr"> %endif - <td><a href="${h.url_for( controller='users', action='specified_month', month=user[0], month_label=user[2].strip(), year_label=user[3] )}">${user[2]} ${user[3]}</a></td> + <td><a href="${h.url_for( controller='users', action='specified_month', specified_date=user[0]+'-01' )}">${user[2]} ${user[3]}</a></td><td>${user[1]}</td></tr><% ctr += 1 %> --- a/templates/webapps/reports/jobs_specified_month_all.mako +++ b/templates/webapps/reports/jobs_specified_month_all.mako @@ -2,7 +2,7 @@ <%namespace file="/message.mako" import="render_msg" /> %if message: - ${render_msg( message, status )} + ${render_msg( message, 'done' )} %endif <div class="toolForm"> --- a/templates/webapps/reports/jobs_per_month_all.mako +++ b/templates/webapps/reports/jobs_per_month_all.mako @@ -2,7 +2,7 @@ <%namespace file="/message.mako" import="render_msg" /> %if message: - ${render_msg( message, status )} + ${render_msg( message, 'done' )} %endif <div class="toolForm"> @@ -26,7 +26,7 @@ %else: <tr class="tr"> %endif - <td><a href="${h.url_for( controller='jobs', action='specified_month_all', month=job[0] )}">${job[4]} ${job[5]}</a></td> + <td><a href="${h.url_for( controller='jobs', action='specified_month_all', specified_date=job[0]+'-01' )}">${job[4]} ${job[5]}</a></td><td>${job[1]}</td><td>${job[2]}</td><td>${job[3]}</td>