[hg] galaxy 1734: Have DataMetaFilter in dynamic_options.py retu...
by Greg Von Kuster
details: http://www.bx.psu.edu/hg/galaxy/rev/dc753c56c745
changeset: 1734:dc753c56c745
user: Dan Blankenberg <dan(a)bx.psu.edu>
date: Fri Feb 06 13:32:44 2009 -0500
description:
Have DataMetaFilter in dynamic_options.py return a list of UnvalidatedValues when the referenced metadata value is not set.
2 file(s) affected in this change:
lib/galaxy/tools/parameters/basic.py
lib/galaxy/tools/parameters/dynamic_options.py
diffs (40 lines):
diff -r a4214de3752e -r dc753c56c745 lib/galaxy/tools/parameters/basic.py
--- a/lib/galaxy/tools/parameters/basic.py Tue Feb 03 18:34:27 2009 -0500
+++ b/lib/galaxy/tools/parameters/basic.py Fri Feb 06 13:32:44 2009 -0500
@@ -485,8 +485,12 @@
else:
return self.static_options
def get_legal_values( self, trans, other_values ):
+ def _get_UnvalidatedValue_value( value ):
+ if isinstance( value, UnvalidatedValue ):
+ return value.value
+ return value
if self.options:
- return set( v for _, v, _ in self.options.get_options( trans, other_values ) )
+ return map( _get_UnvalidatedValue_value, set( v for _, v, _ in self.options.get_options( trans, other_values ) ) )
elif self.dynamic_options:
return set( v for _, v, _ in eval( self.dynamic_options, self.tool.code_namespace, other_values ) )
else:
@@ -511,6 +515,9 @@
field = form_builder.SelectField( self.name, self.multiple, self.display, self.refresh_on_change )
options = self.get_options( trans, other_values )
for text, optval, selected in options:
+ if isinstance( optval, UnvalidatedValue ):
+ optval = optval.value
+ text = "%s (unvalidated)" % text
if value:
selected = ( optval in value )
field.add_option( text, optval, selected )
diff -r a4214de3752e -r dc753c56c745 lib/galaxy/tools/parameters/dynamic_options.py
--- a/lib/galaxy/tools/parameters/dynamic_options.py Tue Feb 03 18:34:27 2009 -0500
+++ b/lib/galaxy/tools/parameters/dynamic_options.py Fri Feb 06 13:32:44 2009 -0500
@@ -108,7 +108,8 @@
if not isinstance( ref, self.dynamic_option.tool_param.tool.app.model.HistoryDatasetAssociation ):
return [] #not a valid dataset
meta_value = ref.metadata.get( self.key, None )
- assert meta_value is not None, "Required metadata value '%s' not found in referenced dataset" % self.key
+ if meta_value is None: #assert meta_value is not None, "Required metadata value '%s' not found in referenced dataset" % self.key
+ return [ ( disp_name, basic.UnvalidatedValue( optval ), selected ) for disp_name, optval, selected in options ]
if self.column is not None:
rval = []
10 years, 6 months
[hg] galaxy 1735: Fix for displaying a history's activatable dat...
by Greg Von Kuster
details: http://www.bx.psu.edu/hg/galaxy/rev/9fb93cfc2449
changeset: 1735:9fb93cfc2449
user: Greg Von Kuster <greg(a)bx.psu.edu>
date: Thu Feb 19 16:25:05 2009 -0500
description:
Fix for displaying a history's activatable datasets.
1 file(s) affected in this change:
templates/root/history.mako
diffs (26 lines):
diff -r dc753c56c745 -r 9fb93cfc2449 templates/root/history.mako
--- a/templates/root/history.mako Fri Feb 06 13:32:44 2009 -0500
+++ b/templates/root/history.mako Thu Feb 19 16:25:05 2009 -0500
@@ -248,13 +248,20 @@
<%namespace file="history_common.mako" import="render_dataset" />
-%if ( show_deleted and not history.activatable_datasets ) or ( not show_deleted and not history.active_datasets ):
+<%
+ activatable_datasets = []
+ for hda in history.datasets:
+ if not hda.dataset.purged:
+ activatable_datasets.append( hda )
+%>
+
+%if ( show_deleted and not activatable_datasets ) or ( not show_deleted and not history.active_datasets ):
<div class="infomessagesmall" id="emptyHistoryMessage">
%else:
<%
if show_deleted:
#all datasets
- datasets_to_show = history.activatable_datasets
+ datasets_to_show = activatable_datasets
else:
#active (not deleted)
datasets_to_show = history.active_datasets
10 years, 6 months
[svn] [2778] Typo
by nate＠bx.psu.edu
Revision: 2778
Author: nate
Date: 2009-02-18 16:19:31 -0500 (Wed, 18 Feb 2009)
Log Message:
-----------
Typo
Modified Paths:
--------------
dependencies/taxonomy2tree/taxonomy2tree.c
dependencies/tree2PS-fast/tree2ps.c
Modified: dependencies/taxonomy2tree/taxonomy2tree.c
===================================================================
--- dependencies/taxonomy2tree/taxonomy2tree.c 2009-02-18 21:12:38 UTC (rev 2777)
+++ dependencies/taxonomy2tree/taxonomy2tree.c 2009-02-18 21:19:31 UTC (rev 2778)
@@ -8,7 +8,7 @@
#include <ctype.h>
#ifndef __APPLE__
-#define isnumber (c) ( (c>='0') && (c<='9'))
+#define isnumber(c) ( (c>='0') && (c<='9'))
#endif
#define DEFAULT_STRING_ALLOC 16L
Modified: dependencies/tree2PS-fast/tree2ps.c
===================================================================
--- dependencies/tree2PS-fast/tree2ps.c 2009-02-18 21:12:38 UTC (rev 2777)
+++ dependencies/tree2PS-fast/tree2ps.c 2009-02-18 21:19:31 UTC (rev 2778)
@@ -8,7 +8,7 @@
#include <ctype.h>
#ifndef __APPLE__
-#define isnumber (c) ( (c>='0') && (c<='9'))
+#define isnumber(c) ( (c>='0') && (c<='9'))
#endif
#define DEFAULT_STRING_ALLOC 16L
10 years, 6 months
[svn] [2777] Make taxonomy tools compile without changing the source
by nate＠bx.psu.edu
Revision: 2777
Author: nate
Date: 2009-02-18 16:12:38 -0500 (Wed, 18 Feb 2009)
Log Message:
-----------
Make taxonomy tools compile without changing the source
Modified Paths:
--------------
dependencies/taxBuilder/tb.readme
dependencies/taxonomy2tree/t2t.readme
dependencies/taxonomy2tree/taxonomy2tree.c
dependencies/tree2PS-fast/t2ps.readme
dependencies/tree2PS-fast/tree2ps.c
Modified: dependencies/taxBuilder/tb.readme
===================================================================
--- dependencies/taxBuilder/tb.readme 2009-02-05 15:19:08 UTC (rev 2776)
+++ dependencies/taxBuilder/tb.readme 2009-02-18 21:12:38 UTC (rev 2777)
@@ -5,17 +5,9 @@
$gcc -o taxBuilder -fast avl.c main.c
On Linux:
+$gcc -o taxBuilder -O3 avl.c main.c
-Insert the following near the top of main.c
-
- #define isnumber (c) ( (c>='0') && (c<='9'))
-
-Replace -fast with -O3 (the letter 'O'):
-
- $gcc -o taxBuilder -O3 avl.c main.c
-
INSTALL
=======
-Place the binary into a directory that is listed in setup_paths.sh script located in the root of Galaxy installation
-
+Place the binary into a directory that is in your Galaxy user's $PATH.
Modified: dependencies/taxonomy2tree/t2t.readme
===================================================================
--- dependencies/taxonomy2tree/t2t.readme 2009-02-05 15:19:08 UTC (rev 2776)
+++ dependencies/taxonomy2tree/t2t.readme 2009-02-18 21:12:38 UTC (rev 2777)
@@ -5,18 +5,9 @@
$gcc -o taxonomy2tree -fast avl.c taxonomy2tree.c
On Linux:
+$gcc -o taxonomy2tree -O3 avl.c taxonomy2tree.c
-Insert the following near the top of taxonomy2tree.c
-
- #define isnumber (c) ( (c>='0') && (c<='9'))
-
-Replace -fast with -O3 (the letter 'O'):
-
- $gcc -o taxonomy2tree -O3 avl.c taxonomy2tree.c
-
INSTALL
=======
-Place the binary into a directory that is listed in setup_paths.sh script located in the root of Galaxy installation
-
-
+Place the binary into a directory that is in your Galaxy user's $PATH.
Modified: dependencies/taxonomy2tree/taxonomy2tree.c
===================================================================
--- dependencies/taxonomy2tree/taxonomy2tree.c 2009-02-05 15:19:08 UTC (rev 2776)
+++ dependencies/taxonomy2tree/taxonomy2tree.c 2009-02-18 21:12:38 UTC (rev 2777)
@@ -7,6 +7,10 @@
#include <time.h>
#include <ctype.h>
+#ifndef __APPLE__
+#define isnumber (c) ( (c>='0') && (c<='9'))
+#endif
+
#define DEFAULT_STRING_ALLOC 16L
#define NUMBER_OF_FIELDS 24
#define NUMBER_OF_TAX_FIELDS 22
Modified: dependencies/tree2PS-fast/t2ps.readme
===================================================================
--- dependencies/tree2PS-fast/t2ps.readme 2009-02-05 15:19:08 UTC (rev 2776)
+++ dependencies/tree2PS-fast/t2ps.readme 2009-02-18 21:12:38 UTC (rev 2777)
@@ -5,18 +5,9 @@
$gcc -o tree2PS-fast -fast avl.c tree2ps.c
On Linux:
+$gcc -o tree2PS-fast -O3 avl.c tree2ps.c
-Insert the following near the top of tree2ps.c
-
- #define isnumber (c) ( (c>='0') && (c<='9'))
-
-Replace -fast with -O3 (the letter 'O'):
-
- $gcc -o tree2PS-fast -O3 avl.c tree2ps.c
-
INSTALL
=======
-Place the binary into a directory that is listed in setup_paths.sh script located in the root of Galaxy installation
-
-
+Place the binary into a directory that is in your Galaxy user's $PATH.
Modified: dependencies/tree2PS-fast/tree2ps.c
===================================================================
--- dependencies/tree2PS-fast/tree2ps.c 2009-02-05 15:19:08 UTC (rev 2776)
+++ dependencies/tree2PS-fast/tree2ps.c 2009-02-18 21:12:38 UTC (rev 2777)
@@ -7,6 +7,10 @@
#include <time.h>
#include <ctype.h>
+#ifndef __APPLE__
+#define isnumber (c) ( (c>='0') && (c<='9'))
+#endif
+
#define DEFAULT_STRING_ALLOC 16L
static char *const Usage = "tree2ps newick_file ps_output_file max_tree_level (<=0 to show all levels) font_size (in 2-255, 8 is a good default) \nmax_leaves (<=0 to show all) count_duplicate_tax_id (0 or 1; with 0 multiple copies of the same taxid count as 1)\n";
10 years, 6 months
Project structure
by Ayton Meintjes
Hi all
I'd be interested to hear comments/advice on how to best deploy Galaxy
locally with the following requirements:
1) Need to keep it in sync with central source repository. Or are there
"stable releases" I should be using? I'm just updating from the svn tree.
2) Need to add own tools, data formats, tests etc. I have currently created
my own repository which mirrors the galaxy structure. I then link to these
files from the Galaxy folders. Tedious.
Even just descriptions of your own setups (and problems) would help.
thanks in advance!
10 years, 7 months
[hg] galaxy 1730: Handle exceptions when building the app (witho...
by Greg Von Kuster
details: http://www.bx.psu.edu/hg/galaxy/rev/0424f713d58d
changeset: 1730:0424f713d58d
user: Nate Coraor <nate(a)bx.psu.edu>
date: Tue Feb 03 14:14:02 2009 -0500
description:
Handle exceptions when building the app (without a try block, you have to manually kill the process if there's an exception during startup)
1 file(s) affected in this change:
lib/galaxy/web/buildapp.py
diffs (19 lines):
diff -r 743237dd0e65 -r 0424f713d58d lib/galaxy/web/buildapp.py
--- a/lib/galaxy/web/buildapp.py Mon Feb 02 18:57:07 2009 -0500
+++ b/lib/galaxy/web/buildapp.py Tue Feb 03 14:14:02 2009 -0500
@@ -51,8 +51,13 @@
if 'app' in kwargs:
app = kwargs.pop( 'app' )
else:
- from galaxy.app import UniverseApplication
- app = UniverseApplication( global_conf = global_conf, **kwargs )
+ try:
+ from galaxy.app import UniverseApplication
+ app = UniverseApplication( global_conf = global_conf, **kwargs )
+ except:
+ import traceback, sys
+ traceback.print_exc()
+ sys.exit( 1 )
atexit.register( app.shutdown )
# Create the universe WSGI application
webapp = galaxy.web.framework.WebApplication( app, session_cookie='galaxysession' )
10 years, 7 months
[hg] galaxy 1731: Stop jobs if any of their output datasets grow...
by Greg Von Kuster
details: http://www.bx.psu.edu/hg/galaxy/rev/e26741c8c642
changeset: 1731:e26741c8c642
user: Nate Coraor <nate(a)bx.psu.edu>
date: Tue Feb 03 14:15:09 2009 -0500
description:
Stop jobs if any of their output datasets grow larger than a defined limit
5 file(s) affected in this change:
lib/galaxy/config.py
lib/galaxy/jobs/__init__.py
lib/galaxy/jobs/runners/local.py
lib/galaxy/jobs/runners/pbs.py
lib/galaxy/tools/__init__.py
diffs (309 lines):
diff -r 0424f713d58d -r e26741c8c642 lib/galaxy/config.py
--- a/lib/galaxy/config.py Tue Feb 03 14:14:02 2009 -0500
+++ b/lib/galaxy/config.py Tue Feb 03 14:15:09 2009 -0500
@@ -47,6 +47,7 @@
self.cluster_files_directory = os.path.abspath( kwargs.get( "cluster_files_directory", "database/pbs" ) )
self.job_working_directory = resolve_path( kwargs.get( "job_working_directory", "database/job_working_directory" ), self.root )
self.outputs_to_working_directory = string_as_bool( kwargs.get( 'outputs_to_working_directory', False ) )
+ self.output_size_limit = int( kwargs.get( 'output_size_limit', 0 ) )
self.admin_pass = kwargs.get('admin_pass',"galaxy")
self.sendmail_path = kwargs.get('sendmail_path',"/usr/sbin/sendmail")
self.mailing_join_addr = kwargs.get('mailing_join_addr',"galaxy-user-join(a)bx.psu.edu")
diff -r 0424f713d58d -r e26741c8c642 lib/galaxy/jobs/__init__.py
--- a/lib/galaxy/jobs/__init__.py Tue Feb 03 14:14:02 2009 -0500
+++ b/lib/galaxy/jobs/__init__.py Tue Feb 03 14:15:09 2009 -0500
@@ -305,6 +305,7 @@
# and job recovery fail.
self.working_directory = \
os.path.join( self.app.config.job_working_directory, str( self.job_id ) )
+ self.output_paths = None
def get_param_dict( self ):
"""
@@ -342,7 +343,7 @@
incoming['userId'] = userId
incoming['userEmail'] = userEmail
# Build params, done before hook so hook can use
- param_dict = self.tool.build_param_dict( incoming, inp_data, out_data, self.working_directory )
+ param_dict = self.tool.build_param_dict( incoming, inp_data, out_data, self.get_output_fnames() )
# Certain tools require tasks to be completed prior to job execution
# ( this used to be performed in the "exec_before_job" hook, but hooks are deprecated ).
if self.tool.tool_type is not None:
@@ -371,7 +372,7 @@
self.param_dict = param_dict
self.extra_filenames = extra_filenames
return extra_filenames
-
+
def fail( self, message, exception=False ):
"""
Indicate job failure by setting state and message on all output
@@ -385,14 +386,14 @@
# Do this first in case we generate a traceback below
if exception:
job.traceback = traceback.format_exc()
- for dataset_assoc in job.output_datasets:
- if self.app.config.outputs_to_working_directory:
- false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % dataset_assoc.dataset.dataset.id ) )
+ if self.app.config.outputs_to_working_directory:
+ for dataset_path in self.get_output_fnames():
try:
- shutil.move( false_path, dataset_assoc.dataset.file_name )
- log.debug( "fail(): Moved %s to %s" % ( false_path, dataset_assoc.dataset.file_name ) )
+ shutil.move( dataset_path.false_path, dataset_path.real_path )
+ log.debug( "fail(): Moved %s to %s" % ( dataset_path.false_path, dataset_path.real_path ) )
except ( IOError, OSError ), e:
log.error( "fail(): Missing output file in working directory: %s" % e )
+ for dataset_assoc in job.output_datasets:
dataset = dataset_assoc.dataset
dataset.refresh()
dataset.state = dataset.states.ERROR
@@ -452,15 +453,15 @@
job.state = "error"
else:
job.state = 'ok'
+ if self.app.config.outputs_to_working_directory:
+ for dataset_path in self.get_output_fnames():
+ try:
+ shutil.move( dataset_path.false_path, dataset_path.real_path )
+ log.debug( "finish(): Moved %s to %s" % ( dataset_path.false_path, dataset_path.real_path ) )
+ except ( IOError, OSError ):
+ self.fail( "Job %s's output dataset(s) could not be read" % job.id )
+ return
for dataset_assoc in job.output_datasets:
- if self.app.config.outputs_to_working_directory:
- false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % dataset_assoc.dataset.dataset.id ) )
- try:
- shutil.move( false_path, dataset_assoc.dataset.file_name )
- log.debug( "finish(): Moved %s to %s" % ( false_path, dataset_assoc.dataset.file_name ) )
- except ( IOError, OSError ):
- self.fail( "The job's output dataset(s) could not be read" )
- return
for dataset in dataset_assoc.dataset.dataset.history_associations: #need to update all associated output hdas, i.e. history was shared with job running
dataset.blurb = 'done'
dataset.peek = 'no peek'
@@ -519,7 +520,7 @@
for fname in self.extra_filenames:
os.remove( fname )
if self.working_directory is not None:
- os.rmdir( self.working_directory )
+ shutil.rmtree( self.working_directory )
except:
log.exception( "Unable to cleanup job %d" % self.job_id )
@@ -543,9 +544,36 @@
return filenames
def get_output_fnames( self ):
+ if self.output_paths is not None:
+ return self.output_paths
+
+ class DatasetPath( object ):
+ def __init__( self, real_path, false_path = None ):
+ self.real_path = real_path
+ self.false_path = false_path
+ def __str__( self ):
+ if false_path is None:
+ return self.real_path
+ else:
+ return self.false_path
+
job = model.Job.get( self.job_id )
- return [ da.dataset.file_name for da in job.output_datasets ]
+ if self.app.config.outputs_to_working_directory:
+ self.output_paths = []
+ for name, data in [ ( da.name, da.dataset ) for da in job.output_datasets ]:
+ false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % data.id ) )
+ self.output_paths.append( DatasetPath( data.file_name, false_path ) )
+ else:
+ self.output_paths = [ DatasetPath( da.dataset.file_name ) for da in job.output_datasets ]
+ return self.output_paths
+ def check_output_sizes( self ):
+ sizes = []
+ output_paths = self.get_output_fnames()
+ for outfile in [ str( o ) for o in output_paths ]:
+ sizes.append( ( outfile, os.stat( outfile ).st_size ) )
+ return sizes
+
class DefaultJobDispatcher( object ):
def __init__( self, app ):
self.app = app
diff -r 0424f713d58d -r e26741c8c642 lib/galaxy/jobs/runners/local.py
--- a/lib/galaxy/jobs/runners/local.py Tue Feb 03 14:14:02 2009 -0500
+++ b/lib/galaxy/jobs/runners/local.py Tue Feb 03 14:15:09 2009 -0500
@@ -4,6 +4,7 @@
import threading
from galaxy import model
+from galaxy.datatypes.data import nice_size
import os, errno
from time import sleep
@@ -68,9 +69,31 @@
env = env,
preexec_fn = os.setpgrp )
job_wrapper.set_runner( 'local:///', proc.pid )
+ if self.app.config.output_size_limit > 0:
+ sleep_time = 1
+ while proc.poll() is None:
+ for outfile, size in job_wrapper.check_output_sizes():
+ if size > self.app.config.output_size_limit:
+ # Error the job immediately
+ job_wrapper.fail( 'Job output grew too large (greater than %s), please try different job parameters or' \
+ % nice_size( self.app.config.output_size_limit ) )
+ log.warning( 'Terminating job %s due to output %s growing larger than %s limit' \
+ % ( job_wrapper.job_id, os.path.basename( outfile ), nice_size( self.app.config.output_size_limit ) ) )
+ # Then kill it
+ os.killpg( proc.pid, 15 )
+ sleep( 1 )
+ if proc.poll() is None:
+ os.killpg( proc.pid, 9 )
+ proc.wait() # reap
+ log.debug( 'Job %s (pid %s) terminated' % ( job_wrapper.job_id, proc.pid ) )
+ return
+ sleep( sleep_time )
+ if sleep_time < 8:
+ # So we don't stat every second
+ sleep_time *= 2
stdout = proc.stdout.read()
stderr = proc.stderr.read()
- proc.wait()
+ proc.wait() # reap
log.debug('execution finished: %s' % command_line)
except Exception, exc:
job_wrapper.fail( "failure running job", exception=True )
diff -r 0424f713d58d -r e26741c8c642 lib/galaxy/jobs/runners/pbs.py
--- a/lib/galaxy/jobs/runners/pbs.py Tue Feb 03 14:14:02 2009 -0500
+++ b/lib/galaxy/jobs/runners/pbs.py Tue Feb 03 14:15:09 2009 -0500
@@ -2,6 +2,8 @@
from Queue import Queue, Empty
from galaxy import model
+from galaxy.datatypes.data import nice_size
+
from paste.deploy.converters import asbool
import pkg_resources
@@ -60,6 +62,7 @@
self.ofile = None
self.efile = None
self.runner_url = None
+ self.check_count = 0
class PBSJobRunner( object ):
"""
@@ -71,6 +74,8 @@
# Check if PBS was importable, fail if not
if pbs is None:
raise Exception( "PBSJobRunner requires pbs-python which was not found" )
+ if app.config.pbs_application_server and app.config.outputs_to_working_directory:
+ raise Exception( "pbs_application_server (file staging) and outputs_to_working_directory options are mutually exclusive" )
self.app = app
# 'watched' and 'queue' are both used to keep track of jobs to watch.
# 'queue' is used to add new watched jobs, and can be called from
@@ -132,6 +137,8 @@
self.queue_job( obj )
elif op == 'finish':
self.finish_job( obj )
+ elif op == 'fail_oversize_job':
+ self.fail_oversize_job( obj )
except:
log.exception( "Uncaught exception %sing job" % op )
@@ -173,8 +180,9 @@
if self.app.config.pbs_application_server:
pbs_ofile = self.app.config.pbs_application_server + ':' + ofile
pbs_efile = self.app.config.pbs_application_server + ':' + efile
- stagein = self.get_stage_in_out( job_wrapper.get_input_fnames() + job_wrapper.get_output_fnames(), symlink=True )
- stageout = self.get_stage_in_out( job_wrapper.get_output_fnames() )
+ output_files = [ str( o ) for o in job_wrapper.get_output_fnames() ]
+ stagein = self.get_stage_in_out( job_wrapper.get_input_fnames() + output_files, symlink=True )
+ stageout = self.get_stage_in_out( output_files )
job_attrs = pbs.new_attropl(5)
job_attrs[0].name = pbs.ATTR_o
job_attrs[0].value = pbs_ofile
@@ -298,6 +306,20 @@
if state == "R" and not pbs_job_state.running:
pbs_job_state.running = True
pbs_job_state.job_wrapper.change_state( "running" )
+ if self.app.config.output_size_limit > 0 and state == "R" and (pbs_job_state.check_count % 10) == 0:
+ # Every 10th time a job is checked, check the size of its outputs.
+ fail = False
+ for outfile, size in pbs_job_state.job_wrapper.check_output_sizes():
+ if size > self.app.config.output_size_limit:
+ pbs_job_state.fail_message = 'Job output grew too large (greater than %s), please try different job parameters or' \
+ % nice_size( self.app.config.output_size_limit )
+ log.warning( '(%s/%s) Dequeueing job due to output %s growing larger than %s limit' \
+ % ( galaxy_job_id, job_id, os.path.basename( outfile ), nice_size( self.app.config.output_size_limit ) ) )
+ self.work_queue.put( ( 'fail_oversize_job', pbs_job_state ) )
+ fail = True
+ break
+ if fail:
+ continue
pbs_job_state.old_state = state
new_watched.append( pbs_job_state )
else:
@@ -329,6 +351,7 @@
pbs_server_name = self.determine_pbs_server( pbs_job_state.runner_url )
if pbs_server_name not in servers:
servers.append( pbs_server_name )
+ pbs_job_state.check_count += 1
for pbs_server_name in servers:
c = pbs.pbs_connect( pbs_server_name )
if c <= 0:
@@ -386,6 +409,14 @@
# clean up the pbs files
self.cleanup( ( ofile, efile, job_file ) )
+ def fail_oversize_job( self, pbs_job_state ):
+ """
+ Seperated out so we can use the worker threads for it.
+ """
+ self.stop_job( self.app.model.Job.get( pbs_job_state.job_wrapper.job_id ) )
+ pbs_job_state.job_wrapper.fail( pbs_job_state.fail_message )
+ self.cleanup( ( pbs_job_state.ofile, pbs_job_state.efile, pbs_job_state.job_file ) )
+
def cleanup( self, files ):
if not asbool( self.app.config.get( 'debug', False ) ):
for file in files:
@@ -431,7 +462,7 @@
return
pbs.pbs_deljob( c, str( job.job_runner_external_id ), 'NULL' )
pbs.pbs_disconnect( c )
- log.debug( "(%s/%s) Removed from PBS queue at user's request" % ( job.id, job.job_runner_external_id ) )
+ log.debug( "(%s/%s) Removed from PBS queue before job completion" % ( job.id, job.job_runner_external_id ) )
def recover( self, job, job_wrapper ):
"""Recovers jobs stuck in the queued/running state when Galaxy started"""
diff -r 0424f713d58d -r e26741c8c642 lib/galaxy/tools/__init__.py
--- a/lib/galaxy/tools/__init__.py Tue Feb 03 14:14:02 2009 -0500
+++ b/lib/galaxy/tools/__init__.py Tue Feb 03 14:15:09 2009 -0500
@@ -1045,7 +1045,7 @@
input.validate( value, None )
input_values[ input.name ] = value
- def build_param_dict( self, incoming, input_datasets, output_datasets, working_directory = None ):
+ def build_param_dict( self, incoming, input_datasets, output_datasets, output_paths ):
"""
Build the dictionary of parameters for substituting into the command
line. Each value is wrapped in a `InputValueWrapper`, which allows
@@ -1098,10 +1098,14 @@
param_dict[ "_CHILD___%s___%s" % ( name, child.designation ) ] = DatasetFilenameWrapper( child )
for name, data in output_datasets.items():
# Write outputs to the working directory (for security purposes) if desired.
- if self.app.config.outputs_to_working_directory and working_directory is not None:
- false_path = os.path.abspath( os.path.join( working_directory, "galaxy_dataset_%d.dat" % data.dataset.id ) )
- param_dict[name] = DatasetFilenameWrapper( data, false_path = false_path )
- open( false_path, 'w' ).close()
+ if self.app.config.outputs_to_working_directory:
+ try:
+ false_path = [ dp.false_path for dp in output_paths if dp.real_path == data.file_name ][0]
+ param_dict[name] = DatasetFilenameWrapper( data, false_path = false_path )
+ open( false_path, 'w' ).close()
+ except IndexError:
+ log.warning( "Unable to determine alternate path for writing job outputs, outputs will be written to their real paths" )
+ param_dict[name] = DatasetFilenameWrapper( data )
else:
param_dict[name] = DatasetFilenameWrapper( data )
# Provide access to a path to store additional files
10 years, 7 months
[hg] galaxy 1732: Removed lzo requirement from two tools- getInd...
by Greg Von Kuster
details: http://www.bx.psu.edu/hg/galaxy/rev/4d4de72c57c8
changeset: 1732:4d4de72c57c8
user: guru
date: Tue Feb 03 15:06:21 2009 -0500
description:
Removed lzo requirement from two tools- getIndels and qualityFilter.
Also added validator to the regression tools.
5 file(s) affected in this change:
tools/regVariation/best_regression_subsets.xml
tools/regVariation/getIndels.py
tools/regVariation/linear_regression.xml
tools/regVariation/quality_filter.py
tools/regVariation/rcve.xml
diffs (94 lines):
diff -r e26741c8c642 -r 4d4de72c57c8 tools/regVariation/best_regression_subsets.xml
--- a/tools/regVariation/best_regression_subsets.xml Tue Feb 03 14:15:09 2009 -0500
+++ b/tools/regVariation/best_regression_subsets.xml Tue Feb 03 15:06:21 2009 -0500
@@ -13,7 +13,9 @@
<inputs>
<param format="tabular" name="input1" type="data" label="Select data" help="Query missing? See TIP below."/>
<param name="response_col" label="Response column (Y)" type="data_column" data_ref="input1" />
- <param name="predictor_cols" label="Predictor columns (X)" type="data_column" data_ref="input1" multiple="true" />
+ <param name="predictor_cols" label="Predictor columns (X)" type="data_column" data_ref="input1" multiple="true" >
+ <validator type="no_options" message="Please select at least one column."/>
+ </param>
</inputs>
<outputs>
<data format="input" name="out_file1" metadata_source="input1" />
diff -r e26741c8c642 -r 4d4de72c57c8 tools/regVariation/getIndels.py
--- a/tools/regVariation/getIndels.py Tue Feb 03 14:15:09 2009 -0500
+++ b/tools/regVariation/getIndels.py Tue Feb 03 15:06:21 2009 -0500
@@ -1,7 +1,7 @@
#!/usr/bin/python
"""
-Estimate INDELs for pait-wise alignments.
+Estimate INDELs for pair-wise alignments.
usage: %prog maf_input out_file1 out_file2
"""
@@ -10,22 +10,10 @@
from galaxy import eggs
import pkg_resources
pkg_resources.require( "bx-python" )
-pkg_resources.require( "lrucache" )
-try:
- pkg_resources.require("numpy")
- pkg_resources.require( "python-lzo" )
-except:
- pass
+pkg_resources.require("numpy")
import psyco_full
import sys
-import os, os.path
-from UserDict import DictMixin
-import bx.wiggle
-from bx.binned_array import BinnedArray, FileBinnedArray
-from bx.bitset import *
-from bx.bitset_builders import *
-from fpconst import isNaN
from bx.cookbook import doc_optparse
from galaxy.tools.exception_handling import *
import bx.align.maf
diff -r e26741c8c642 -r 4d4de72c57c8 tools/regVariation/linear_regression.xml
--- a/tools/regVariation/linear_regression.xml Tue Feb 03 14:15:09 2009 -0500
+++ b/tools/regVariation/linear_regression.xml Tue Feb 03 15:06:21 2009 -0500
@@ -12,7 +12,9 @@
<inputs>
<param format="tabular" name="input1" type="data" label="Select data" help="Query missing? See TIP below."/>
<param name="response_col" label="Response column (Y)" type="data_column" data_ref="input1" />
- <param name="predictor_cols" label="Predictor columns (X)" type="data_column" data_ref="input1" multiple="true" />
+ <param name="predictor_cols" label="Predictor columns (X)" type="data_column" data_ref="input1" multiple="true" >
+ <validator type="no_options" message="Please select at least one column."/>
+ </param>
</inputs>
<outputs>
<data format="input" name="out_file1" metadata_source="input1" />
diff -r e26741c8c642 -r 4d4de72c57c8 tools/regVariation/quality_filter.py
--- a/tools/regVariation/quality_filter.py Tue Feb 03 14:15:09 2009 -0500
+++ b/tools/regVariation/quality_filter.py Tue Feb 03 15:06:21 2009 -0500
@@ -12,11 +12,7 @@
import pkg_resources
pkg_resources.require( "bx-python" )
pkg_resources.require( "lrucache" )
-try:
- pkg_resources.require("numpy")
- pkg_resources.require( "python-lzo" )
-except:
- pass
+pkg_resources.require("numpy")
import psyco_full
import sys
diff -r e26741c8c642 -r 4d4de72c57c8 tools/regVariation/rcve.xml
--- a/tools/regVariation/rcve.xml Tue Feb 03 14:15:09 2009 -0500
+++ b/tools/regVariation/rcve.xml Tue Feb 03 15:06:21 2009 -0500
@@ -11,7 +11,9 @@
<inputs>
<param format="tabular" name="input1" type="data" label="Select data" help="Query missing? See TIP below."/>
<param name="response_col" label="Response column (Y)" type="data_column" data_ref="input1" />
- <param name="predictor_cols" label="Predictor columns (X)" type="data_column" data_ref="input1" multiple="true" />
+ <param name="predictor_cols" label="Predictor columns (X)" type="data_column" data_ref="input1" multiple="true">
+ <validator type="no_options" message="Please select at least one column."/>
+ </param>
</inputs>
<outputs>
<data format="input" name="out_file1" metadata_source="input1" />
10 years, 7 months
[hg] galaxy 1733: Fixes to get functional tests working.
by Greg Von Kuster
details: http://www.bx.psu.edu/hg/galaxy/rev/a4214de3752e
changeset: 1733:a4214de3752e
user: guru
date: Tue Feb 03 18:34:27 2009 -0500
description:
Fixes to get functional tests working.
4 file(s) affected in this change:
test-data/branchlength_out.dat
tools/hyphy/hyphy_branch_lengths_wrapper.xml
tools/regVariation/getIndels.py
tools/regVariation/quality_filter.py
diffs (95 lines):
diff -r 4d4de72c57c8 -r a4214de3752e test-data/branchlength_out.dat
--- a/test-data/branchlength_out.dat Tue Feb 03 15:06:21 2009 -0500
+++ b/test-data/branchlength_out.dat Tue Feb 03 18:34:27 2009 -0500
@@ -7,27 +7,27 @@
1 Node4 4.93827e-31 0 0.23542
1 canFam1.chr14(+):42826409-42826418|canFam1_0 4.93827e-31 0 0.23542
1 Total Tree 3.45679e-30 2.60643e-34 2.54701e-20
-2 hg17.chr7(+):26907310-26907326|hg17_1 1.48148e-31 0 0.0915385
-2 panTro1.chr6(+):28037328-28037344|panTro1_1 1.48148e-31 0 0.0915385
-2 Node1 6.69055e-19 0 0.0915385
-2 mm5.chr6(+):52104031-52104047|mm5_1 5.00368e-19 0 0.0915385
-2 rn3.chr4(+):80734404-80734420|rn3_1 0.0435526 0.00241411 0.219651
-2 Node4 1.48148e-31 0 0.0915385
-2 canFam1.chr14(+):42826418-42826434|canFam1_1 1.76058e-19 0 0.0915385
-2 Total Tree 0.0435526 0.00241411 0.219651
-3 hg17.chr7(+):26907326-26907338|hg17_2 1.70011e-31 0 0.143452
-3 panTro1.chr6(+):28037344-28037356|panTro1_2 1.70011e-31 0 0.143452
-3 Node1 1.27931e-17 0 0.143452
-3 mm5.chr6(+):52104047-52104059|mm5_2 1.70011e-31 0 0.143452
-3 rn3.chr4(+):80734420-80734432|rn3_2 0 0 0.143452
-3 Node4 0 0 0.143452
-3 canFam1.chr14(+):42826434-42826446|canFam1_2 1.70011e-31 0 0.143452
-3 Total Tree 1.27931e-17 9.64599e-22 9.4261e-08
-4 hg17.chr7(+):26907338-26907654|hg17_3 1.98974e-31 0 0.00662576
-4 panTro1.chr6(+):28037356-28037672|panTro1_3 1.98974e-31 0 0.00662576
-4 Node1 0.014806 0.00246541 0.0381554
-4 mm5.chr6(+):52104059-52104375|mm5_3 0.0275935 0.0109456 0.0548103
-4 rn3.chr4(+):80734432-80734748|rn3_3 0.0217688 0.00736645 0.0466551
-4 Node4 0.0451597 0.0224663 0.0792997
-4 canFam1.chr14(+):42826446-42826762|canFam1_3 0.0320496 0.013941 0.0605175
-4 Total Tree 0.141378 0.100447 0.192467
+2 hg17.chr7(+):26907310-26907326|hg17_1 1.90313e-31 0 0.13592
+2 panTro1.chr6(+):28037328-28037344|panTro1_1 1.90313e-31 0 0.13592
+2 Node1 4.10285e-19 0 0.13592
+2 mm5.chr6(+):52104031-52104047|mm5_1 2.02063e-19 0 0.13592
+2 rn3.chr4(+):80734404-80734420|rn3_1 0.069102 0.00372954 0.421893
+2 Node4 1.90313e-31 0 0.13592
+2 canFam1.chr14(+):42826418-42826434|canFam1_1 1.13513e-19 0 0.13592
+2 Total Tree 0.069102 0.00372954 0.421893
+3 hg17.chr7(+):26907326-26907338|hg17_2 8.84162e-30 0 0.167446
+3 panTro1.chr6(+):28037344-28037356|panTro1_2 8.84162e-30 0 0.167446
+3 Node1 0 0 0.167446
+3 mm5.chr6(+):52104047-52104059|mm5_2 0 0 0.24858
+3 rn3.chr4(+):80734420-80734432|rn3_2 0.201494 0.0309976 0.7364
+3 Node4 0.31691 0.0691821 1.08477
+3 canFam1.chr14(+):42826434-42826446|canFam1_2 0 0 0.167446
+3 Total Tree 0.518404 0.168855 1.26753
+4 hg17.chr7(+):26907338-26907654|hg17_3 1.11429e-19 0 0.0061265
+4 panTro1.chr6(+):28037356-28037672|panTro1_3 1.11866e-19 0 0.0061265
+4 Node1 0.0583134 0.0318643 0.0951333
+4 mm5.chr6(+):52104059-52104375|mm5_3 0.0357704 0.0167845 0.0644066
+4 rn3.chr4(+):80734432-80734748|rn3_3 0.0320354 0.0141331 0.0594834
+4 Node4 0.0770295 0.0463471 0.11848
+4 canFam1.chr14(+):42826446-42826762|canFam1_3 0.0575307 0.0316345 0.0936592
+4 Total Tree 0.260679 0.205255 0.325684
diff -r 4d4de72c57c8 -r a4214de3752e tools/hyphy/hyphy_branch_lengths_wrapper.xml
--- a/tools/hyphy/hyphy_branch_lengths_wrapper.xml Tue Feb 03 15:06:21 2009 -0500
+++ b/tools/hyphy/hyphy_branch_lengths_wrapper.xml Tue Feb 03 18:34:27 2009 -0500
@@ -1,4 +1,4 @@
-<?xml version="1.1.0"?>
+<?xml version="1.2.0"?>
<tool name="Branch Lengths" id="hyphy_branch_lengths_wrapper1">
<description>Estimation</description>
diff -r 4d4de72c57c8 -r a4214de3752e tools/regVariation/getIndels.py
--- a/tools/regVariation/getIndels.py Tue Feb 03 15:06:21 2009 -0500
+++ b/tools/regVariation/getIndels.py Tue Feb 03 18:34:27 2009 -0500
@@ -10,8 +10,10 @@
from galaxy import eggs
import pkg_resources
pkg_resources.require( "bx-python" )
-pkg_resources.require("numpy")
-
+try:
+ pkg_resources.require("numpy")
+except:
+ pass
import psyco_full
import sys
from bx.cookbook import doc_optparse
diff -r 4d4de72c57c8 -r a4214de3752e tools/regVariation/quality_filter.py
--- a/tools/regVariation/quality_filter.py Tue Feb 03 15:06:21 2009 -0500
+++ b/tools/regVariation/quality_filter.py Tue Feb 03 18:34:27 2009 -0500
@@ -12,7 +12,10 @@
import pkg_resources
pkg_resources.require( "bx-python" )
pkg_resources.require( "lrucache" )
-pkg_resources.require("numpy")
+try:
+ pkg_resources.require("numpy")
+except:
+ pass
import psyco_full
import sys
10 years, 7 months
[hg] galaxy 1729: Corrected a bug in branch lengths batchfile, a...
by Greg Von Kuster
details: http://www.bx.psu.edu/hg/galaxy/rev/743237dd0e65
changeset: 1729:743237dd0e65
user: guru
date: Mon Feb 02 18:57:07 2009 -0500
description:
Corrected a bug in branch lengths batchfile, as per Sergei's suggestions.
1 file(s) affected in this change:
lib/galaxy/tools/util/hyphy_util.py
diffs (306 lines):
diff -r cabfdc9dea12 -r 743237dd0e65 lib/galaxy/tools/util/hyphy_util.py
--- a/lib/galaxy/tools/util/hyphy_util.py Fri Jan 30 11:37:51 2009 -0500
+++ b/lib/galaxy/tools/util/hyphy_util.py Mon Feb 02 18:57:07 2009 -0500
@@ -368,143 +368,281 @@
BranchLengthsMF = """
VERBOSITY_LEVEL = -1;
+
fscanf (PROMPT_FOR_FILE, "Lines", inLines);
+
+
_linesIn = Columns (inLines);
+
+
/*---------------------------------------------------------*/
+
+
_currentGene = 1;
+
_currentState = 0;
+
geneSeqs = "";
+
geneSeqs * 128;
+
+
for (l=0; l<_linesIn; l=l+1)
+
{
+
if (Abs(inLines[l]) == 0)
+
{
+
if (_currentState == 1)
+
{
+
geneSeqs * 0;
+
DataSet ds = ReadFromString (geneSeqs);
+
_processAGene (_currentGene);
- geneSeqs * 128;
+
+ geneSeqs * 128;
+
_currentGene = _currentGene + 1;
+
}
+
}
+
else
+
{
+
if (_currentState == 0)
+
{
+
_currentState = 1;
+
}
+
geneSeqs * inLines[l];
+
geneSeqs * "\\n";
+
}
+
}
+
+
if (_currentState == 1)
+
{
+
geneSeqs * 0;
+
if (Abs(geneSeqs))
+
{
+
DataSet ds = ReadFromString (geneSeqs);
+
_processAGene (_currentGene);
+
}
+
}
+
+
fprintf (resultFile,CLOSE_FILE);
+
+
/*---------------------------------------------------------*/
+
+
function _processAGene (_geneID)
+
{
+
DataSetFilter filteredData = CreateFilter (ds,1);
+
if (_currentGene == 1)
+
{
+
SelectTemplateModel (filteredData);
+
- SetDialogPrompt ("Tree file");
+
+ SetDialogPrompt ("Tree file");
+
fscanf (PROMPT_FOR_FILE, "Tree", givenTree);
+
fscanf (stdin, "String", resultFile);
+
+
/* do sequence to branch map */
+
+
validNames = {};
+
taxonNameMap = {};
+
+
for (k=0; k<TipCount(givenTree); k=k+1)
+
{
+
validNames[TipName(givenTree,k)&&1] = 1;
+
}
+
+
for (k=0; k<BranchCount(givenTree); k=k+1)
+
{
+
thisName = BranchName(givenTree,k);
+
taxonNameMap[thisName&&1] = thisName;
+
}
+
+
storeValidNames = validNames;
+
fprintf (resultFile,CLEAR_FILE,KEEP_OPEN,"Block\\tBranch\\tLength\\tLowerBound\\tUpperBound\\n");
+
}
+
else
+
{
+
+ HarvestFrequencies (vectorOfFrequencies, filteredData, 1,1,1);
+
validNames = storeValidNames;
+
}
+
+
for (k=0; k<ds.species; k=k+1)
+
{
+
GetString (thisName, ds,k);
+
shortName = (thisName^{{"\\\\..+",""}})&&1;
+
if (validNames[shortName])
+
{
+
taxonNameMap[shortName] = thisName;
+
validNames - (shortName);
+
SetParameter (ds,k,shortName);
+
}
+
else
+
{
+
fprintf (resultFile,"ERROR:", thisName, " could not be matched to any of the leaves in tree ", givenTree,"\\n");
+
return 0;
+
}
+
}
+
+
/* */
+
+
LikelihoodFunction lf = (filteredData,givenTree);
+
+ Optimize (res,lf);
+
- Optimize (res,lf);
+
+ timer = Time(0)-timer;
+
- timer = Time(0)-timer;
+
+ branchNames = BranchName (givenTree,-1);
+
+ branchLengths = BranchLength (givenTree,-1);
+
- branchNames = BranchName (givenTree,-1);
- branchLengths = BranchLength (givenTree,-1);
+
+
+ for (k=0; k<Columns(branchNames)-1; k=k+1)
+
+ {
+
+ COVARIANCE_PARAMETER = "givenTree."+branchNames[k]+".t";
+
+ COVARIANCE_PRECISION = 0.95;
+
+ CovarianceMatrix (cmx,lf);
+
+ if (k==0)
+
+ {
+
+ /* compute a scaling factor */
+
+ ExecuteCommands ("givenTree."+branchNames[0]+".t=1");
+
+ scaleFactor = BranchLength (givenTree,0);
+
+ ExecuteCommands ("givenTree."+branchNames[0]+".t="+cmx[0][1]);
+
+ }
+
+ fprintf (resultFile,_geneID,"\\t",taxonNameMap[branchNames[k]&&1],"\\t",branchLengths[k],"\\t",scaleFactor*cmx[0][0],"\\t",scaleFactor*cmx[0][2],"\\n");
+
+ }
+
- for (k=0; k<Columns(branchNames)-1; k=k+1)
- {
- COVARIANCE_PARAMETER = "givenTree."+branchNames[k]+".t";
- COVARIANCE_PRECISION = 0.95;
- CovarianceMatrix (cmx,lf);
- if (k==0)
- {
- /* compute a scaling factor */
- ExecuteCommands ("givenTree."+branchNames[0]+".t=1");
- scaleFactor = BranchLength (givenTree,0);
- ExecuteCommands ("givenTree."+branchNames[0]+".t="+cmx[0][1]);
- }
- fprintf (resultFile,_geneID,"\\t",taxonNameMap[branchNames[k]&&1],"\\t",branchLengths[k],"\\t",scaleFactor*cmx[0][0],"\\t",scaleFactor*cmx[0][2],"\\n");
- }
-
+
ttl = (branchLengths*(Transpose(branchLengths["1"])))[0];
+
global treeScaler = 1;
+
ReplicateConstraint ("this1.?.t:=treeScaler*this2.?.t__",givenTree,givenTree);
+
COVARIANCE_PARAMETER = "treeScaler";
+
COVARIANCE_PRECISION = 0.95;
+
CovarianceMatrix (cmx,lf);
+
fprintf (resultFile,_geneID,"\\tTotal Tree\\t",ttl,"\\t",ttl*cmx[0][0],"\\t",ttl*cmx[0][2],"\\n");
+
ClearConstraints (givenTree);
+
return 0;
+
}
"""
10 years, 7 months
