- galaxy-dev - lists.galaxyproject.org

[hg] galaxy 1655: Bug fix for last commit to Tabular.set_meta()
by Greg Von Kuster 10 Dec '08

10 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/b1514bd5244e changeset: 1655:b1514bd5244e user: Dan Blankenberg <dan(a)bx.psu.edu> date: Tue Dec 09 15:51:12 2008 -0500 description: Bug fix for last commit to Tabular.set_meta() 1 file(s) affected in this change: lib/galaxy/datatypes/tabular.py diffs (12 lines): diff -r f77ec6315c7c -r b1514bd5244e lib/galaxy/datatypes/tabular.py --- a/lib/galaxy/datatypes/tabular.py Tue Dec 09 14:53:10 2008 -0500 +++ b/lib/galaxy/datatypes/tabular.py Tue Dec 09 15:51:12 2008 -0500 @@ -134,7 +134,7 @@ #Now we fill any unknown (None) column_types with data from first line for i in range( len( column_types ) ): if column_types[i] is None: - if first_line_column_types[i] is None: + if len( first_line_column_types ) <= i or first_line_column_types[i] is None: column_types[i] = default_column_type else: column_types[i] = first_line_column_types[i]

1 0

[hg] galaxy 1654: Update Tabular.set_meta() to use column types ...
by Greg Von Kuster 10 Dec '08

10 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/f77ec6315c7c changeset: 1654:f77ec6315c7c user: Dan Blankenberg <dan(a)bx.psu.edu> date: Tue Dec 09 14:53:10 2008 -0500 description: Update Tabular.set_meta() to use column types as guessed between rows, when column data is missing. Also better handling of the presumed header line (first line is treated as a header, but information is retained and used if needed, whereas this line would be previously tossed out.) 2 file(s) affected in this change: lib/galaxy/datatypes/interval.py lib/galaxy/datatypes/tabular.py diffs (212 lines): diff -r 06c63a161985 -r f77ec6315c7c lib/galaxy/datatypes/interval.py --- a/lib/galaxy/datatypes/interval.py Fri Dec 05 11:39:38 2008 -0500 +++ b/lib/galaxy/datatypes/interval.py Tue Dec 09 14:53:10 2008 -0500 @@ -758,6 +758,9 @@ """Initialize interval datatype, by adding UCSC display app""" Tabular.__init__(self, **kwd) self.add_display_app ( 'ucsc', 'display at UCSC', 'as_ucsc_display_file', 'ucsc_links' ) + def set_readonly_meta( self, dataset, skip=1, **kwd ): + """Resets the values of readonly metadata elements.""" + Tabular.set_readonly_meta( self, dataset, skip = skip, **kwd ) def set_meta( self, dataset, overwrite = True, **kwd ): Tabular.set_meta( self, dataset, overwrite = overwrite, skip = 1 ) def display_peek( self, dataset ): @@ -869,7 +872,11 @@ """Initialize datatype, by adding GBrowse display app""" Tabular.__init__(self, **kwd) self.add_display_app ('elegans', 'display in GBrowse', 'as_gbrowse_display_file', 'gbrowse_links' ) - + + def set_readonly_meta( self, dataset, skip=1, **kwd ): + """Resets the values of readonly metadata elements.""" + Tabular.set_readonly_meta( self, dataset, skip = skip, **kwd ) + def set_meta( self, dataset, overwrite = True, **kwd ): Tabular.set_meta( self, dataset, overwrite = overwrite, skip = 1 ) diff -r 06c63a161985 -r f77ec6315c7c lib/galaxy/datatypes/tabular.py --- a/lib/galaxy/datatypes/tabular.py Fri Dec 05 11:39:38 2008 -0500 +++ b/lib/galaxy/datatypes/tabular.py Tue Dec 09 14:53:10 2008 -0500 @@ -23,75 +23,125 @@ def init_meta( self, dataset, copy_from=None ): data.Text.init_meta( self, dataset, copy_from=copy_from ) - def set_readonly_meta( self, dataset, skip=1, **kwd ): + def set_readonly_meta( self, dataset, skip=None, **kwd ): """Resets the values of readonly metadata elements.""" Tabular.set_meta( self, dataset, overwrite = True, skip = skip ) - def set_meta( self, dataset, overwrite = True, skip = 1, **kwd ): + def set_meta( self, dataset, overwrite = True, skip = None, **kwd ): """ Tries to determine the number of columns as well as those columns that contain numerical values in the dataset. A skip parameter is used because various tabular data types reuse this function, and their data type classes are responsible to determine how many invalid - comment lines should be skipped. + comment lines should be skipped. Using None for skip will cause skip + to be zero, but the first line will be processed as a header. """ #we treat 'overwrite' as always True (we always want to set tabular metadata when called) + #if a tabular file has no data, it will have one column of type str + + num_check_lines = 100 #we will only check up to this many lines into the file + requested_skip = skip #store original skip value to check with later + if skip is None: + skip = 0 + + column_type_set_order = [ 'int', 'float', 'list', 'str' ] #Order to set column types in + default_column_type = column_type_set_order[-1] # Default column type is lowest in list + column_type_compare_order = list( column_type_set_order ) #Order to compare column types + column_type_compare_order.reverse() + def type_overrules_type( column_type1, column_type2 ): + if column_type1 is None or column_type1 == column_type2: + return False + if column_type2 is None: + return True + for column_type in column_type_compare_order: + if column_type1 == column_type: + return True + if column_type2 == column_type: + return False + #neither column type was found in our ordered list, this cannot happen + raise "Tried to compare unknown column types" + def is_int( column_text ): + try: + int( column_text ) + return True + except: + return False + def is_float( column_text ): + try: + float( column_text ) + return True + except: + if column_text.strip().lower() == 'na': + return True #na is special cased to be a float + return False + def is_list( column_text ): + return "," in column_text + def is_str( column_text ): + #anything, except an empty string, is True + if column_text == "": + return False + return True + is_column_type = {} #Dict to store column type string to checking function + for column_type in column_type_set_order: + is_column_type[column_type] = locals()[ "is_%s" % ( column_type ) ] + def guess_column_type( column_text ): + for column_type in column_type_set_order: + if is_column_type[column_type]( column_text ): + return column_type + return None + + column_types = [] + first_line_column_types = [default_column_type] # default value is one column of type str if dataset.has_data(): - column_types = [] - + #NOTE: if skip > num_check_lines, we won't detect any metadata, and will use default for i, line in enumerate( file ( dataset.file_name ) ): - if i < skip: + line = line.rstrip('\r\n') + if i < skip or not line or line.startswith( '#' ): continue - line = line.rstrip('\r\n') - if line and not line.startswith( '#' ): - elems = line.split( '\t' ) - elems_len = len( elems ) - if elems_len > 0: - # Set the columns metadata attribute - if elems_len != dataset.metadata.columns: - dataset.metadata.columns = elems_len - # Set the column_types metadata attribute - for col in range( 0, elems_len ): - col_type = None - val = elems[ col ] - if not val: - if i == 100: - # We're about to end our loop, so default col_type to 'str' - col_type = 'str' - else: - # Missing a column value, so go to the next line - column_types = [] - break - if not col_type and val.find( '.' ) < 0: - try: - int( val ) - col_type = 'int' - except: - pass - if not col_type: - try: - float( val ) - col_type = 'float' - except: - if val.strip().lower() == 'na': - col_type = 'float' - if not col_type: - val_elems = val.split( ',' ) - if len( val_elems ) > 1: - col_type = 'list' - if not col_type: - # All parameters are strings, so this will be the default - col_type = 'str' - if col_type: - column_types.append( col_type ) - else: - # Couldn't determine column type, so go to the next line - column_types = [] - break - if column_types: - break - if i > 100: - break # Hopefully we never get here... - dataset.metadata.column_types = column_types + + fields = line.split( '\t' ) + for field_count, field in enumerate( fields ): + if field_count >= len( column_types ): #found a previously unknown column, we append None + column_types.append( None ) + column_type = guess_column_type( field ) + if type_overrules_type( column_type, column_types[field_count] ): + column_types[field_count] = column_type + + if i == 0 and requested_skip is None: + #this is our first line, people seem to like to upload files that have a header line, but do not start with '#' (i.e. all column types would then most likely be detected as str) + #we will assume that the first line is always a header (this was previous behavior - it was always skipped) when the requested skip is None + #we only use the data from the first line if we have no other data for a column + #this is far from perfect, as: + #1,2,3 1.1 2.2 qwerty + #0 0 1,2,3 + #will detect as + #"column_types": ["int", "int", "float", "list"] + #instead of: + #"column_types": ["list", "float", "float", "str"] *** would seem to be the 'Truth' by manual observation that the first line should be included as data + #old method would have detected as: + #"column_types": ["int", "int", "str", "list"] + first_line_column_types = column_types + column_types = [ None for col in first_line_column_types ] + elif ( column_types and None not in column_types ) or i > num_check_lines: + #found and set all known columns, or we exceeded our max check lines + break + + #we error on the larger number of columns + #first we pad our column_types by using data from first line + if len( first_line_column_types ) > len( column_types ): + for column_type in first_line_column_types[len( column_types ):]: + column_types.append( column_type ) + + #Now we fill any unknown (None) column_types with data from first line + for i in range( len( column_types ) ): + if column_types[i] is None: + if first_line_column_types[i] is None: + column_types[i] = default_column_type + else: + column_types[i] = first_line_column_types[i] + + dataset.metadata.column_types = column_types + dataset.metadata.columns = len( column_types ) + def make_html_table( self, dataset, skipchars=[] ): """Create HTML table, used for displaying peek""" out = ['<table cellspacing="0" cellpadding="3">']

1 0

[hg] galaxy 1652: Forgot to change the test data for the gsummar...
by Greg Von Kuster 10 Dec '08

10 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/ef029a5a6684 changeset: 1652:ef029a5a6684 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Fri Dec 05 09:58:23 2008 -0500 description: Forgot to change the test data for the gsummary tool. 1 file(s) affected in this change: test-data/gsummary_out1.tabular diffs (8 lines): diff -r 1734db81efb0 -r ef029a5a6684 test-data/gsummary_out1.tabular --- a/test-data/gsummary_out1.tabular Fri Dec 05 09:01:22 2008 -0500 +++ b/test-data/gsummary_out1.tabular Fri Dec 05 09:58:23 2008 -0500 @@ -1,2 +1,2 @@ -#sum mean stdev 0% 25% 50% 75% 100% -5559213881.000 85526367.400 53783882.151 142908.000 33593260.000 108299600.000 128764156.000 220229609.000 +#sum mean stdev 0% 25% 50% 75% 100% +5.55921e+09 8.55264e+07 5.37839e+07 142908 3.35933e+07 1.083e+08 1.28764e+08 2.2023e+08

1 0

[hg] galaxy 1651: Increase the version in the grouping tool due ...
by Greg Von Kuster 05 Dec '08

05 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/1734db81efb0 changeset: 1651:1734db81efb0 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Fri Dec 05 09:01:22 2008 -0500 description: Increase the version in the grouping tool due to previously changed output, change the output and increase the version of the gsummary tool. 3 file(s) affected in this change: tools/stats/grouping.xml tools/stats/gsummary.py tools/stats/gsummary.xml diffs (30 lines): diff -r 6041065f86a9 -r 1734db81efb0 tools/stats/grouping.xml --- a/tools/stats/grouping.xml Thu Dec 04 15:09:57 2008 -0500 +++ b/tools/stats/grouping.xml Fri Dec 05 09:01:22 2008 -0500 @@ -1,4 +1,4 @@ -<tool id="Grouping1" name="Group" version="1.2.0"> +<tool id="Grouping1" name="Group" version="1.3.0"> <description>data by a column and perform aggregate operation on other columns.</description> <command interpreter="python"> grouping.py diff -r 6041065f86a9 -r 1734db81efb0 tools/stats/gsummary.py --- a/tools/stats/gsummary.py Thu Dec 04 15:09:57 2008 -0500 +++ b/tools/stats/gsummary.py Fri Dec 05 09:01:22 2008 -0500 @@ -100,7 +100,7 @@ stop_err( "Computation resulted in the following error: %s" % str( s ) ) summary = summary.as_py( BASIC_CONVERSION ) outfile.write( "#%s\n" % headings_str ) - outfile.write( "%s\n" % "\t".join( [ "%.3f" % ( summary[ k ] ) for k in headings ] ) ) + outfile.write( "%s\n" % "\t".join( [ "%g" % ( summary[ k ] ) for k in headings ] ) ) outfile.close() if skipped_lines: diff -r 6041065f86a9 -r 1734db81efb0 tools/stats/gsummary.xml --- a/tools/stats/gsummary.xml Thu Dec 04 15:09:57 2008 -0500 +++ b/tools/stats/gsummary.xml Fri Dec 05 09:01:22 2008 -0500 @@ -1,4 +1,4 @@ -<tool id="Summary_Statistics1" name="Summary Statistics"> +<tool id="Summary_Statistics1" name="Summary Statistics" version="1.1.0"> <description>for any numerical column</description> <command interpreter="python">gsummary.py $input $out_file1 "$cond"</command> <inputs>

1 0

[hg] galaxy 1650: Update GMAJ help.
by Greg Von Kuster 05 Dec '08

05 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/6041065f86a9 changeset: 1650:6041065f86a9 user: Dan Blankenberg <dan(a)bx.psu.edu> date: Thu Dec 04 15:09:57 2008 -0500 description: Update GMAJ help. 1 file(s) affected in this change: tools/visualization/GMAJ.xml diffs (33 lines): diff -r 8b9cd2a1d5c2 -r 6041065f86a9 tools/visualization/GMAJ.xml --- a/tools/visualization/GMAJ.xml Thu Dec 04 11:40:42 2008 -0500 +++ b/tools/visualization/GMAJ.xml Thu Dec 04 15:09:57 2008 -0500 @@ -176,13 +176,25 @@ <data name="out_file1" format="gmaj.zip"/> </outputs> <help> -You can use this tool to view a set of MAF alignments. You may also include optional annotation data for one or more of the organisms. +.. class:: infomark + +**Reference Sequence:** +The default option, "First sequence in each block", is the correct choice for the vast majority of MAF alignments. The alternative, "Any sequence", will allow you to flip the blocks to view them with any of the MAF sequences as the reference, but this is only appropriate if the file was generated by a sequence-symmetric alignment program such as TBA_. Using "Any sequence" with an ordinary MAF will **not** give the same results as if that alignment had been run with a different reference sequence. + +.. class:: infomark + +**Annotation Style:** +The default style, "Galaxy", specifies one set of annotations for each species in the MAF file; it assumes that if you have, say, exons for several chromosomes of one species, they are all together in one file. The other style, "Basic", is more flexible but cumbersome: a separate set of files is specified for each sequence (e.g. chromosome), and you must fill in the full sequence name as it appears in the MAF. The Basic style also allows you to provide a display offset that GMAJ will add to all of the position labels for that sequence. With either style, specifying more than one set of annotations for the same sequence will result in an error message from GMAJ. + +---- + +**What it does** + +GMAJ is an interactive viewer for MAF alignments, with support for optional annotation data. In addition to browsing the alignments, you can select and export them according to a variety of criteria and send the output back to your Galaxy history. For detailed information on GMAJ, click here_. .. _here: /static/gmaj/docs/gmaj_readme.html - -Gmaj is a tool for viewing and manipulating Generalized Multiple Alignments (GMAs) produced by programs such as TBA (though it can also be used with maf-format alignments from other sources). It can display interactive graphical and text representations of the alignments, a diagram showing the locations of exons and repeats, and other annotations -- all with the user's choice of reference sequence. - +.. _TBA: http://www.bx.psu.edu/miller_lab/ </help> </tool>

1 0

[hg] galaxy 1649: Fix a couple outdated download urls
by Nate Coraor 04 Dec '08

04 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/8b9cd2a1d5c2 changeset: 1649:8b9cd2a1d5c2 user: Nate Coraor <nate(a)bx.psu.edu> date: Thu Dec 04 11:40:42 2008 -0500 description: Fix a couple outdated download urls 1 file(s) affected in this change: eggs.ini diffs (24 lines): diff -r 682c179a2f84 -r 8b9cd2a1d5c2 eggs.ini --- a/eggs.ini Thu Dec 04 10:47:41 2008 -0500 +++ b/eggs.ini Thu Dec 04 11:40:42 2008 -0500 @@ -52,7 +52,7 @@ [tags] psycopg2 = _8.2.6_static pysqlite = _3.5.4_static -MySQL_python = _5.0.51a_static +MySQL_python = _5.0.67_static python_lzo = _static flup = .dev_r2311 bx_python = _dev_r456 @@ -61,9 +61,9 @@ ; source location, necessary for scrambling [source] bx_python = http://dist.g2.bx.psu.edu/bx-python_dist-r456.tar.bz2 -Cheetah = http://umn.dl.sourceforge.net/sourceforge/cheetahtemplate/Cheetah-1.0.tar.gz +Cheetah = http://voxel.dl.sourceforge.net/sourceforge/cheetahtemplate/Cheetah-1.0.tar… DRMAA_python = http://gridengine.sunsource.net/files/documents/7/36/DRMAA-python-0.2.tar.gz -MySQL_python = http://superb-west.dl.sourceforge.net/sourceforge/mysql-python/MySQL-python… http://mysql.mirrors.pair.com/Downloads/MySQL-5.0/mysql-5.0.51a.tar.gz +MySQL_python = http://superb-west.dl.sourceforge.net/sourceforge/mysql-python/MySQL-python… http://mysql.mirrors.pair.com/Downloads/MySQL-5.0/mysql-5.0.67.tar.gz pbs_python = http://ftp.sara.nl/pub/outgoing/pbs_python-2.9.4.tar.gz psycopg2 = http://initd.org/pub/software/psycopg/PSYCOPG-2-0/psycopg2-2.0.6.tar.gz http://ftp8.us.postgresql.org/postgresql/source/v8.2.6/postgresql-8.2.6.tar… pycrypto = http://www.amk.ca/files/python/crypto/pycrypto-2.0.1.tar.gz

1 0

[hg] galaxy 1645: imported patch job-queue-sessions
by Nate Coraor 04 Dec '08

04 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/1dc78a13070c changeset: 1645:1dc78a13070c user: James Taylor <james(a)jamestaylor.org> date: Tue Dec 02 13:49:29 2008 -0500 description: imported patch job-queue-sessions 1 file(s) affected in this change: lib/galaxy/jobs/__init__.py diffs (229 lines): diff -r ab46f22b4624 -r 1dc78a13070c lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py Tue Dec 02 11:41:25 2008 -0500 +++ b/lib/galaxy/jobs/__init__.py Tue Dec 02 13:49:29 2008 -0500 @@ -63,9 +63,8 @@ def __init__( self, app, dispatcher ): """Start the job manager""" self.app = app - # Should we use IPC to communicate (needed if forking) + # Should we read jobs form the database, or use an in memory queue self.track_jobs_in_database = app.config.get_bool( 'track_jobs_in_database', False ) - # Check if any special scheduling policy should be used. If not, default is FIFO. sched_policy = app.config.get('job_scheduler_policy', 'FIFO') # Parse the scheduler policy string. The policy class implements a special queue. @@ -104,13 +103,13 @@ self.sleeper = Sleeper() self.running = True self.dispatcher = dispatcher - self.monitor_thread = threading.Thread( target=self.monitor ) + self.monitor_thread = threading.Thread( target=self.__monitor ) self.monitor_thread.start() log.info( "job manager started" ) if app.config.get_bool( 'enable_job_recovery', True ): - self.check_jobs_at_startup() + self.__check_jobs_at_startup() - def check_jobs_at_startup( self ): + def __check_jobs_at_startup( self ): """ Checks all jobs that are in the 'running' or 'queued' state in the database and requeues or cleans up as necessary. Only run as the @@ -125,10 +124,10 @@ for job in model.Job.filter( (model.Job.c.state == model.Job.states.RUNNING) | (model.Job.c.state == model.Job.states.QUEUED) ).all(): if job.job_runner_name is not None: # why are we passing the queue to the wrapper? - job_wrapper = JobWrapper( job.id, self.app.toolbox.tools_by_id[ job.tool_id ], self ) + job_wrapper = JobWrapper( job, self.app.toolbox.tools_by_id[ job.tool_id ], self ) self.dispatcher.recover( job, job_wrapper ) - def monitor( self ): + def __monitor( self ): """ Continually iterate the waiting jobs, checking is each is ready to run and dispatching if so. @@ -137,13 +136,13 @@ time.sleep( 10 ) while self.running: try: - self.monitor_step() + self.__monitor_step() except: log.exception( "Exception in monitor_step" ) # Sleep self.sleeper.sleep( 1 ) - def monitor_step( self ): + def __monitor_step( self ): """ Called repeatedly by `monitor` to process waiting jobs. Gets any new jobs (either from the database or from its own queue), then iterates @@ -153,12 +152,13 @@ it is marked as having errors and removed from the queue. Otherwise, the job is dispatched. """ + # Get an orm session + session = mapping.Session() # Pull all new jobs from the queue at once new_jobs = [] if self.track_jobs_in_database: - model = self.app.model - for j in model.Job.filter( model.Job.c.state==model.Job.states.NEW ).all(): - job = JobWrapper( j.id, self.app.toolbox.tools_by_id[ j.tool_id ], self ) + for j in session.query( model.Job ).filter( model.Job.c.state == model.Job.states.NEW ).all(): + job = JobWrapper( j, self.app.toolbox.tools_by_id[ j.tool_id ], self ) new_jobs.append( job ) else: try: @@ -169,18 +169,26 @@ # Unpack the message job_id, tool_id = message # Create a job wrapper from it - job = JobWrapper( job_id, self.app.toolbox.tools_by_id[ tool_id ], self ) + job_entity = session.query( model.Job ).get( job_id ) + job = JobWrapper( job_entity, self.app.toolbox.tools_by_id[ tool_id ], self ) # Append to watch queue new_jobs.append( job ) except Empty: - pass + pass # Iterate over new and waiting jobs and look for any that are # ready to run new_waiting = [] for job in ( new_jobs + self.waiting ): try: + # Clear the session for each job so we get fresh states for + # job and all datasets + session.clear() + # Get the real job entity corresponding to the wrapper (if we + # are tracking in the database this is probably cached in + # the session from the origianl query above) + job_entity = session.query( model.Job ).get( job.job_id ) # Check the job's dependencies, requeue if they're not done - job_state = job.check_if_ready_to_run() + job_state = self.__check_if_ready_to_run( job, job_entity ) if job_state == JOB_WAIT: if not self.track_jobs_in_database: new_waiting.append( job ) @@ -195,7 +203,7 @@ if self.use_policy : self.squeue.put( job ) log.debug( "job %d put in policy queue" % job.job_id ) - else : # or dispatch the job directly + else: # or dispatch the job directly self.dispatcher.put( job ) log.debug( "job %d dispatched" % job.job_id) elif job_state == JOB_DELETED: @@ -225,6 +233,43 @@ except Exception, e: # if something else breaks while dispatching job.fail( "failure running job %d: %s" % ( sjob.job_id, str( e ) ) ) log.exception( "failure running job %d" % sjob.job_id ) + # Done with the session + mapping.Session.remove() + + def __check_if_ready_to_run( self, job_wrapper, job ): + """ + Check if a job is ready to run by verifying that each of its input + datasets is ready (specifically in the OK state). If any input dataset + has an error, fail the job and return JOB_INPUT_ERROR. If any input + dataset is deleted, fail the job and return JOB_INPUT_DELETED. If all + input datasets are in OK state, return JOB_READY indicating that the + job can be dispatched. Otherwise, return JOB_WAIT indicating that input + datasets are still being prepared. + """ + if self.app.memory_usage: + # Keep track of memory usage + m0 = self.app.memory_usage.memory() + for dataset_assoc in job.input_datasets: + idata = dataset_assoc.dataset + if not idata: + continue + # don't run jobs for which the input dataset was deleted + if idata.deleted: + job_wrapper.fail( "input data %d (file: %s) was deleted before the job started" % ( idata.hid, idata.file_name ) ) + return JOB_INPUT_DELETED + # an error in the input data causes us to bail immediately + elif idata.state == idata.states.ERROR: + job_wrapper.fail( "input data %d (file: %s) is in an error state" % ( idata.hid, idata.file_name ) ) + return JOB_INPUT_ERROR + elif idata.state != idata.states.OK: + # need to requeue + return JOB_WAIT + if self.app.memory_usage: + m1 = self.app.memory_usage.memory( m0, pretty=True ) + log.info("End of check_if_ready_to_run for job id %d, memory used increased by %s" % ( job.id, m1 ) ) + if job.state == model.Job.states.DELETED: + return JOB_DELETED + return JOB_READY def put( self, job_id, tool ): """Add a job to the queue (by job identifier)""" @@ -251,8 +296,10 @@ Wraps a 'model.Job' with convience methods for running processes and state management. """ - def __init__(self, job_id, tool, queue ): - self.job_id = job_id + def __init__(self, job, tool, queue ): + self.job_id = job.id + # This is immutable, we cache it for the scheduling policy to use if needed + self.session_id = job.session_id self.tool = tool self.queue = queue self.app = queue.app @@ -401,45 +448,6 @@ job.job_runner_name = runner_url job.job_runner_external_id = external_id job.flush() - - def check_if_ready_to_run( self ): - """ - Check if a job is ready to run by verifying that each of its input - datasets is ready (specifically in the OK state). If any input dataset - has an error, fail the job and return JOB_INPUT_ERROR. If any input - dataset is deleted, fail the job and return JOB_INPUT_DELETED. If all - input datasets are in OK state, return JOB_READY indicating that the - job can be dispatched. Otherwise, return JOB_WAIT indicating that input - datasets are still being prepared. - """ - if self.app.memory_usage: - # Keep track of memory usage - m0 = self.app.memory_usage.memory() - job = model.Job.get( self.job_id ) - job.refresh() - for dataset_assoc in job.input_datasets: - idata = dataset_assoc.dataset - if not idata: - continue - idata.refresh() - idata.dataset.refresh() #we need to refresh the base Dataset, since that is where 'state' is stored - # don't run jobs for which the input dataset was deleted - if idata.deleted: - self.fail( "input data %d (file: %s) was deleted before the job started" % ( idata.hid, idata.file_name ) ) - return JOB_INPUT_DELETED - # an error in the input data causes us to bail immediately - elif idata.state == idata.states.ERROR: - self.fail( "input data %d (file: %s) is in an error state" % ( idata.hid, idata.file_name ) ) - return JOB_INPUT_ERROR - elif idata.state != idata.states.OK: - # need to requeue - return JOB_WAIT - if self.app.memory_usage: - m1 = self.app.memory_usage.memory( m0, pretty=True ) - log.info("End of check_if_ready_to_run for job id %d, memory used increased by %s" % ( job.id, m1 ) ) - if job.state == model.Job.states.DELETED: - return JOB_DELETED - return JOB_READY def finish( self, stdout, stderr ): """ @@ -538,8 +546,7 @@ return self.command_line def get_session_id( self ): - job = model.Job.get( self.job_id ) - return job.session_id + return self.session_id def get_input_fnames( self ): job = model.Job.get( self.job_id )

1 0

[hg] galaxy 1647: allow users to specify that label for input da...
by Nate Coraor 04 Dec '08

04 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/15bf910890d5 changeset: 1647:15bf910890d5 user: James Taylor <james(a)jamestaylor.org> date: Tue Dec 02 17:17:23 2008 -0500 description: allow users to specify that label for input dataset modules. 4 file(s) affected in this change: lib/galaxy/web/controllers/workflow.py lib/galaxy/workflow/modules.py templates/workflow/editor_generic_form.mako templates/workflow/editor_tool_form.mako diffs (140 lines): diff -r f242b5a3fedc -r 15bf910890d5 lib/galaxy/web/controllers/workflow.py --- a/lib/galaxy/web/controllers/workflow.py Tue Dec 02 13:49:32 2008 -0500 +++ b/lib/galaxy/web/controllers/workflow.py Tue Dec 02 17:17:23 2008 -0500 @@ -166,7 +166,7 @@ return trans.fill_template( "workflow/editor.mako", workflow_id=id ) @web.json - def editor_tool_form( self, trans, type='tool', tool_id=None, **incoming ): + def editor_form_post( self, trans, type='tool', tool_id=None, **incoming ): """ Accepts a tool state and incoming values, and generates a new tool form and some additional information, packed into a json dictionary. diff -r f242b5a3fedc -r 15bf910890d5 lib/galaxy/workflow/modules.py --- a/lib/galaxy/workflow/modules.py Tue Dec 02 13:49:32 2008 -0500 +++ b/lib/galaxy/workflow/modules.py Tue Dec 02 17:17:23 2008 -0500 @@ -5,6 +5,7 @@ from galaxy.tools import DefaultToolState from galaxy.tools.parameters.grouping import Repeat, Conditional from galaxy.util.bunch import Bunch +from galaxy.util.json import from_json_string, to_json_string class WorkflowModule( object ): @@ -78,20 +79,48 @@ class InputDataModule( WorkflowModule ): type = "data_input" name = "Input dataset" - _runtime_inputs = { - 'input' : DataToolParameter( None, Element( "param", name="input", label="Input Dataset", type="data", format="data" ) ) - } - + + @classmethod + def new( Class, trans, tool_id=None ): + module = Class( trans ) + module.state = dict( name="Input Dataset" ) + return module + @classmethod + def from_dict( Class, trans, d ): + module = Class( trans ) + state = from_json_string( d["tool_state"] ) + module.state = dict( name=state.get( "name", "Input Dataset" ) ) + return module + @classmethod + def from_workflow_step( Class, trans, step ): + module = Class( trans ) + module.state = dict( name="Input Dataset" ) + if step.tool_inputs and "name" in step.tool_inputs: + module.state['name'] = step.tool_inputs[ 'name' ] + return module + def save_to_step( self, step ): + step.type = self.type + step.tool_id = None + step.tool_inputs = self.state + def get_data_inputs( self ): return [] def get_data_outputs( self ): return [ dict( name='output', extension='input' ) ] def get_config_form( self ): + form = web.FormBuilder( title=self.name ) \ + .add_text( "name", "Name", value=self.state['name'] ) return self.trans.fill_template( "workflow/editor_generic_form.mako", - form = web.FormBuilder( title=self.name ) ) + module=self, form=form ) + def get_state( self ): + return to_json_string( self.state ) + + def update_state( self, incoming ): + self.state['name'] = incoming.get( 'name', 'Input Dataset' ) def get_runtime_inputs( self ): - return self._runtime_inputs + label = self.state.get( "name", "Input Dataset" ) + return dict( input=DataToolParameter( None, Element( "param", name="input", label=label, type="data", format="data" ) ) ) def get_runtime_state( self ): state = DefaultToolState() state.inputs = dict( input=None ) @@ -106,7 +135,7 @@ return state def update_runtime_state( self, trans, state, values ): errors = {} - for name, param in self._runtime_inputs.iteritems(): + for name, param in self.get_runtime_inputs().iteritems(): value, error = check_param( trans, param, values.get( name, None ), values ) state.inputs[ name ] = value if error: @@ -146,7 +175,6 @@ tool_id = step.tool_id module = Class( trans, tool_id ) module.state = DefaultToolState() - print step.tool_inputs module.state.inputs = module.tool.params_from_strings( step.tool_inputs, trans.app, ignore_errors=True ) module.errors = step.tool_errors return module @@ -229,4 +257,4 @@ type = step.type return self.module_types[type].from_workflow_step( trans, step ) -module_factory = WorkflowModuleFactory( dict( data_input=InputDataModule, tool=ToolModule ) ) \ No newline at end of file +module_factory = WorkflowModuleFactory( dict( data_input=InputDataModule, tool=ToolModule ) ) diff -r f242b5a3fedc -r 15bf910890d5 templates/workflow/editor_generic_form.mako --- a/templates/workflow/editor_generic_form.mako Tue Dec 02 13:49:32 2008 -0500 +++ b/templates/workflow/editor_generic_form.mako Tue Dec 02 17:17:23 2008 -0500 @@ -1,8 +1,8 @@ <div class="toolForm"> <div class="toolFormTitle">${form.title}</div> <div class="toolFormBody"> - <form name="$form.name" action="${h.url_for(form.action)}" method="post"> - <table cellpadding="0" cellspacing="0" border="0"> + <form name="${form.name}" action="${h.url_for( action='editor_form_post' )}" method="post"> + <input type="hidden" name="type" value="${module.type}" /> %if form.inputs: %for input in form.inputs: <% @@ -33,9 +33,9 @@ </div> %endfor - <tr><td></td><td><input type="submit" value="${form.submit_text}"> + <div class="form-row"><input type="submit" value="${form.submit_text}"></div> %else: - <tr><td colspan="2"><i>No options</i></td></tr> + <div class="form-row"><i>No options</i></div> %endif </table> </form> diff -r f242b5a3fedc -r 15bf910890d5 templates/workflow/editor_tool_form.mako --- a/templates/workflow/editor_tool_form.mako Tue Dec 02 13:49:32 2008 -0500 +++ b/templates/workflow/editor_tool_form.mako Tue Dec 02 17:17:23 2008 -0500 @@ -56,7 +56,7 @@ <div class="toolForm"> <div class="toolFormTitle">Tool: ${tool.name}</div> <div class="toolFormBody"> - <form method="get" action="${h.url_for( action='editor_tool_form' )}"> + <form method="post" action="${h.url_for( action='editor_form_post' )}"> <input type="hidden" name="tool_id" value="${tool.id}" /> %for i, inputs in enumerate( tool.inputs_by_page ): %if tool.has_multiple_pages:

1 0

[hg] galaxy 1646: imported patch job-eager-load
by Nate Coraor 04 Dec '08

04 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/f242b5a3fedc changeset: 1646:f242b5a3fedc user: James Taylor <james(a)jamestaylor.org> date: Tue Dec 02 13:49:32 2008 -0500 description: imported patch job-eager-load 1 file(s) affected in this change: lib/galaxy/model/mapping.py diffs (28 lines): diff -r 1dc78a13070c -r f242b5a3fedc lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Tue Dec 02 13:49:29 2008 -0500 +++ b/lib/galaxy/model/mapping.py Tue Dec 02 13:49:32 2008 -0500 @@ -305,19 +305,19 @@ ) ) assign_mapper( context, JobToInputDatasetAssociation, JobToInputDatasetAssociation.table, - properties=dict( job=relation( Job ), dataset=relation( HistoryDatasetAssociation ) ) ) + properties=dict( job=relation( Job ), dataset=relation( HistoryDatasetAssociation, lazy=False ) ) ) assign_mapper( context, JobToOutputDatasetAssociation, JobToOutputDatasetAssociation.table, - properties=dict( job=relation( Job ), dataset=relation( HistoryDatasetAssociation ) ) ) + properties=dict( job=relation( Job ), dataset=relation( HistoryDatasetAssociation, lazy=False ) ) ) assign_mapper( context, JobParameter, JobParameter.table ) assign_mapper( context, Job, Job.table, properties=dict( galaxy_session=relation( GalaxySession ), history=relation( History ), - parameters=relation( JobParameter ), - input_datasets=relation( JobToInputDatasetAssociation ), - output_datasets=relation( JobToOutputDatasetAssociation ) ) ) + parameters=relation( JobParameter, lazy=False ), + input_datasets=relation( JobToInputDatasetAssociation, lazy=False ), + output_datasets=relation( JobToOutputDatasetAssociation, lazy=False ) ) ) assign_mapper( context, Event, Event.table, properties=dict( history=relation( History ),

1 0

[hg] galaxy 1644: Bug fixes for compare, Group and filter tools.
by Nate Coraor 04 Dec '08

04 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/ab46f22b4624 changeset: 1644:ab46f22b4624 user: guru date: Tue Dec 02 11:41:25 2008 -0500 description: Bug fixes for compare, Group and filter tools. 3 file(s) affected in this change: tools/filters/compare.xml tools/stats/filtering.py tools/stats/grouping.py diffs (51 lines): diff -r 74e392ea6ae8 -r ab46f22b4624 tools/filters/compare.xml --- a/tools/filters/compare.xml Tue Dec 02 10:50:54 2008 -0500 +++ b/tools/filters/compare.xml Tue Dec 02 11:41:25 2008 -0500 @@ -3,9 +3,13 @@ <command interpreter="python">joinWrapper.py $input1 $input2 $field1 $field2 $mode $out_file1</command> <inputs> <param format="tabular" name="input1" type="data" label="Compare"/> - <param name="field1" label="Using column" type="data_column" data_ref="input1" /> + <param name="field1" label="Using column" type="data_column" data_ref="input1"> + <validator type="no_options" message="Invalid column choice. Please try again after editing metadata of your input dataset by clicking on the pencil icon next to it."/> + </param> <param format="tabular" name="input2" type="data" label="against" /> - <param name="field2" label="and column" type="data_column" data_ref="input2" /> + <param name="field2" label="and column" type="data_column" data_ref="input2"> + <validator type="no_options" message="Invalid column choice. Please try again after editing metadata of your input dataset by clicking on the pencil icon next to it."/> + </param> <param name="mode" type="select" label="To find" help="See examples below for explanation of these options"> <option value="N">Matching rows of 1st query</option> <option value="V">Non Matching rows of 1st query</option> diff -r 74e392ea6ae8 -r ab46f22b4624 tools/stats/filtering.py --- a/tools/stats/filtering.py Tue Dec 02 10:50:54 2008 -0500 +++ b/tools/stats/filtering.py Tue Dec 02 11:41:25 2008 -0500 @@ -25,6 +25,7 @@ cond_text = sys.argv[3] try: in_columns = int( sys.argv[4] ) + assert sys.argv[5] #check to see that the column types varaible isn't null in_column_types = sys.argv[5].split( ',' ) except: stop_err( "Data does not appear to be tabular. This tool can only be used with tab-delimited data." ) diff -r 74e392ea6ae8 -r ab46f22b4624 tools/stats/grouping.py --- a/tools/stats/grouping.py Tue Dec 02 10:50:54 2008 -0500 +++ b/tools/stats/grouping.py Tue Dec 02 11:41:25 2008 -0500 @@ -128,7 +128,7 @@ if op not in ['c','length','unique','random']: for j, elem in enumerate( prev_vals[i] ): prev_vals[i][j] = float( elem ) - rout = "%f" %( eval( rfunc )( prev_vals[i] )) + rout = "%g" %( eval( rfunc )( prev_vals[i] )) if rounds[i] == 'yes': rout = int(round(float(rout))) else: @@ -179,7 +179,7 @@ if op not in ['c','length','unique','random']: for j, elem in enumerate( prev_vals[i] ): prev_vals[i][j] = float( elem ) - rout = '%f' %( eval( rfunc )( prev_vals[i] )) + rout = '%g' %( eval( rfunc )( prev_vals[i] )) if rounds[i] == 'yes': rout = int(round(float(rout))) else:

1 0