- galaxy-dev - lists.galaxyproject.org

[hg] galaxy 1701: Tool to find the least common ancestor added u...
by Greg Von Kuster 13 Jan '09

13 Jan '09

details: http://www.bx.psu.edu/hg/galaxy/rev/c3a20d503bd3 changeset: 1701:c3a20d503bd3 user: guru date: Tue Jan 13 12:18:08 2009 -0500 description: Tool to find the least common ancestor added under Taxonomy section. 2 file(s) affected in this change: tools/taxonomy/lca.py tools/taxonomy/lca.xml diffs (166 lines): diff -r 03f1e9e78f74 -r c3a20d503bd3 tools/taxonomy/lca.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/taxonomy/lca.py Tue Jan 13 12:18:08 2009 -0500 @@ -0,0 +1,145 @@ +#!/usr/bin/env python +#Guruprasad Ananda +""" +This tool provides the SQL "group by" functionality. +""" +import sys, string, re, commands, tempfile, random +#from rpy import * + +def stop_err(msg): + sys.stderr.write(msg) + sys.exit() + +def main(): + try: + inputfile = sys.argv[1] + outfile = sys.argv[2] + except: + stop_err("Syntax error: Use correct syntax: program infile outfile") + group_col = 0 + + tmpfile = tempfile.NamedTemporaryFile() + + try: + """ + The -k option for the Posix sort command is as follows: + -k, --key=POS1[,POS2] + start a key at POS1, end it at POS2 (origin 1) + In other words, column positions start at 1 rather than 0, so + we need to add 1 to group_col. + if POS2 is not specified, the newer versions of sort will consider the entire line for sorting. To prevent this, we set POS2=POS1. + """ + command_line = "sort -f -k " + str(group_col+1) +"," + str(group_col+1) + " -o " + tmpfile.name + " " + inputfile + except Exception, exc: + stop_err( 'Initialization error -> %s' %str(exc) ) + + error_code, stdout = commands.getstatusoutput(command_line) + + if error_code != 0: + stop_err( "Sorting input dataset resulted in error: %s: %s" %( error_code, stdout )) + + prev_item = "" + prev_vals = [] + remaining_vals = [] + skipped_lines = 0 + first_invalid_line = 0 + invalid_line = '' + invalid_value = '' + invalid_column = 0 + fout = open(outfile, "w") + cols = range(1,25) + block_valid = False + + for ii, line in enumerate( file( tmpfile.name )): + if line and not line.startswith( '#' ): + line = line.rstrip( '\r\n' ) + try: + fields = line.split("\t") + item = fields[group_col] + if prev_item != "": + # At this level, we're grouping on values (item and prev_item) in group_col + if item == prev_item: + # Keep iterating and storing values until a new value is encountered. + if block_valid: + for i, col in enumerate(cols): + if col >= 3: + prev_vals[i].append(fields[col].strip()) + if len(set(prev_vals[i])) > 1: + block_valid = False + break + + else: + """ + When a new value is encountered, write the previous value and the + corresponding aggregate values into the output file. This works + due to the sort on group_col we've applied to the data above. + """ + out_list = ['']*25 + out_list[0] = str(prev_item) + out_list[1] = str(prev_vals[0][0]) + out_list[2] = str(prev_vals[1][0]) + out_list[24] = str(prev_vals[23][0]) + #print >> fout, prev_vals + #sys.exit() + for k, col in enumerate(cols): + if col >= 3 and col < 24: + if len(set(prev_vals[k])) == 1: + out_list[col] = prev_vals[k][0] + else: + break + while k < 23: + out_list[k+1] = 'n' + k += 1 + + print >>fout, '\t'.join(out_list) + + block_valid = True + prev_item = item + prev_vals = [] + for col in cols: + val_list = [] + val_list.append(fields[col].strip()) + prev_vals.append(val_list) + + else: + # This only occurs once, right at the start of the iteration. + block_valid = True + prev_item = item #groupby item + for col in cols: #everyting else + val_list = [] + val_list.append(fields[col].strip()) + prev_vals.append(val_list) + + except Exception, exc: + skipped_lines += 1 + if not first_invalid_line: + first_invalid_line = ii+1 + else: + skipped_lines += 1 + if not first_invalid_line: + first_invalid_line = ii+1 + + # Handle the last grouped value + out_list = ['']*25 + out_list[0] = str(prev_item) + out_list[1] = str(prev_vals[0][0]) + out_list[2] = str(prev_vals[1][0]) + out_list[24] = str(prev_vals[23][0]) + for k, col in enumerate(cols): + if col >= 3 and col < 24: + if len(set(prev_vals[k])) == 1: + out_list[col] = prev_vals[k][0] + else: + break + while k < 23: + out_list[k+1] = 'n' + k += 1 + + print >>fout, '\t'.join(out_list) + + if skipped_lines > 0: + msg= "Skipped %d invalid lines starting with line %d. Value '%s' in column %d is not numeric." % ( skipped_lines, first_invalid_line, invalid_value, invalid_column ) + print msg + +if __name__ == "__main__": + main() \ No newline at end of file diff -r 03f1e9e78f74 -r c3a20d503bd3 tools/taxonomy/lca.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/taxonomy/lca.xml Tue Jan 13 12:18:08 2009 -0500 @@ -0,0 +1,12 @@ +<tool id="lca1" name="Least Common Ancestor" version="1.0.0"> + <description></description> + <command interpreter="python"> + lca.py $input1 $out_file1 + </command> + <inputs> + <param format="taxonomy" name="input1" type="data" label="Select taxonomy dataset"/> + </inputs> + <outputs> + <data format="taxonomy" name="out_file1" metadata_source="input1" /> + </outputs> +</tool>

1 0

[hg] galaxy 1696: Make the PBS script/stdout/stderr directory co...
by Greg Von Kuster 13 Jan '09

13 Jan '09

details: http://www.bx.psu.edu/hg/galaxy/rev/aac0403bbe68 changeset: 1696:aac0403bbe68 user: Nate Coraor <nate(a)bx.psu.edu> date: Fri Jan 09 17:08:49 2009 -0500 description: Make the PBS script/stdout/stderr directory configurable. 3 file(s) affected in this change: lib/galaxy/config.py lib/galaxy/jobs/__init__.py lib/galaxy/jobs/runners/pbs.py diffs (93 lines): diff -r 94df4d059b19 -r aac0403bbe68 lib/galaxy/config.py --- a/lib/galaxy/config.py Fri Jan 09 15:59:29 2009 -0500 +++ b/lib/galaxy/config.py Fri Jan 09 17:08:49 2009 -0500 @@ -44,6 +44,7 @@ self.cluster_job_queue_workers = int( kwargs.get( "cluster_job_queue_workers", "5" ) ) self.job_scheduler_policy = kwargs.get("job_scheduler_policy", "FIFO") self.job_queue_cleanup_interval = int( kwargs.get("job_queue_cleanup_interval", "5") ) + self.cluster_files_directory = resolve_path( kwargs.get( "cluster_files_directory", "database/pbs" ), self.root ) self.job_working_directory = resolve_path( kwargs.get( "job_working_directory", "database/job_working_directory" ), self.root ) self.outputs_to_working_directory = string_as_bool( kwargs.get( 'outputs_to_working_directory', False ) ) self.admin_pass = kwargs.get('admin_pass',"galaxy") @@ -94,7 +95,7 @@ return default def check( self ): # Check that required directories exist - for path in self.root, self.file_path, self.tool_path, self.tool_data_path, self.template_path, self.job_working_directory: + for path in self.root, self.file_path, self.tool_path, self.tool_data_path, self.template_path, self.job_working_directory, self.cluster_files_directory: if not os.path.isdir( path ): raise ConfigurationError("Directory does not exist: %s" % path ) # Check that required files exist diff -r 94df4d059b19 -r aac0403bbe68 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py Fri Jan 09 15:59:29 2009 -0500 +++ b/lib/galaxy/jobs/__init__.py Fri Jan 09 17:08:49 2009 -0500 @@ -1,4 +1,4 @@ -import logging, threading, sys, os, time, subprocess, string, tempfile, re, traceback +import logging, threading, sys, os, time, subprocess, string, tempfile, re, traceback, shutil from galaxy import util, model from galaxy.model import mapping @@ -396,6 +396,13 @@ # if the job was deleted, don't fail it if not job.state == model.Job.states.DELETED: for dataset_assoc in job.output_datasets: + if self.app.config.outputs_to_working_directory: + false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % dataset_assoc.dataset.id ) ) + if os.path.exists( false_path ): + shutil.move( false_path, dataset_assoc.dataset.file_name ) + log.debug( "fail(): Moved %s to %s" % ( false_path, dataset_assoc.dataset.file_name ) ) + else: + log.warning( "fail(): Missing output file in working directory: %s" % false_path ) dataset = dataset_assoc.dataset dataset.refresh() dataset.state = dataset.states.ERROR @@ -474,10 +481,10 @@ if self.app.config.outputs_to_working_directory: false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % dataset_assoc.dataset.id ) ) if os.path.exists( false_path ): - os.rename( false_path, dataset_assoc.dataset.file_name ) - log.debug( "Moved %s to %s" % ( false_path, dataset_assoc.dataset.file_name ) ) + shutil.move( false_path, dataset_assoc.dataset.file_name ) + log.debug( "finish(): Moved %s to %s" % ( false_path, dataset_assoc.dataset.file_name ) ) else: - log.warning( "Missing output file in working directory: %s" % false_path ) + log.warning( "finish(): Missing output file in working directory: %s" % false_path ) for dataset in dataset_assoc.dataset.dataset.history_associations: #need to update all associated output hdas, i.e. history was shared with job running dataset.blurb = 'done' dataset.peek = 'no peek' diff -r 94df4d059b19 -r aac0403bbe68 lib/galaxy/jobs/runners/pbs.py --- a/lib/galaxy/jobs/runners/pbs.py Fri Jan 09 15:59:29 2009 -0500 +++ b/lib/galaxy/jobs/runners/pbs.py Fri Jan 09 17:08:49 2009 -0500 @@ -166,8 +166,8 @@ raise Exception( "Connection to PBS server for submit failed" ) # define job attributes - ofile = "%s/database/pbs/%s.o" % (os.getcwd(), job_wrapper.job_id) - efile = "%s/database/pbs/%s.e" % (os.getcwd(), job_wrapper.job_id) + ofile = "%s/%s.o" % (self.app.config.cluster_files_directory, job_wrapper.job_id) + efile = "%s/%s.e" % (self.app.config.cluster_files_directory, job_wrapper.job_id) # If an application server is set, we're staging if self.app.config.pbs_application_server: @@ -201,7 +201,7 @@ script = pbs_symlink_template % (job_wrapper.galaxy_lib_dir, " ".join(job_wrapper.get_input_fnames() + job_wrapper.get_output_fnames()), self.app.config.pbs_stage_path, exec_dir, command_line) else: script = pbs_template % (job_wrapper.galaxy_lib_dir, exec_dir, command_line) - job_file = "%s/database/pbs/%s.sh" % (os.getcwd(), job_wrapper.job_id) + job_file = "%s/%s.sh" % (self.app.config.cluster_files_directory, job_wrapper.job_id) fh = file(job_file, "w") fh.write(script) fh.close() @@ -436,9 +436,9 @@ def recover( self, job, job_wrapper ): """Recovers jobs stuck in the queued/running state when Galaxy started""" pbs_job_state = PBSJobState() - pbs_job_state.ofile = "%s/database/pbs/%s.o" % (os.getcwd(), job.id) - pbs_job_state.efile = "%s/database/pbs/%s.e" % (os.getcwd(), job.id) - pbs_job_state.job_file = "%s/database/pbs/%s.sh" % (os.getcwd(), job.id) + pbs_job_state.ofile = "%s/%s.o" % (self.app.config.cluster_files_directory, job.id) + pbs_job_state.efile = "%s/%s.e" % (self.app.config.cluster_files_directory, job.id) + pbs_job_state.job_file = "%s/%s.sh" % (self.app.config.cluster_files_directory, job.id) pbs_job_state.job_id = str( job.job_runner_external_id ) pbs_job_state.runner_url = job_wrapper.tool.job_runner job_wrapper.command_line = job.command_line

1 0

[hg] galaxy 1699: Add new tool config for flymine production ser...
by Greg Von Kuster 13 Jan '09

13 Jan '09

details: http://www.bx.psu.edu/hg/galaxy/rev/511c2b1966dc changeset: 1699:511c2b1966dc user: Greg Von Kuster <greg(a)bx.psu.edu> date: Tue Jan 13 10:55:08 2009 -0500 description: Add new tool config for flymine production server. 4 file(s) affected in this change: tool_conf.xml.main tool_conf.xml.sample tools/data_source/flymine.xml tools/data_source/flymine_test.xml diffs (68 lines): diff -r 88fe7fbd28da -r 511c2b1966dc tool_conf.xml.main --- a/tool_conf.xml.main Tue Jan 13 10:34:39 2009 -0500 +++ b/tool_conf.xml.main Tue Jan 13 10:55:08 2009 -0500 @@ -6,6 +6,7 @@ <tool file="data_source/ucsc_tablebrowser_archaea.xml" /> <tool file="data_source/microbial_import.xml" /> <tool file="data_source/biomart.xml" /> + <tool file="data_source/flymine.xml" /> <tool file="data_source/encode_db.xml" /> <tool file="data_source/epigraph_import.xml" /> </section> diff -r 88fe7fbd28da -r 511c2b1966dc tool_conf.xml.sample --- a/tool_conf.xml.sample Tue Jan 13 10:34:39 2009 -0500 +++ b/tool_conf.xml.sample Tue Jan 13 10:55:08 2009 -0500 @@ -11,6 +11,7 @@ <tool file="data_source/wormbase.xml" /> <tool file="data_source/wormbase_test.xml" /> <tool file="data_source/flymine.xml" /> + <tool file="data_source/flymine_test.xml" /> <tool file="data_source/encode_db.xml" /> <tool file="data_source/epigraph_import.xml" /> <tool file="data_source/epigraph_import_test.xml" /> diff -r 88fe7fbd28da -r 511c2b1966dc tools/data_source/flymine.xml --- a/tools/data_source/flymine.xml Tue Jan 13 10:34:39 2009 -0500 +++ b/tools/data_source/flymine.xml Tue Jan 13 10:55:08 2009 -0500 @@ -7,7 +7,7 @@ <tool name="Flymine" id="flymine" tool_type="data_source" URL_method="post"> <description>server</description> <command interpreter="python">data_source.py $output</command> - <inputs action="http://preview.flymine.org/preview/begin.do" check_values="false" method="get" target="_top"> + <inputs action="http://www.flymine.org" check_values="false" method="get" target="_top"> <display>go to Flymine server $GALAXY_URL</display> <param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=flymine" /> </inputs> diff -r 88fe7fbd28da -r 511c2b1966dc tools/data_source/flymine_test.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/data_source/flymine_test.xml Tue Jan 13 10:55:08 2009 -0500 @@ -0,0 +1,30 @@ +<?xml version="1.0"?> + +<tool name="Flymine test" id="flymine_test" tool_type="data_source" URL_method="post"> + <description>server</description> + <command interpreter="python">data_source.py $output</command> + <inputs action="http://preview.flymine.org/preview/begin.do" check_values="false" method="get" target="_top"> + <display>go to Flymine server $GALAXY_URL</display> + <param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=flymine" /> + </inputs> + <request_param_translation> + <request_param galaxy_name="URL" remote_name="URL" missing="" /> + <request_param galaxy_name="dbkey" remote_name="db" missing="?" /> + <request_param galaxy_name="organism" remote_name="organism" missing="" /> + <request_param galaxy_name="table" remote_name="table" missing="" /> + <request_param galaxy_name="description" remote_name="description" missing="" /> + <request_param galaxy_name="name" remote_name="name" missing="FlyMine query" /> + <request_param galaxy_name="info" remote_name="info" missing="" /> + <request_param galaxy_name="data_type" remote_name="data_type" missing="txt" /> + </request_param_translation> + <uihints minwidth="800"/> + <outputs> + <data name="output" format="txt" /> + </outputs> + <options sanitize="False" refresh="True"/> +</tool> +

1 0

[hg] galaxy 1698: Fix for data source tools that were broken due...
by Greg Von Kuster 13 Jan '09

13 Jan '09

details: http://www.bx.psu.edu/hg/galaxy/rev/88fe7fbd28da changeset: 1698:88fe7fbd28da user: Greg Von Kuster <greg(a)bx.psu.edu> date: Tue Jan 13 10:34:39 2009 -0500 description: Fix for data source tools that were broken due to the new "outputs_to_working_directory" config setting. Request parameters are now temporarily written to the output dataset if this config setting is either True or False. 1 file(s) affected in this change: lib/galaxy/tools/__init__.py diffs (43 lines): diff -r a6d6c2e4c021 -r 88fe7fbd28da lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py Fri Jan 09 17:18:18 2009 -0500 +++ b/lib/galaxy/tools/__init__.py Tue Jan 13 10:34:39 2009 -0500 @@ -1068,7 +1068,10 @@ wrap_values( input.cases[current].inputs, values ) elif isinstance( input, DataToolParameter ): input_values[ input.name ] = \ - DatasetFilenameWrapper( input_values[ input.name ], datatypes_registry = self.app.datatypes_registry, tool = self, name = input.name ) + DatasetFilenameWrapper( input_values[ input.name ], + datatypes_registry = self.app.datatypes_registry, + tool = self, + name = input.name ) else: input_values[ input.name ] = \ InputValueWrapper( input, input_values[ input.name ], param_dict ) @@ -1085,7 +1088,10 @@ # but this should be considered DEPRECATED, instead use: # $dataset.get_child( 'name' ).filename for name, data in input_datasets.items(): - param_dict[name] = DatasetFilenameWrapper( data, datatypes_registry = self.app.datatypes_registry, tool = self, name = name ) + param_dict[name] = DatasetFilenameWrapper( data, + datatypes_registry = self.app.datatypes_registry, + tool = self, + name = name ) if data: for child in data.children: param_dict[ "_CHILD___%s___%s" % ( name, child.designation ) ] = DatasetFilenameWrapper( child ) @@ -1263,8 +1269,13 @@ if data_type not in app.datatypes_registry.datatypes_by_extension: data_type = 'interval' data = app.datatypes_registry.change_datatype( data, data_type ) - # Store external data source's request parameters temporarily in output file - out = open( data.file_name, 'w' ) + # Store external data source's request parameters temporarily in output file. + # In case the config setting for "outputs_to_working_directory" is True, we must write to + # the DatasetFilenameWrapper object in the param_dict since it's "false_path" attribute + # is the temporary path to the output dataset ( until the job is run ). However, + # even if the "outputs_to_working_directory" setting is False, we can still open the file + # the same way for temporarily storing the request parameters. + out = open( str( param_dict.get( name ) ), 'w' ) for key, value in param_dict.items(): print >> out, '%s\t%s' % ( key, value ) out.close()

1 0

[hg] galaxy 1697: galaxy.config.resolve_path is not absolute...
by Greg Von Kuster 13 Jan '09

13 Jan '09

details: http://www.bx.psu.edu/hg/galaxy/rev/a6d6c2e4c021 changeset: 1697:a6d6c2e4c021 user: Nate Coraor <nate(a)bx.psu.edu> date: Fri Jan 09 17:18:18 2009 -0500 description: galaxy.config.resolve_path is not absolute... 1 file(s) affected in this change: lib/galaxy/config.py diffs (21 lines): diff -r aac0403bbe68 -r a6d6c2e4c021 lib/galaxy/config.py --- a/lib/galaxy/config.py Fri Jan 09 17:08:49 2009 -0500 +++ b/lib/galaxy/config.py Fri Jan 09 17:18:18 2009 -0500 @@ -15,7 +15,7 @@ if not( os.path.isabs( path ) ): path = os.path.join( root, path ) return path - + class ConfigurationError( Exception ): pass @@ -44,7 +44,7 @@ self.cluster_job_queue_workers = int( kwargs.get( "cluster_job_queue_workers", "5" ) ) self.job_scheduler_policy = kwargs.get("job_scheduler_policy", "FIFO") self.job_queue_cleanup_interval = int( kwargs.get("job_queue_cleanup_interval", "5") ) - self.cluster_files_directory = resolve_path( kwargs.get( "cluster_files_directory", "database/pbs" ), self.root ) + self.cluster_files_directory = os.path.abspath( kwargs.get( "cluster_files_directory", "database/pbs" ) ) self.job_working_directory = resolve_path( kwargs.get( "job_working_directory", "database/job_working_directory" ), self.root ) self.outputs_to_working_directory = string_as_bool( kwargs.get( 'outputs_to_working_directory', False ) ) self.admin_pass = kwargs.get('admin_pass',"galaxy")

1 0

Showing dataset state in History List View
by Assaf Gordon 13 Jan '09

13 Jan '09

Hello, I'd like to suggest a new feature: In the history list view, instead of just showing the 'size' of the history (= how many datasets are in the history) - show the state of each dataset. The reason is that some users run long jobs (or workflows with many steps) in several histories in parallel - and they want to quickly know: 1. which jobs are running, 2. which jobs are completed, 3. Were there any errors Currently, they have to switch to each history, and look at the state of the datasets. With this feature, all one needs to do is look at the history view. Using the same color keys for ok/queued/running/error states, users can quickly know: 1. If there's a grey box - some jobs are still queued. 2. If there are no grey boxes but some yellow boxes - some jobs are still running. 3. if there are no grey boxes and no yellow boxes - all jobs have been completed. 4. If there are red boxes - some jobs failed. Attached pictures illustrate the feature (at different states of jobs). To add this feature: Extract the attached 'list.mako.tar.gz' to GALAXY/templates/history (overriding the current list.mako). You'll also need to add the following function to GALAXY/lib/galaxy/model/__init__.py class History, line ~399 # returns number of datasets matching the requested state # Added by gordon, 24dec2008 def get_dataset_count( self, state ): count = 0 for data in self.datasets: if data.state == state and not data.deleted : count += 1 return count Another feature in this list.mako is the separation of 'switch to' link from the 'delete' and 'rename' links - The 'switch to' is much more important and frequently-used than the other two, and users have been complaining about the difficulty of clicking it (and accidentally clicking delete or rename). Comments are welcome, Gordon.

2 1

[hg] galaxy 1695: Add an option to override the output dataset(s...
by Nate Coraor 09 Jan '09

09 Jan '09

details: http://www.bx.psu.edu/hg/galaxy/rev/94df4d059b19 changeset: 1695:94df4d059b19 user: Nate Coraor <nate(a)bx.psu.edu> date: Fri Jan 09 15:59:29 2009 -0500 description: Add an option to override the output dataset(s) path, so they may be written to the working directory. This allows administrators to mount datasets read-only on cluster nodes. 3 file(s) affected in this change: lib/galaxy/config.py lib/galaxy/jobs/__init__.py lib/galaxy/tools/__init__.py diffs (100 lines): diff -r 58744d4bda7b -r 94df4d059b19 lib/galaxy/config.py --- a/lib/galaxy/config.py Fri Jan 09 14:07:16 2009 -0500 +++ b/lib/galaxy/config.py Fri Jan 09 15:59:29 2009 -0500 @@ -45,6 +45,7 @@ self.job_scheduler_policy = kwargs.get("job_scheduler_policy", "FIFO") self.job_queue_cleanup_interval = int( kwargs.get("job_queue_cleanup_interval", "5") ) self.job_working_directory = resolve_path( kwargs.get( "job_working_directory", "database/job_working_directory" ), self.root ) + self.outputs_to_working_directory = string_as_bool( kwargs.get( 'outputs_to_working_directory', False ) ) self.admin_pass = kwargs.get('admin_pass',"galaxy") self.sendmail_path = kwargs.get('sendmail_path',"/usr/sbin/sendmail") self.mailing_join_addr = kwargs.get('mailing_join_addr',"galaxy-user-join(a)bx.psu.edu") diff -r 58744d4bda7b -r 94df4d059b19 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py Fri Jan 09 14:07:16 2009 -0500 +++ b/lib/galaxy/jobs/__init__.py Fri Jan 09 15:59:29 2009 -0500 @@ -350,7 +350,7 @@ incoming['userId'] = userId incoming['userEmail'] = userEmail # Build params, done before hook so hook can use - param_dict = self.tool.build_param_dict( incoming, inp_data, out_data ) + param_dict = self.tool.build_param_dict( incoming, inp_data, out_data, self.working_directory ) # Certain tools require tasks to be completed prior to job execution # ( this used to be performed in the "exec_before_job" hook, but hooks are deprecated ). if self.tool.tool_type is not None: @@ -471,6 +471,13 @@ else: job.state = 'ok' for dataset_assoc in job.output_datasets: + if self.app.config.outputs_to_working_directory: + false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % dataset_assoc.dataset.id ) ) + if os.path.exists( false_path ): + os.rename( false_path, dataset_assoc.dataset.file_name ) + log.debug( "Moved %s to %s" % ( false_path, dataset_assoc.dataset.file_name ) ) + else: + log.warning( "Missing output file in working directory: %s" % false_path ) for dataset in dataset_assoc.dataset.dataset.history_associations: #need to update all associated output hdas, i.e. history was shared with job running dataset.blurb = 'done' dataset.peek = 'no peek' diff -r 58744d4bda7b -r 94df4d059b19 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py Fri Jan 09 14:07:16 2009 -0500 +++ b/lib/galaxy/tools/__init__.py Fri Jan 09 15:59:29 2009 -0500 @@ -1045,7 +1045,7 @@ input.validate( value, None ) input_values[ input.name ] = value - def build_param_dict( self, incoming, input_datasets, output_datasets ): + def build_param_dict( self, incoming, input_datasets, output_datasets, working_directory = None ): """ Build the dictionary of parameters for substituting into the command line. Each value is wrapped in a `InputValueWrapper`, which allows @@ -1090,7 +1090,13 @@ for child in data.children: param_dict[ "_CHILD___%s___%s" % ( name, child.designation ) ] = DatasetFilenameWrapper( child ) for name, data in output_datasets.items(): - param_dict[name] = DatasetFilenameWrapper( data ) + # Write outputs to the working directory (for security purposes) if desired. + if self.app.config.outputs_to_working_directory and working_directory is not None: + false_path = os.path.abspath( os.path.join( working_directory, "galaxy_dataset_%d.dat" % data.id ) ) + param_dict[name] = DatasetFilenameWrapper( data, false_path = false_path ) + open( false_path, 'w' ).close() + else: + param_dict[name] = DatasetFilenameWrapper( data ) # Provide access to a path to store additional files # TODO: path munging for cluster/dataset server relocatability param_dict[name].files_path = os.path.abspath(os.path.join(self.app.config.new_file_path, "dataset_%s_files" % (data.id) )) @@ -1433,7 +1439,7 @@ def items( self ): return iter( [ ( k, self.get( k ) ) for k, v in self.metadata.items() ] ) - def __init__( self, dataset, datatypes_registry = None, tool = None, name = None ): + def __init__( self, dataset, datatypes_registry = None, tool = None, name = None, false_path = None ): if not dataset: try: #TODO: allow this to work when working with grouping @@ -1444,10 +1450,17 @@ else: self.dataset = dataset self.metadata = self.MetadataWrapper( dataset.metadata ) + self.false_path = false_path def __str__( self ): - return self.dataset.file_name + if self.false_path is not None: + return self.false_path + else: + return self.dataset.file_name def __getattr__( self, key ): - return getattr( self.dataset, key ) + if self.false_path is not None and key == 'file_name': + return self.false_path + else: + return getattr( self.dataset, key ) def json_fix( val ): if isinstance( val, list ): @@ -1467,4 +1480,4 @@ value[composite_key] = incoming[key + "_" + composite_key] return value else: - return incoming.get( key, default ) \ No newline at end of file + return incoming.get( key, default )

1 0

[hg] galaxy 1694: Allow upstream proxy to set wsgi.url_scheme, s...
by Nate Coraor 09 Jan '09

09 Jan '09

details: http://www.bx.psu.edu/hg/galaxy/rev/58744d4bda7b changeset: 1694:58744d4bda7b user: Nate Coraor <nate(a)bx.psu.edu> date: Fri Jan 09 14:07:16 2009 -0500 description: Allow upstream proxy to set wsgi.url_scheme, so link generation is correct if the proxy uses a different transport with clients than it does with Galaxy. This doesn't work automatically. In Apache, you'd need to set the header yourself: RequestHeader set X-URL-SCHEME https For a discussion on how this affects links to the UCSC Genome Browser, see: http://mail.bx.psu.edu/pipermail/galaxy-user/2009-January/000430.html 2 file(s) affected in this change: lib/galaxy/web/framework/middleware/remoteuser.py lib/galaxy/web/framework/middleware/xforwardedhost.py diffs (25 lines): diff -r 4c1d1ce9265f -r 58744d4bda7b lib/galaxy/web/framework/middleware/remoteuser.py --- a/lib/galaxy/web/framework/middleware/remoteuser.py Thu Jan 08 11:04:19 2009 -0500 +++ b/lib/galaxy/web/framework/middleware/remoteuser.py Fri Jan 09 14:07:16 2009 -0500 @@ -62,6 +62,7 @@ # in the event of a lookup failure, deny access host = None if host in UCSC_SERVERS: + environ[ 'HTTP_REMOTE_USER' ] = 'ucsc_browser_display(a)example.org' return self.app( environ, start_response ) # Apache sets REMOTE_USER to the string '(null)' when using the # Rewrite* method for passing REMOTE_USER and a user is diff -r 4c1d1ce9265f -r 58744d4bda7b lib/galaxy/web/framework/middleware/xforwardedhost.py --- a/lib/galaxy/web/framework/middleware/xforwardedhost.py Thu Jan 08 11:04:19 2009 -0500 +++ b/lib/galaxy/web/framework/middleware/xforwardedhost.py Fri Jan 09 14:07:16 2009 -0500 @@ -14,4 +14,8 @@ if x_forwarded_for: environ[ 'ORGINAL_REMOTE_ADDR' ] = environ[ 'REMOTE_ADDR' ] environ[ 'REMOTE_ADDR' ] = x_forwarded_for - return self.app( environ, start_response ) \ No newline at end of file + x_url_scheme = environ.get( 'HTTP_X_URL_SCHEME', None ) + if x_url_scheme: + environ[ 'original_wsgi.url_scheme' ] = environ[ 'wsgi.url_scheme' ] + environ[ 'wsgi.url_scheme' ] = x_url_scheme + return self.app( environ, start_response )

1 0

[hg] galaxy 1693: Fix from James Casbon for a problem he has exp...
by Nate Coraor 08 Jan '09

08 Jan '09

details: http://www.bx.psu.edu/hg/galaxy/rev/4c1d1ce9265f changeset: 1693:4c1d1ce9265f user: Nate Coraor <nate(a)bx.psu.edu> date: Thu Jan 08 11:04:19 2009 -0500 description: Fix from James Casbon for a problem he has experienced when using remote_user (a new user creation seems to be attempted before a session exists). 1 file(s) affected in this change: lib/galaxy/web/framework/__init__.py diffs (29 lines): diff -r 93b946a7d8cb -r 4c1d1ce9265f lib/galaxy/web/framework/__init__.py --- a/lib/galaxy/web/framework/__init__.py Wed Jan 07 15:33:35 2009 -0500 +++ b/lib/galaxy/web/framework/__init__.py Thu Jan 08 11:04:19 2009 -0500 @@ -138,13 +138,22 @@ event.message = message % kwargs except: event.message = message - event.history = self.history + try: + event.history = self.history + except: + event.history = None try: event.history_id = self.history.id except: event.history_id = None - event.user = self.user - event.session_id = self.galaxy_session.id + try: + event.user = self.user + except: + event.user = None + try: + event.session_id = self.galaxy_session.id + except: + event.session_id = None event.flush() def get_cookie( self, name='galaxysession' ): """Convenience method for getting a session cookie"""

1 0

[hg] galaxy 1692: Fix for my last commit.
by Greg Von Kuster 07 Jan '09

07 Jan '09

details: http://www.bx.psu.edu/hg/galaxy/rev/93b946a7d8cb changeset: 1692:93b946a7d8cb user: Greg Von Kuster <greg(a)bx.psu.edu> date: Wed Jan 07 15:33:35 2009 -0500 description: Fix for my last commit. 1 file(s) affected in this change: test/functional/test_sniffing_and_metadata_settings.py diffs (13 lines): diff -r 6e9dd26f782f -r 93b946a7d8cb test/functional/test_sniffing_and_metadata_settings.py --- a/test/functional/test_sniffing_and_metadata_settings.py Wed Jan 07 12:20:35 2009 -0500 +++ b/test/functional/test_sniffing_and_metadata_settings.py Wed Jan 07 15:33:35 2009 -0500 @@ -110,8 +110,7 @@ self.delete_history_item( 1 ) def test_60_blastxml_datatype( self ): """Testing correctly sniffing blastxml data type upon upload""" - self.upload_file( 'megablast_xml_parser_test1.blastxml' ) - self.verify_dataset_correctness( 'megablast_xml_parser_test1.blastxml' ) + self.upload_file( 'megablast_xml_parser_test1.gz' ) self.check_history_for_string( 'NCBI Blast XML data' ) self.check_history_for_string( 'format: <span class="blastxml">blastxml</span>' ) self.delete_history_item( 1 )

1 0