- galaxy-dev - lists.galaxyproject.org

[hg] galaxy 2414: Move files_path to the job working directory a...
by Nate Coraor 21 May '09

21 May '09

details: http://www.bx.psu.edu/hg/galaxy/rev/1a82088aac38 changeset: 2414:1a82088aac38 user: Nate Coraor <nate(a)bx.psu.edu> date: Thu May 21 11:32:25 2009 -0400 description: Move files_path to the job working directory and fix ugly bugs that broke files_path. 2 file(s) affected in this change: lib/galaxy/jobs/__init__.py lib/galaxy/tools/__init__.py diffs (89 lines): diff -r a54fd974365d -r 1a82088aac38 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py Thu May 21 10:17:56 2009 -0400 +++ b/lib/galaxy/jobs/__init__.py Thu May 21 11:32:25 2009 -0400 @@ -351,7 +351,7 @@ incoming['userId'] = userId incoming['userEmail'] = userEmail # Build params, done before hook so hook can use - param_dict = self.tool.build_param_dict( incoming, inp_data, out_data, self.get_output_fnames() ) + param_dict = self.tool.build_param_dict( incoming, inp_data, out_data, self.get_output_fnames(), self.working_directory ) # Certain tools require tasks to be completed prior to job execution # ( this used to be performed in the "exec_before_job" hook, but hooks are deprecated ). if self.tool.tool_type is not None: @@ -529,7 +529,7 @@ param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] ) # why not re-use self.param_dict here? ##dunno...probably should, this causes tools.parameters.basic.UnvalidatedValue to be used in following methods instead of validated and transformed values during i.e. running workflows param_dict = self.tool.params_from_strings( param_dict, self.app ) # Check for and move associated_files - self.tool.collect_associated_files(out_data) + self.tool.collect_associated_files(out_data, self.working_directory) # Create generated output children and primary datasets and add to param_dict collected_datasets = {'children':self.tool.collect_child_datasets(out_data),'primary':self.tool.collect_primary_datasets(out_data)} param_dict.update({'__collected_datasets__':collected_datasets}) @@ -596,7 +596,7 @@ job = model.Job.get( self.job_id ) if self.app.config.outputs_to_working_directory: self.output_paths = [] - for name, data in [ ( da.name, da.dataset ) for da in job.output_datasets ]: + for name, data in [ ( da.name, da.dataset.dataset ) for da in job.output_datasets ]: false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % data.id ) ) self.output_paths.append( DatasetPath( data.file_name, false_path ) ) else: diff -r a54fd974365d -r 1a82088aac38 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py Thu May 21 10:17:56 2009 -0400 +++ b/lib/galaxy/tools/__init__.py Thu May 21 11:32:25 2009 -0400 @@ -1140,7 +1140,7 @@ message = e.message return message - def build_param_dict( self, incoming, input_datasets, output_datasets, output_paths ): + def build_param_dict( self, incoming, input_datasets, output_datasets, output_paths, job_working_directory ): """ Build the dictionary of parameters for substituting into the command line. Each value is wrapped in a `InputValueWrapper`, which allows @@ -1191,22 +1191,22 @@ if data: for child in data.children: param_dict[ "_CHILD___%s___%s" % ( name, child.designation ) ] = DatasetFilenameWrapper( child ) - for name, data in output_datasets.items(): + for name, hda in output_datasets.items(): # Write outputs to the working directory (for security purposes) if desired. if self.app.config.outputs_to_working_directory: try: - false_path = [ dp.false_path for dp in output_paths if dp.real_path == data.file_name ][0] - param_dict[name] = DatasetFilenameWrapper( data, false_path = false_path ) + false_path = [ dp.false_path for dp in output_paths if dp.real_path == hda.file_name ][0] + param_dict[name] = DatasetFilenameWrapper( hda, false_path = false_path ) open( false_path, 'w' ).close() except IndexError: log.warning( "Unable to determine alternate path for writing job outputs, outputs will be written to their real paths" ) - param_dict[name] = DatasetFilenameWrapper( data ) + param_dict[name] = DatasetFilenameWrapper( hda ) else: - param_dict[name] = DatasetFilenameWrapper( data ) + param_dict[name] = DatasetFilenameWrapper( hda ) # Provide access to a path to store additional files # TODO: path munging for cluster/dataset server relocatability - param_dict[name].files_path = os.path.abspath(os.path.join(self.app.config.new_file_path, "dataset_%s_files" % (data.id) )) - for child in data.children: + param_dict[name].files_path = os.path.abspath(os.path.join( job_working_directory, "dataset_%s_files" % (hda.dataset.id) )) + for child in hda.children: param_dict[ "_CHILD___%s___%s" % ( name, child.designation ) ] = DatasetFilenameWrapper( child ) # We add access to app here, this allows access to app.config, etc param_dict['__app__'] = RawObjectWrapper( self.app ) @@ -1394,12 +1394,12 @@ data.set_peek() data.flush() - def collect_associated_files( self, output ): - for name, outdata in output.items(): - temp_file_path = os.path.join( self.app.config.new_file_path, "dataset_%s_files" % ( outdata.id ) ) + def collect_associated_files( self, output, job_working_directory ): + for name, hda in output.items(): + temp_file_path = os.path.join( job_working_directory, "dataset_%s_files" % ( hda.dataset.id ) ) try: if len( os.listdir( temp_file_path ) ) > 0: - store_file_path = os.path.join( os.path.join( self.app.config.file_path, *directory_hash_id( outdata.id ) ), "dataset_%d_files" % outdata.id ) + store_file_path = os.path.join( os.path.join( self.app.config.file_path, *directory_hash_id( hda.dataset.id ) ), "dataset_%d_files" % hda.dataset.id ) shutil.move( temp_file_path, store_file_path ) except: continue

1 0

New tool: sequence shuffler
by Felix Muerdter 21 May '09

21 May '09

Dear all, I would like to share a little tool for galaxy that I wrote. The tool takes fasta files as input, keeps the header of each sequence but shuffles the sequence itself in a random order. Therefore the output file is a fasta file with sequences of the same base composition and length, but shuffled. It is a very simple idea but gives me the opportunity to statistically evaluate certain analysis for ³real² libraries with the newly created ³in silico² libraries. I hope some of you guys find this tool useful, comments are more than welcome! Best, Felix

1 1

Subversion update?
by Stewart Stevens 19 May '09

19 May '09

Hi, We've been using your Galaxy system for some months now. Thanks :-) We're finding it very useful and have been adding modules relevant to our research. When we first established the system here in New Zealand we followed installation instructions using subversion to obtain the latest (developmental) version of galaxy. Now we'd like to update. Unfortunately when issuing "svn update" we get the following response: > svn: PROPFIND request failed on '/svn/galaxy' > svn: PROPFIND of '/svn/galaxy': 405 Method Not Allowed (http://www.bx.psu.edu > ) > I also notice that the current install instructions all revolve around mercurial. I've tried running mercurial but there is no ".hg" file. Can you suggest how we can do an update? Hope this is ok going to galaxy-dev rather than galaxy-user. :-) Thanks, Stewart -- Stewart Stevens Software Developer - Integrated Genomics University of Otago Department of Biochemistry Tel: +64(0)3 479 7863 Fax: +64(0)3 479 7866

2 1

[Fwd: Re: ucsc browser/galaxy for teaching]
by anton＠bx.psu.edu 15 May '09

15 May '09

---------------------------- Original Message ---------------------------- Subject: Re: ucsc browser/galaxy for teaching From: "Robert Kuhn" <kuhn(a)soe.ucsc.edu> Date: Thu, May 14, 2009 8:27 pm To: albin(a)binf.ku.dk kuhn(a)soe.ucsc.edu -------------------------------------------------------------------------- Hi, albin, Just a heads-up for you. I have been sending some queries to galaxy to test that ttraffic from them is free to get through (we would see your queries via galaxy as coming from them), and I see that their site seems to be a bit stalled. my data queries are queued up but not being executed. I emailed anton at PSU about it, so maybe he or one of his people will see it before you need it. We'd like to be sure that any group of potential users of either site has a smooth time of it! BTW, it is no longer necessary to worry about a class being blocked if the traffic comes directly to ucsc. we have implemented a system that now distinguishes individual machines, rather than whole IPs, so interactive use in a class should be fine. Problem is, interactive use of a class using galaxy would likely be seen as all coming from one machine at PSU to us. do let us know if you have any trouble and we'll try to help for your second class. all the best, --b0b > From albin(a)binf.ku.dk Thu Apr 16 09:14:26 2009 > To: kuhn(a)soe.ucsc.edu > Subject: ucsc browser/galaxy for teaching > > Dear Robert, > I hope you might remember me from the Encode pilot. I am now at > Copenhagen University, heading a small group. > > The reason for me writing to you at this time is that we have a > course which, among other things, use galaxy and the ucsc browser > heavily at some lectures. We experienced last year that all the users > got cut off after a few minutes while using galaxy (but this only > happened when >5 users were online). My colleague Rasmus Nielsen let > me know that you were able to help him with a similar issue a while > back, so I hope you can help me with this as well. > > Specifically, would it be possible to let our IP address > (130.225.125.195) stay alive the following dates? > PET time > From To > Thursday, May 14, 2009 at 11:00:00 PM Friday, May 15, 2009 at 3:00:00 AM > Monday, May 18, 2009 at 11:00:00 PM Tuesday, May 19, 2009 at 8:00:00 AM > > I would be very grateful for your help! > Albin Sandelin > > Albin Sandelin, Associate Professor, PhD > The Bioinformatics Centre > Department of Biology & Biotech Research and Innovation Centre > University of Copenhagen > > Email: albin(a)binf.ku.dk, albin.sandelin(a)gmail.com

1 0

[hg] galaxy 2409: MAQ mapping pipeline for SOLiD data
by gua110＠scofield.bx.psu.edu 14 May '09

14 May '09

details: http://www.bx.psu.edu/hg/galaxy/rev/25d736f5cbf7 changeset: 2409:25d736f5cbf7 user: guru date: Wed May 13 21:54:45 2009 -0400 description: MAQ mapping pipeline for SOLiD data 4 file(s) affected in this change: tool_conf.xml.sample tools/solid_tools/maq_cs_wrapper.py tools/solid_tools/maq_cs_wrapper.xml tools/solid_tools/maq_cs_wrapper_code.py diffs (418 lines): diff -r 3af0dea9a5c7 -r 25d736f5cbf7 tool_conf.xml.sample --- a/tool_conf.xml.sample Mon May 11 17:02:54 2009 -0400 +++ b/tool_conf.xml.sample Wed May 13 21:54:45 2009 -0400 @@ -284,6 +284,7 @@ <section name="SOLiD Data Analysis" id="solid_tools"> <tool file="solid_tools/solid_qual_stats.xml" /> <tool file="solid_tools/solid_qual_boxplot.xml" /> + <tool file="solid_tools/maq_cs_wrapper.xml" /> </section> <section name="FASTA manipulation" id="fasta_manipulation"> <tool file="fasta_tools/fasta_compute_length.xml" /> diff -r 3af0dea9a5c7 -r 25d736f5cbf7 tools/solid_tools/maq_cs_wrapper.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/solid_tools/maq_cs_wrapper.py Wed May 13 21:54:45 2009 -0400 @@ -0,0 +1,270 @@ +#! /usr/bin/python +#Guruprasad Ananda +#MAQ mapper for SOLiD colourspace-reads + +import sys, os, zipfile, tempfile, subprocess + +def stop_err( msg ): + sys.stderr.write( "%s\n" % msg ) + sys.exit() + +def __main__(): + + out_fname = sys.argv[1].strip() + out_f2 = open(sys.argv[2].strip(),'r+') + ref_fname = sys.argv[3].strip() + f3_read_fname = sys.argv[4].strip() + f3_qual_fname = sys.argv[5].strip() + paired = sys.argv[6] + if paired == 'yes': + r3_read_fname = sys.argv[7].strip() + r3_qual_fname = sys.argv[8].strip() + min_mapqual = int(sys.argv[9].strip()) + max_mismatch = int(sys.argv[10].strip()) + out_f3name = sys.argv[11].strip() + subprocess_dict = {} + + ref_csfa = tempfile.NamedTemporaryFile() + ref_bfa = tempfile.NamedTemporaryFile() + ref_csbfa = tempfile.NamedTemporaryFile() + cmd2_1 = 'maq fasta2csfa %s > %s 2>&1' %(ref_fname,ref_csfa.name) + cmd2_2 = 'maq fasta2bfa %s %s 2>&1' %(ref_csfa.name,ref_csbfa.name) + cmd2_3 = 'maq fasta2bfa %s %s 2>&1' %(ref_fname,ref_bfa.name) + try: + os.system(cmd2_1) + os.system(cmd2_2) + os.system(cmd2_3) + except Exception, erf: + stop_err(str(erf)+"Error processing reference sequence") + + if paired == 'yes': #paired end reads + tmpf = tempfile.NamedTemporaryFile() #forward reads + tmpr = tempfile.NamedTemporaryFile() #reverse reads + tmps = tempfile.NamedTemporaryFile() #single reads + tmpffastq = tempfile.NamedTemporaryFile() + tmprfastq = tempfile.NamedTemporaryFile() + tmpsfastq = tempfile.NamedTemporaryFile() + + cmd1 = "solid2fastq_modified.pl 'yes' %s %s %s %s %s %s %s 2>&1" %(tmpf.name,tmpr.name,tmps.name,f3_read_fname,f3_qual_fname,r3_read_fname,r3_qual_fname) + try: + os.system(cmd1) + os.system('zcat -f %s >> %s' %(tmpf.name,tmpffastq.name)) + os.system('zcat -f %s >> %s' %(tmpr.name,tmprfastq.name)) + os.system('zcat -f %s >> %s' %(tmps.name,tmpsfastq.name)) + + except Exception, eq: + stop_err("Error converting data to fastq format." + str(eq)) + + #make a temp directory where the split fastq files will be stored + try: + split_dir = tempfile.mkdtemp() + split_file_prefix_f = tempfile.mktemp(dir=split_dir) + split_file_prefix_r = tempfile.mktemp(dir=split_dir) + splitcmd_f = 'split -a 2 -l %d %s %s' %(32000000,tmpffastq.name,split_file_prefix_f) #32M lines correspond to 8M reads + splitcmd_r = 'split -a 2 -l %d %s %s' %(32000000,tmprfastq.name,split_file_prefix_r) #32M lines correspond to 8M reads + + os.system(splitcmd_f) + os.system(splitcmd_r) + os.chdir(split_dir) + ii = 0 + for fastq in os.listdir(split_dir): + if not fastq.startswith(split_file_prefix_f.split("/")[-1]): + continue + fastq_r = split_file_prefix_r + fastq.split(split_file_prefix_f.split("/")[-1])[1] #find the reverse strand fastq corresponding to formward strand fastq + tmpbfq_f = tempfile.NamedTemporaryFile() + tmpbfq_r = tempfile.NamedTemporaryFile() + cmd3 = 'maq fastq2bfq %s %s 2>&1; maq fastq2bfq %s %s 2>&1; maq map -c %s.csmap %s %s %s 1>/dev/null 2>&1; maq mapview %s.csmap > %s.txt' %(fastq,tmpbfq_f.name,fastq_r,tmpbfq_r.name,fastq,ref_csbfa.name,tmpbfq_f.name,tmpbfq_r.name,fastq,fastq) + subprocess_dict['sp'+str(ii+1)] = subprocess.Popen([cmd3],shell=True,stdout=subprocess.PIPE) + ii += 1 + while True: + all_done = True + for j,k in enumerate(subprocess_dict.keys()): + if subprocess_dict['sp'+str(j+1)].wait() != 0: + err = subprocess_dict['sp'+str(j+1)].communicate()[1] + if err != None: + stop_err("Mapping error: %s" %err) + all_done = False + if all_done: + break + cmdout = "for map in *.txt; do cat $map >> %s; done" %(out_fname) + os.system(cmdout) + + tmpcsmap = tempfile.NamedTemporaryFile() + cmd_cat_csmap = "for csmap in *.csmap; do cat $csmap >> %s; done" %(tmpcsmap.name) + os.system(cmd_cat_csmap) + + tmppileup = tempfile.NamedTemporaryFile() + cmdpileup = "maq pileup -m %s -q %s %s %s > %s" %(max_mismatch,min_mapqual,ref_bfa.name,tmpcsmap.name,tmppileup.name) + os.system(cmdpileup) + tmppileup.seek(0) + print >> out_f2, "#chr\tposition\tref_nt\tcoverage\tSNP_count\tA_count\tT_count\tG_count\tC_count" + for line in file(tmppileup.name): + elems = line.strip().split() + ref_nt = elems[2].capitalize() + read_nt = elems[4] + coverage = int(elems[3]) + a,t,g,c = 0,0,0,0 + ref_nt_count = 0 + for ch in read_nt: + ch = ch.capitalize() + if ch not in ['A','T','G','C',',','.']: + continue + if ch in [',','.']: + ch = ref_nt + ref_nt_count += 1 + try: + nt_ind = ['A','T','G','C'].index(ch) + if nt_ind == 0: + a+=1 + elif nt_ind == 1: + t+=1 + elif nt_ind == 2: + g+=1 + else: + c+=1 + except ValueError, we: + print >>sys.stderr, we + print >> out_f2, "%s\t%s\t%s\t%s\t%s\t%s" %("\t".join(elems[:4]),coverage-ref_nt_count,a,t,g,c) + except Exception, er2: + stop_err("Encountered error while mapping: %s" %(str(er2))) + + + else: #single end reads + tmpf = tempfile.NamedTemporaryFile() + tmpfastq = tempfile.NamedTemporaryFile() + cmd1 = "solid2fastq_modified.pl 'no' %s %s %s %s %s %s %s 2>&1" %(tmpf.name,None,None,f3_read_fname,f3_qual_fname,None,None) + try: + os.system(cmd1) + os.system('zcat -f %s >> %s' %(tmpf.name,tmpfastq.name)) + tmpf.close() + except: + stop_err("Error converting data to fastq format.") + + #make a temp directory where the split fastq files will be stored + try: + split_dir = tempfile.mkdtemp() + split_file_prefix = tempfile.mktemp(dir=split_dir) + splitcmd = 'split -a 2 -l %d %s %s' %(32000000,tmpfastq.name,split_file_prefix) #32M lines correspond to 8M reads + os.system(splitcmd) + os.chdir(split_dir) + for i,fastq in enumerate(os.listdir(split_dir)): + tmpbfq = tempfile.NamedTemporaryFile() + cmd3 = 'maq fastq2bfq %s %s 2>&1; maq map -c %s.csmap %s %s 1>/dev/null 2>&1; maq mapview %s.csmap > %s.txt' %(fastq,tmpbfq.name,fastq,ref_csbfa.name,tmpbfq.name,fastq,fastq) + subprocess_dict['sp'+str(i+1)] = subprocess.Popen([cmd3],shell=True,stdout=subprocess.PIPE) + + while True: + all_done = True + for j,k in enumerate(subprocess_dict.keys()): + if subprocess_dict['sp'+str(j+1)].wait() != 0: + err = subprocess_dict['sp'+str(j+1)].communicate()[1] + if err != None: + stop_err("Mapping error: %s" %err) + all_done = False + if all_done: + break + + cmdout = "for map in *.txt; do cat $map >> %s; done" %(out_fname) + os.system(cmdout) + + tmpcsmap = tempfile.NamedTemporaryFile() + cmd_cat_csmap = "for csmap in *.csmap; do cat $csmap >> %s; done" %(tmpcsmap.name) + os.system(cmd_cat_csmap) + + tmppileup = tempfile.NamedTemporaryFile() + cmdpileup = "maq pileup -m %s -q %s %s %s > %s" %(max_mismatch,min_mapqual,ref_bfa.name,tmpcsmap.name,tmppileup.name) + os.system(cmdpileup) + tmppileup.seek(0) + print >> out_f2, "#chr\tposition\tref_nt\tcoverage\tSNP_count\tA_count\tT_count\tG_count\tC_count" + for line in file(tmppileup.name): + elems = line.strip().split() + ref_nt = elems[2].capitalize() + read_nt = elems[4] + coverage = int(elems[3]) + a,t,g,c = 0,0,0,0 + ref_nt_count = 0 + for ch in read_nt: + ch = ch.capitalize() + if ch not in ['A','T','G','C',',','.']: + continue + if ch in [',','.']: + ch = ref_nt + ref_nt_count += 1 + try: + nt_ind = ['A','T','G','C'].index(ch) + if nt_ind == 0: + a+=1 + elif nt_ind == 1: + t+=1 + elif nt_ind == 2: + g+=1 + else: + c+=1 + except: + pass + print >> out_f2, "%s\t%s\t%s\t%s\t%s\t%s" %("\t".join(elems[:4]),coverage-ref_nt_count,a,t,g,c) + except Exception, er2: + stop_err("Encountered error while mapping: %s" %(str(er2))) + + #Build custom track from pileup + chr_list=[] + out_f2.seek(0) + fcov = tempfile.NamedTemporaryFile() + fout_a = tempfile.NamedTemporaryFile() + fout_t = tempfile.NamedTemporaryFile() + fout_g = tempfile.NamedTemporaryFile() + fout_c = tempfile.NamedTemporaryFile() + fcov.write('''track type=wiggle_0 name="Coverage track" description="Coverage track (from Galaxy)" color=0,0,0 visibility=2\n''') + fout_a.write('''track type=wiggle_0 name="Track A" description="Track A (from Galaxy)" color=255,0,0 visibility=2\n''') + fout_t.write('''track type=wiggle_0 name="Track T" description="Track T (from Galaxy)" color=0,255,0 visibility=2\n''') + fout_g.write('''track type=wiggle_0 name="Track G" description="Track G (from Galaxy)" color=0,0,255 visibility=2\n''') + fout_c.write('''track type=wiggle_0 name="Track C" description="Track C (from Galaxy)" color=255,0,255 visibility=2\n''') + + for line in out_f2: + if line.startswith("#"): + continue + elems = line.split() + chr = elems[0] + + if chr not in chr_list: + chr_list.append(chr) + if not (chr.startswith('chr') or chr.startswith('scaffold')): + chr = 'chr' + header = "variableStep chrom=%s" %(chr) + fcov.write("%s\n" %(header)) + fout_a.write("%s\n" %(header)) + fout_t.write("%s\n" %(header)) + fout_g.write("%s\n" %(header)) + fout_c.write("%s\n" %(header)) + try: + pos = int(elems[1]) + cov = int(elems[3]) + a = int(elems[5]) + t = int(elems[6]) + g = int(elems[7]) + c = int(elems[8]) + except: + continue + fcov.write("%s\t%s\n" %(pos,cov)) + try: + a_freq = a*100./cov + t_freq = t*100./cov + g_freq = g*100./cov + c_freq = c*100./cov + except ZeroDivisionError: + a_freq=t_freq=g_freq=c_freq=0 + fout_a.write("%s\t%s\n" %(pos,a_freq)) + fout_t.write("%s\t%s\n" %(pos,t_freq)) + fout_g.write("%s\t%s\n" %(pos,g_freq)) + fout_c.write("%s\t%s\n" %(pos,c_freq)) + + fcov.seek(0) + fout_a.seek(0) + fout_g.seek(0) + fout_t.seek(0) + fout_c.seek(0) + os.system("cat %s %s %s %s %s | cat > %s" %(fcov.name,fout_a.name,fout_t.name,fout_g.name,fout_c.name,out_f3name)) + +if __name__=="__main__": + __main__() + + diff -r 3af0dea9a5c7 -r 25d736f5cbf7 tools/solid_tools/maq_cs_wrapper.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/solid_tools/maq_cs_wrapper.xml Wed May 13 21:54:45 2009 -0400 @@ -0,0 +1,120 @@ +<tool id="maq_cs_wrapper" name="MAQ for SOLiD" version="1.0.0"> + <description> </description> + <command interpreter="python"> + maq_cs_wrapper.py + $output1 + $output2 + $ref + $library_type.f3_reads + $library_type.f3_qual + $library_type.is_paired + #if $library_type.is_paired == "yes": + $library_type.r3_reads + $library_type.r3_qual + #else: + "None" + "None" + #end if + $min_mapqual + $max_mismatch + $output3 + + </command> + + <inputs> + <param name="ref" type="data" format="fasta" label="Target Genome"/> + <conditional name="library_type"> + <param name="is_paired" type="select" label="Is the library mate-paired?" multiple="false"> + <option value="no">No</option> + <option value="yes">Yes</option> + </param> + <when value="no"> + <param name="f3_reads" type="data" format="csfasta" label="F3 reads file"/> + <param format="qualsolid" name="f3_qual" type="data" label="F3 quality file" help="If your dataset doesn't show up in the menu, click the pencil icon next to your dataset and set the datatype to 'qualsolid'" /> + </when> + <when value="yes"> + <param name="f3_reads" type="data" format="csfasta" label="F3 reads file"/> + <param format="qualsolid" name="f3_qual" type="data" label="F3 quality file" help="If your dataset doesn't show up in the menu, click the pencil icon next to your dataset and set the datatype to 'qualsolid'" /> + <param name="r3_reads" type="data" format="csfasta" label="R3 reads file"/> + <param format="qualsolid" name="r3_qual" type="data" label="R3 quality file" help="If your dataset doesn't show up in the menu, click the pencil icon next to your dataset and set the datatype to 'qualsolid'" /> + </when> + </conditional> + <param name="min_mapqual" type="integer" size="3" value="0" label="Minimum mapping quality allowed for a read to be used" help="Reads below the specified mapping quality will not be considered in coverage and SNP analysis."/> + <param name="max_mismatch" type="integer" size="3" value="7" label="Maximum number of mismatches allowed for a read to be used" help="Reads above the specified threshold will not be considered in coverage and SNP analysis."/> + </inputs> + <outputs> + <data format="tabular" name="output1" metadata_source="ref" /> + <data format="tabular" name="output2" metadata_source="ref" /> + <data format="customtrack" name="output3" metadata_source="ref" /> + </outputs> + +  +<help> + +.. class:: infomark + +**What it does** + +This tool maps SOLiD colour-space reads against the target genome using MAQ. It produces three output datasets: + + +**ALIGNMENT INFO** : contains the read alignment information, + +**PILEUP** : contains the coverage and SNP statistics for every nucleotide of the target genome, + +**CUSTOM TRACK** : contains the coverage and SNP statistics as custom tracks displayable in the UCSC browser. + +----- + +**The ALIGNMENT INFO dataset will contain the following fields:** + +* column 1 = read name +* column 2 = chromosome +* column 3 = position +* column 4 = strand +* column 5 = insert size from the outer coorniates of a pair +* column 6 = paired flag +* column 7 = mapping quality +* column 8 = single-end mapping quality +* column 9 = alternative mapping quality +* column 10 = number of mismatches of the best hit +* column 11 = sum of qualities of mismatched bases of the best hit +* column 12 = number of 0-mismatch hits of the first 24bp +* column 13 = number of 1-mismatch hits of the first 24bp on the reference +* column 14 = length of the read +* column 15 = read sequence +* column 16 = read quality + + +**The PILEUP dataset will contain the following fields:** + +* column 1 = chromosome +* column 2 = position +* column 3 = reference nucleotide +* column 4 = coverage (number of reads that cover this position) +* column 5 = number of SNPs +* column 6 = number of As +* column 7 = number of Ts +* column 8 = number of Gs +* column 9 = number of Cs + +</help> +<code file="maq_cs_wrapper_code.py"/> + +</tool> diff -r 3af0dea9a5c7 -r 25d736f5cbf7 tools/solid_tools/maq_cs_wrapper_code.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/solid_tools/maq_cs_wrapper_code.py Wed May 13 21:54:45 2009 -0400 @@ -0,0 +1,5 @@ +def exec_before_job(app, inp_data, out_data, param_dict, tool): + out_data['output1'].name = out_data['output1'].name + " [ ALIGNMENT INFO ]" + out_data['output2'].name = out_data['output2'].name + " [ PILEUP ]" + out_data['output3'].name = out_data['output3'].name + " [ CUSTOM TRACK ]" +

1 0

[hg] galaxy 2408: Output example image file for SOLiD quality bo...
by Greg Von Kuster 12 May '09

12 May '09

details: http://www.bx.psu.edu/hg/galaxy/rev/3af0dea9a5c7 changeset: 2408:3af0dea9a5c7 user: guru date: Mon May 11 17:02:54 2009 -0400 description: Output example image file for SOLiD quality boxplot tool. 1 file(s) affected in this change: static/images/solid_qual.png diffs (2 lines): diff -r 60071136a638 -r 3af0dea9a5c7 static/images/solid_qual.png Binary file static/images/solid_qual.png has changed

1 0

[hg] galaxy 2407: Fix problem where delted workflows were still ...
by Nate Coraor 07 May '09

07 May '09

details: http://www.bx.psu.edu/hg/galaxy/rev/60071136a638 changeset: 2407:60071136a638 user: James Taylor <james(a)jamestaylor.org> date: Wed May 06 19:21:00 2009 -0400 description: Fix problem where delted workflows were still showing up when shared with another user (missing join before filter) 1 file(s) affected in this change: lib/galaxy/web/controllers/workflow.py diffs (15 lines): diff -r 23786eef94cd -r 60071136a638 lib/galaxy/web/controllers/workflow.py --- a/lib/galaxy/web/controllers/workflow.py Wed May 06 14:17:13 2009 -0400 +++ b/lib/galaxy/web/controllers/workflow.py Wed May 06 19:21:00 2009 -0400 @@ -35,8 +35,9 @@ shared_by_others = trans.sa_session \ .query( model.StoredWorkflowUserShareAssociation ) \ .filter_by( user=user ) \ - .filter( model.StoredWorkflow.c.deleted == False ) \ - .order_by( desc( model.StoredWorkflow.c.update_time ) ) \ + .join( 'stored_workflow' ) \ + .filter( model.StoredWorkflow.deleted == False ) \ + .order_by( desc( model.StoredWorkflow.update_time ) ) \ .all() return trans.fill_template( "workflow/list.mako", workflows = workflows,

1 0

[hg] galaxy 2405: Simpler threadless tarball streaming as sugges...
by Greg Von Kuster 06 May '09

06 May '09

details: http://www.bx.psu.edu/hg/galaxy/rev/c915cd0cbf13 changeset: 2405:c915cd0cbf13 user: Nate Coraor <nate(a)bx.psu.edu> date: Wed May 06 13:46:18 2009 -0400 description: Simpler threadless tarball streaming as suggested by James 2 file(s) affected in this change: lib/galaxy/util/streamball.py lib/galaxy/web/controllers/library.py diffs (81 lines): diff -r 8783747cbae8 -r c915cd0cbf13 lib/galaxy/util/streamball.py --- a/lib/galaxy/util/streamball.py Wed May 06 13:17:02 2009 -0400 +++ b/lib/galaxy/util/streamball.py Wed May 06 13:46:18 2009 -0400 @@ -1,50 +1,26 @@ """ -A simple wrapper for writing tarballs as a stream. The work is performed in a -thread and data is written to a Queue instead of a file. +A simple wrapper for writing tarballs as a stream. """ import logging, tarfile -from Queue import Queue, Empty, Full -from threading import Thread - log = logging.getLogger( __name__ ) - -class QueueArchive( object ): - queue_size = 32 - def __init__( self ): - self.queue = Queue( QueueArchive.queue_size ) - self.get = self.queue.get - self.empty = self.queue.empty - def write( self, data ): - self.queue.put( data, block=True, timeout=300 ) - def tell( self ): - return 0 class StreamBall( object ): def __init__( self, mode, members={} ): self.mode = mode self.members = members - self.tarfileobj = QueueArchive() + self.wsgi_status = None + self.wsgi_headeritems = None def add( self, file, relpath ): self.members[file] = relpath - def stream( self ): - t = Thread( target=self.thread_write ) - t.start() - while t.isAlive(): - try: - yield self.tarfileobj.get( block=False ) - except Empty: - pass - t.join() - # exhaust the queue - while not self.tarfileobj.empty(): - yield self.tarfileobj.get() - def thread_write( self ): - tf = tarfile.open( mode=self.mode, fileobj=self.tarfileobj ) - try: - for file, rel in self.members.items(): - tf.add( file, arcname=rel ) - tf.close() - except Full: - log.warning( 'Queue full for longer than 300 seconds, timing out' ) + def stream( self, environ, start_response ): + response_write = start_response( self.wsgi_status, self.wsgi_headeritems ) + class tarfileobj: + def write( self, *args, **kwargs ): + response_write( *args, **kwargs ) + tf = tarfile.open( mode=self.mode, fileobj=tarfileobj() ) + for file, rel in self.members.items(): + tf.add( file, arcname=rel ) + tf.close() + return [] diff -r 8783747cbae8 -r c915cd0cbf13 lib/galaxy/web/controllers/library.py --- a/lib/galaxy/web/controllers/library.py Wed May 06 13:17:02 2009 -0400 +++ b/lib/galaxy/web/controllers/library.py Wed May 06 13:46:18 2009 -0400 @@ -304,7 +304,9 @@ else: trans.response.set_content_type( "application/x-tar" ) trans.response.headers[ "Content-Disposition" ] = "attachment; filename=GalaxyLibraryFiles.%s" % params.do_action - return archive.stream() + archive.wsgi_status = trans.response.wsgi_status() + archive.wsgi_headeritems = trans.response.wsgi_headeritems() + return archive.stream @web.expose def download_dataset_from_folder(self, trans, id, library_id=None, **kwd): """Catches the dataset id and displays file contents as directed"""

1 0

[hg] galaxy 2406: Allow users to clone (copy) their own workflows
by Greg Von Kuster 06 May '09

06 May '09

details: http://www.bx.psu.edu/hg/galaxy/rev/23786eef94cd changeset: 2406:23786eef94cd user: James Taylor <james(a)jamestaylor.org> date: Wed May 06 14:17:13 2009 -0400 description: Allow users to clone (copy) their own workflows 1 file(s) affected in this change: templates/workflow/list.mako diffs (11 lines): diff -r c915cd0cbf13 -r 23786eef94cd templates/workflow/list.mako --- a/templates/workflow/list.mako Wed May 06 13:46:18 2009 -0400 +++ b/templates/workflow/list.mako Wed May 06 14:17:13 2009 -0400 @@ -46,6 +46,7 @@ <div popupmenu="wf-${i}-popup"> <a class="action-button" href="${h.url_for( action='editor', id=trans.security.encode_id(workflow.id) )}" target="_parent">Edit</a> <a class="action-button" href="${h.url_for( controller='root', action='index', workflow_id=trans.security.encode_id(workflow.id) )}" target="_parent">Run</a> + <a class="action-button" href="${h.url_for( action='clone', id=trans.security.encode_id(workflow.id) )}">Clone</a> <a class="action-button" href="${h.url_for( action='rename', id=trans.security.encode_id(workflow.id) )}">Rename</a> <a class="action-button" href="${h.url_for( action='sharing', id=trans.security.encode_id(workflow.id) )}">Sharing</a> <a class="action-button" confirm="Are you sure you want to delete workflow '${workflow.name}'?" href="${h.url_for( action='delete', id=trans.security.encode_id(workflow.id) )}">Delete</a>

1 0

[hg] galaxy 2404: Fix problem constructing workflows from histor...
by Greg Von Kuster 06 May '09

06 May '09

details: http://www.bx.psu.edu/hg/galaxy/rev/8783747cbae8 changeset: 2404:8783747cbae8 user: James Taylor <james(a)jamestaylor.org> date: Wed May 06 13:17:02 2009 -0400 description: Fix problem constructing workflows from histories -- problem existed when only a single dataset or job was selected, it was testing for the parameter being of type str, but parameters are now unicode, so (for example) the id 147 was converted to [1,4,7]. Fixed by testing for list (multiple parameter values should always come in as a list. There may be other similar problems where parameters are assumed to be str. 1 file(s) affected in this change: lib/galaxy/web/controllers/workflow.py diffs (26 lines): diff -r 8be3989335a9 -r 8783747cbae8 lib/galaxy/web/controllers/workflow.py --- a/lib/galaxy/web/controllers/workflow.py Tue May 05 22:44:30 2009 -0400 +++ b/lib/galaxy/web/controllers/workflow.py Wed May 06 13:17:02 2009 -0400 @@ -441,11 +441,11 @@ # Ensure job_ids and dataset_ids are lists (possibly empty) if job_ids is None: job_ids = [] - elif type( job_ids ) == str: + elif type( job_ids ) is not list: job_ids = [ job_ids ] if dataset_ids is None: dataset_ids = [] - elif type( dataset_ids ) == str: + elif type( dataset_ids ) is not list: dataset_ids = [ dataset_ids ] # Convert both sets of ids to integers job_ids = [ int( id ) for id in job_ids ] @@ -455,7 +455,7 @@ jobs, warnings = get_job_dict( trans ) jobs_by_id = dict( ( job.id, job ) for job in jobs.keys() ) steps = [] - steps_by_job_id= {} + steps_by_job_id = {} hid_to_output_pair = {} # Input dataset steps for hid in dataset_ids:

1 0