details: http://www.bx.psu.edu/hg/galaxy/rev/d3ff52561d78 changeset: 3664:d3ff52561d78 user: jeremy goecks <jeremy.goecks@emory.edu> date: Mon Apr 19 11:08:25 2010 -0400 description: Complete Cufflinks wrapper. diffstat: tools/ngs_rna/cufflinks_wrapper.py | 64 +++++++++++++++++++------- tools/ngs_rna/cufflinks_wrapper.xml | 88 ++++++++++++++++++++++++++++++++++-- 2 files changed, 128 insertions(+), 24 deletions(-) diffs (244 lines): diff -r efd404f7a60b -r d3ff52561d78 tools/ngs_rna/cufflinks_wrapper.py --- a/tools/ngs_rna/cufflinks_wrapper.py Fri Apr 16 18:46:35 2010 -0400 +++ b/tools/ngs_rna/cufflinks_wrapper.py Mon Apr 19 11:08:25 2010 -0400 @@ -10,20 +10,20 @@ #Parse Command Line parser = optparse.OptionParser() parser.add_option( '-1', '--input', dest='input', help=' file of RNA-Seq read alignments in the SAM format. SAM is a standard short read alignment, that allows aligners to attach custom tags to individual alignments, and Cufflinks requires that the alignments you supply have some of these tags. Please see Input formats for more details.' ) - parser.add_option( '-s', '--inner-dist-std-dev', help='The standard deviation for the distribution on inner distances between mate pairs. The default is 20bp.' ) - parser.add_option( '-I', '--max-intron-length', help='The minimum intron length. Cufflinks will not report transcripts with introns longer than this, and will ignore SAM alignments with REF_SKIP CIGAR operations longer than this. The default is 300,000.' ) - parser.add_option( '-F', '--min-isoform-fraction', help='After calculating isoform abundance for a gene, Cufflinks filters out transcripts that it believes are very low abundance, because isoforms expressed at extremely low levels often cannot reliably be assembled, and may even be artifacts of incompletely spliced precursors of processed transcripts. This parameter is also used to filter out introns that have far fewer spliced alignments supporting them. The default is 0.05, or 5% of the most abundant isoform (the major isoform) of the gene.' ) - parser.add_option( '-j', '--pre-mrna-fraction', help='Some RNA-Seq protocols produce a significant amount of reads that originate from incompletely spliced transcripts, and these reads can confound the assembly of fully spliced mRNAs. Cufflinks uses this parameter to filter out alignments that lie within the intronic intervals implied by the spliced alignments. The minimum depth of coverage in the intronic region covered by the alignment is divided by the number of spliced reads, and if the result is lower than this parameter value, the intronic alignments are ignored. The default is 5%.' ) - parser.add_option( '-p', '--num-threads', help='Use this many threads to align reads. The default is 1.' ) + parser.add_option( '-s', '--inner-dist-std-dev', dest='inner_dist_std_dev', help='The standard deviation for the distribution on inner distances between mate pairs. The default is 20bp.' ) + parser.add_option( '-I', '--max-intron-length', dest='max_intron_len', help='The minimum intron length. Cufflinks will not report transcripts with introns longer than this, and will ignore SAM alignments with REF_SKIP CIGAR operations longer than this. The default is 300,000.' ) + parser.add_option( '-F', '--min-isoform-fraction', dest='min_isoform_fraction', help='After calculating isoform abundance for a gene, Cufflinks filters out transcripts that it believes are very low abundance, because isoforms expressed at extremely low levels often cannot reliably be assembled, and may even be artifacts of incompletely spliced precursors of processed transcripts. This parameter is also used to filter out introns that have far fewer spliced alignments supporting them. The default is 0.05, or 5% of the most abundant isoform (the major isoform) of the gene.' ) + parser.add_option( '-j', '--pre-mrna-fraction', dest='pre_mrna_fraction', help='Some RNA-Seq protocols produce a significant amount of reads that originate from incompletely spliced transcripts, and these reads can confound the assembly of fully spliced mRNAs. Cufflinks uses this parameter to filter out alignments that lie within the intronic intervals implied by the spliced alignments. The minimum depth of coverage in the intronic region covered by the alignment is divided by the number of spliced reads, and if the result is lower than this parameter value, the intronic alignments are ignored. The default is 5%.' ) + parser.add_option( '-p', '--num-threads', dest='num_threads', help='Use this many threads to align reads. The default is 1.' ) parser.add_option( '-m', '--inner-mean-dist', dest='inner_mean_dist', help='This is the expected (mean) inner distance between mate pairs. \ For, example, for paired end runs with fragments selected at 300bp, \ where each end is 50bp, you should set -r to be 200. The default is 45bp.') - parser.add_option( '-Q', '--min-mapqual', help='Instructs Cufflinks to ignore alignments with a SAM mapping quality lower than this number. The default is 0.' ) - parser.add_option( '-L', '--label', help='Cufflinks will report transfrags in GTF format, with a prefix given by this option. The default prefix is "CUFF".' ) - parser.add_option( '-G', '--GTF', help='Tells Cufflinks to use the supplied reference annotation to estimate isoform expression. It will not assemble novel transcripts, and the program will ignore alignments not structurally compatible with any reference transcript.' ) + parser.add_option( '-Q', '--min-mapqual', dest='min_mapqual', help='Instructs Cufflinks to ignore alignments with a SAM mapping quality lower than this number. The default is 0.' ) + parser.add_option( '-G', '--GTF', dest='GTF', help='Tells Cufflinks to use the supplied reference annotation to estimate isoform expression. It will not assemble novel transcripts, and the program will ignore alignments not structurally compatible with any reference transcript.' ) + # Advanced Options: - parser.add_option( '--num-importance-samples', help='Sets the number of importance samples generated for each locus during abundance estimation. Default: 1000' ) - parser.add_option( '--max-mle-iterations', help='Sets the number of iterations allowed during maximum likelihood estimation of abundances. Default: 5000' ) + parser.add_option( '--num-importance-samples', dest='num_importance_samples', help='Sets the number of importance samples generated for each locus during abundance estimation. Default: 1000' ) + parser.add_option( '--max-mle-iterations', dest='max_mle_iterations', help='Sets the number of iterations allowed during maximum likelihood estimation of abundances. Default: 5000' ) # Wrapper / Galaxy options. parser.add_option( '-A', '--assembled-isoforms-output', dest='assembled_isoforms_output_file', help='Assembled isoforms output file; formate is GTF.' ) @@ -41,31 +41,61 @@ cmd = "cufflinks" # Add options. + if options.inner_dist_std_dev: + cmd += ( " -s %i" % int ( options.inner_dist_std_dev ) ) + if options.max_intron_len: + cmd += ( " -I %i" % int ( options.max_intron_len ) ) + if options.min_isoform_fraction: + cmd += ( " -F %f" % float ( options.min_isoform_fraction ) ) + if options.pre_mrna_fraction: + cmd += ( " -j %f" % float ( options.pre_mrna_fraction ) ) + if options.num_threads: + cmd += ( " -p %i" % int ( options.num_threads ) ) if options.inner_mean_dist: cmd += ( " -m %i" % int ( options.inner_mean_dist ) ) + if options.min_mapqual: + cmd += ( " -Q %i" % int ( options.min_mapqual ) ) + if options.GTF: + cmd += ( " -G %i" % options.GTF ) + if options.num_importance_samples: + cmd += ( " --num-importance-samples %i" % int ( options.num_importance_samples ) ) + if options.max_mle_iterations: + cmd += ( " --max-mle-iterations %i" % int ( options.max_mle_iterations ) ) # Add input files. cmd += " " + options.input - - # Run + print cmd + + # Run command. try: - proc = subprocess.Popen( args=cmd, shell=True, cwd=tmp_output_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) + tmp_name = tempfile.NamedTemporaryFile( dir=tmp_output_dir ).name + tmp_stderr = open( tmp_name, 'wb' ) + proc = subprocess.Popen( args=cmd, shell=True, cwd=tmp_output_dir, stderr=tmp_stderr.fileno() ) returncode = proc.wait() + tmp_stderr.close() + + # Get stderr, allowing for case where it's very large. + tmp_stderr = open( tmp_name, 'rb' ) stderr = '' buffsize = 1048576 try: while True: - stderr += proc.stderr.read( buffsize ) + stderr += tmp_stderr.read( buffsize ) if not stderr or len( stderr ) % buffsize != 0: break except OverflowError: pass + tmp_stderr.close() + + # Error checking. if returncode != 0: raise Exception, stderr + + # check that there are results in the output file + if len( open( tmp_output_dir + "/transcripts.gtf", 'rb' ).read().strip() ) == 0: + raise Exception, 'The main output file is empty, there may be an error with your input file or settings.' except Exception, e: - stop_err( 'Error in cufflinks:\n' + str( e ) ) - - # TODO: look for errors in program output. + stop_err( 'Error running cufflinks. ' + str( e ) ) # Copy output files from tmp directory to specified files. try: diff -r efd404f7a60b -r d3ff52561d78 tools/ngs_rna/cufflinks_wrapper.xml --- a/tools/ngs_rna/cufflinks_wrapper.xml Fri Apr 16 18:46:35 2010 -0400 +++ b/tools/ngs_rna/cufflinks_wrapper.xml Mon Apr 19 11:08:25 2010 -0400 @@ -1,5 +1,5 @@ <tool id="cufflinks" name="Cufflinks" version="0.8.2"> - <description>Transcript assembly, differential expression, and differential regulation for RNA-Seq</description> + <description>transcript assembly, differential expression, and differential regulation for RNA-Seq</description> <command interpreter="python"> cufflinks_wrapper.py --input=$input @@ -7,24 +7,46 @@ --transcripts-expression-output=$transcripts_expression --genes-expression-output=$genes_expression --num-threads="4" + -I $max_intron_len + -F $min_isoform_fraction + -j $pre_mrna_fraction + -Q $min_map_quality + #if $reference_annotation.use_ref == "Yes": + -G $reference_annotation.reference_annotation_file + #end if #if $singlePaired.sPaired == "paired": -r $singlePaired.mean_inner_distance + -s $singlePaired.inner_distance_std_dev #end if </command> <inputs> <param format="sam" name="input" type="data" label="SAM file of aligned RNA-Seq reads" help=""/> + <param name="max_intron_len" type="integer" value="300000" label="Max Intron Length" help=""/> + <param name="min_isoform_fraction" type="float" value="0.05" label="Min Isoform Fraction" help=""/> + <param name="pre_mrna_fraction" type="float" value="0.05" label="Pre MRNA Fraction" help=""/> + <param name="min_map_quality" type="integer" value="0" label="Min SAM Map Quality" help=""/> + <conditional name="reference_annotation"> + <param name="use_ref" type="select" label="Use Reference Annotation?"> + <option value="No">No</option> + <option value="Yes">Yes</option> + </param> + <when value="No"></when> + <when value="Yes"> + <param format="gtf" name="reference_annotation_file" type="data" label="Reference Annotation" help=""/> + </when> + </conditional> <conditional name="singlePaired"> <param name="sPaired" type="select" label="Is this library mate-paired?"> <option value="single">Single-end</option> <option value="paired">Paired-end</option> </param> - <when value="single"> - - </when> + <when value="single"></when> <when value="paired"> <param name="mean_inner_distance" type="integer" value="20" label="Mean Inner Distance between Mate Pairs"/> + <param name="inner_distance_std_dev" type="integer" value="20" label="Standard Deviation for Inner Distance between Mate Pairs"/> </when> </conditional> + </inputs> <outputs> @@ -67,19 +89,64 @@ **Input formats** -Cufflinks accepts files in SAM format. +Cufflinks takes a text file of SAM alignments as input. The RNA-Seq read mapper TopHat produces output in this format, and is recommended for use with Cufflinks. However Cufflinks will accept SAM alignments generated by any read mapper. Here's an example of an alignment Cufflinks will accept:: + + s6.25mer.txt-913508 16 chr1 4482736 255 14M431N11M * 0 0 \ + CAAGATGCTAGGCAAGTCTTGGAAG IIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 XS:A:- + +Note the use of the custom tag XS. This attribute, which must have a value of "+" or "-", indicates which strand the RNA that produced this read came from. While this tag can be applied to any alignment, including unspliced ones, it must be present for all spliced alignment records (those with a 'N' operation in the CIGAR string). +The SAM file supplied to Cufflinks must be sorted by reference position. If you aligned your reads with TopHat, your alignments will be properly sorted already. If you used another tool, you may want to make sure they are properly sorted as follows:: + + sort -k 3,3 -k 4,4n hits.sam > hits.sam.sorted + +NOTE: Cufflinks currently only supports SAM alignments with the CIGAR match ('M') and reference skip ('N') operations. Support for the other operations, such as insertions, deletions, and clipping, will be added in the future. ------ **Outputs** -TODO +Cufflinks produces three output files: + +Transcripts and Genes: + +This GTF file contains Cufflinks' assembled isoforms. The first 7 columns are standard GTF, and the last column contains attributes, some of which are also standardized (e.g. gene_id, transcript_id). There one GTF record per row, and each record represents either a transcript or an exon within a transcript. The columns are defined as follows:: + + Column number Column name Example Description + ----------------------------------------------------- + 1 seqname chrX Chromosome or contig name + 2 source Cufflinks The name of the program that generated this file (always 'Cufflinks') + 3 feature exon The type of record (always either "transcript" or "exon"). + 4 start 77696957 The leftmost coordinate of this record (where 0 is the leftmost possible coordinate) + 5 end 77712009 The rightmost coordinate of this record, inclusive. + 6 score 77712009 The most abundant isoform for each gene is assigned a score of 1000. Minor isoforms are scored by the ratio (minor FPKM/major FPKM) + 7 strand + Cufflinks' guess for which strand the isoform came from. Always one of '+', '-' '.' + 7 frame . Cufflinks does not predict where the start and stop codons (if any) are located within each transcript, so this field is not used. + 8 attributes See below + +Each GTF record is decorated with the following attributes:: + + Attribute Example Description + ----------------------------------------- + gene_id CUFF.1 Cufflinks gene id + transcript_id CUFF.1.1 Cufflinks transcript id + FPKM 101.267 Isoform-level relative abundance in Reads Per Kilobase of exon model per Million mapped reads + frac 0.7647 Reserved. Please ignore, as this attribute may be deprecated in the future + conf_lo 0.07 Lower bound of the 95% confidence interval of the abundance of this isoform, as a fraction of the isoform abundance. That is, lower bound = FPKM * (1.0 - conf_lo) + conf_hi 0.1102 Upper bound of the 95% confidence interval of the abundance of this isoform, as a fraction of the isoform abundance. That is, upper bound = FPKM * (1.0 + conf_lo) + cov 100.765 Estimate for the absolute depth of read coverage across the whole transcript + + +Transcripts only: + This file is simply a tab delimited file containing one row per transcript and with columns containing the attributes above. There are a few additional attributes not in the table above, but these are reserved for debugging, and may change or disappear in the future. + +Genes only: +This file contains gene-level coordinates and expression values. ------- **Cufflinks settings** -All of the options have a default value. You can change any of them. Some of the options in Cufflinks have been implemented here. +All of the options have a default value. You can change any of them. Most of the options in Cufflinks have been implemented here. ------ @@ -87,5 +154,12 @@ This is a list of implemented Cufflinks options:: + -m INT This is the expected (mean) inner distance between mate pairs. For, example, for paired end runs with fragments selected at 300bp, where each end is 50bp, you should set -r to be 200. The default is 45bp. + -s INT The standard deviation for the distribution on inner distances between mate pairs. The default is 20bp. + -I INT The minimum intron length. Cufflinks will not report transcripts with introns longer than this, and will ignore SAM alignments with REF_SKIP CIGAR operations longer than this. The default is 300,000. + -F After calculating isoform abundance for a gene, Cufflinks filters out transcripts that it believes are very low abundance, because isoforms expressed at extremely low levels often cannot reliably be assembled, and may even be artifacts of incompletely spliced precursors of processed transcripts. This parameter is also used to filter out introns that have far fewer spliced alignments supporting them. The default is 0.05, or 5% of the most abundant isoform (the major isoform) of the gene. + -j Some RNA-Seq protocols produce a significant amount of reads that originate from incompletely spliced transcripts, and these reads can confound the assembly of fully spliced mRNAs. Cufflinks uses this parameter to filter out alignments that lie within the intronic intervals implied by the spliced alignments. The minimum depth of coverage in the intronic region covered by the alignment is divided by the number of spliced reads, and if the result is lower than this parameter value, the intronic alignments are ignored. The default is 5%. + -Q Instructs Cufflinks to ignore alignments with a SAM mapping quality lower than this number. The default is 0. + -G Tells Cufflinks to use the supplied reference annotation to estimate isoform expression. It will not assemble novel transcripts, and the program will ignore alignments not structurally compatible with any reference transcript. </help> </tool>