- galaxy-dev - lists.galaxyproject.org

[hg] galaxy 2737: Galaxy AMQP messaging now uses amqp & sqlalche...
by Nate Coraor 25 Sep '09

25 Sep '09

details: http://www.bx.psu.edu/hg/galaxy/rev/7a4e6f19aa17 changeset: 2737:7a4e6f19aa17 user: rc date: Mon Sep 21 12:06:33 2009 -0400 description: Galaxy AMQP messaging now uses amqp & sqlalchemy eggs 2 file(s) affected in this change: scripts/galaxy_messaging/amqp_consumer.py scripts/galaxy_messaging/galaxydb_interface.py diffs (58 lines): diff -r f2e5b1725b50 -r 7a4e6f19aa17 scripts/galaxy_messaging/amqp_consumer.py --- a/scripts/galaxy_messaging/amqp_consumer.py Mon Sep 21 11:21:35 2009 -0400 +++ b/scripts/galaxy_messaging/amqp_consumer.py Mon Sep 21 12:06:33 2009 -0400 @@ -9,13 +9,23 @@ this script can be run to receive messages and update the Galaxy database accordingly ''' - -from amqplib import client_0_8 as amqp import ConfigParser -import sys +import sys, os import optparse import xml.dom.minidom from galaxydb_interface import GalaxyDbInterface + +assert sys.version_info[:2] >= ( 2, 4 ) +new_path = [ os.path.join( os.getcwd(), "lib" ) ] +new_path.extend( sys.path[1:] ) # remove scripts/ from the path +sys.path = new_path + +from galaxy import eggs +import pkg_resources +pkg_resources.require( "amqplib" ) + +from amqplib import client_0_8 as amqp + galaxy_config_file = 'universe_wsgi.ini' global dbconnstr diff -r f2e5b1725b50 -r 7a4e6f19aa17 scripts/galaxy_messaging/galaxydb_interface.py --- a/scripts/galaxy_messaging/galaxydb_interface.py Mon Sep 21 11:21:35 2009 -0400 +++ b/scripts/galaxy_messaging/galaxydb_interface.py Mon Sep 21 12:06:33 2009 -0400 @@ -1,13 +1,22 @@ #/usr/bin/python -from sqlalchemy import * -from sqlalchemy.orm import sessionmaker from datetime import datetime, timedelta import sys import optparse import os import time import logging + +assert sys.version_info[:2] >= ( 2, 4 ) +new_path = [ os.path.join( os.getcwd(), "lib" ) ] +new_path.extend( sys.path[1:] ) # remove scripts/ from the path +sys.path = new_path +from galaxy import eggs +import pkg_resources +#pkg_resources.require( "psycopg2 >= 2.0.6" ) +pkg_resources.require( "SQLAlchemy >= 0.4" ) +from sqlalchemy import * +from sqlalchemy.orm import sessionmaker logging.basicConfig(level=logging.DEBUG) log = logging.getLogger( 'GalaxyDbInterface' )

1 0

[hg] galaxy 2738: added psycopg2 import stmt in galaxy_interface.py
by Nate Coraor 25 Sep '09

25 Sep '09

details: http://www.bx.psu.edu/hg/galaxy/rev/2b730844747b changeset: 2738:2b730844747b user: rc date: Mon Sep 21 12:41:39 2009 -0400 description: added psycopg2 import stmt in galaxy_interface.py 1 file(s) affected in this change: scripts/galaxy_messaging/galaxydb_interface.py diffs (13 lines): diff -r 7a4e6f19aa17 -r 2b730844747b scripts/galaxy_messaging/galaxydb_interface.py --- a/scripts/galaxy_messaging/galaxydb_interface.py Mon Sep 21 12:06:33 2009 -0400 +++ b/scripts/galaxy_messaging/galaxydb_interface.py Mon Sep 21 12:41:39 2009 -0400 @@ -13,7 +13,8 @@ sys.path = new_path from galaxy import eggs import pkg_resources -#pkg_resources.require( "psycopg2 >= 2.0.6" ) +pkg_resources.require( "psycopg2" ) +import psycopg2 pkg_resources.require( "SQLAlchemy >= 0.4" ) from sqlalchemy import * from sqlalchemy.orm import sessionmaker

1 0

[hg] galaxy 2732: tags made available in history grid
by Nate Coraor 25 Sep '09

25 Sep '09

details: http://www.bx.psu.edu/hg/galaxy/rev/c2e3183756da changeset: 2732:c2e3183756da user: jeremy goecks <jeremy.goecks(a)emory.edu> date: Mon Sep 21 10:39:58 2009 -0400 description: tags made available in history grid 3 file(s) affected in this change: lib/galaxy/web/controllers/history.py static/scripts/autocomplete_tagging.js templates/tagging_common.mako diffs (91 lines): diff -r ebf68a725c09 -r c2e3183756da lib/galaxy/web/controllers/history.py --- a/lib/galaxy/web/controllers/history.py Sun Sep 20 11:48:51 2009 -0400 +++ b/lib/galaxy/web/controllers/history.py Mon Sep 21 10:39:58 2009 -0400 @@ -45,9 +45,10 @@ def get_value( self, trans, grid, history ): self.tag_elt_id_gen += 1 - return trans.fill_template( "/tagging_common.mako", trans=trans, - tagged_item=history, - elt_id="tagging-elt" + str(self.tag_elt_id_gen) ) + elt_id="tagging-elt" + str( self.tag_elt_id_gen ) + div_elt = "<div id=%s></div>" % elt_id + return div_elt + trans.fill_template( "/tagging_common.mako", trans=trans, + tagged_item=history, elt_id = elt_id, in_form="true", input_size="20" ) # Grid definition title = "Stored histories" @@ -59,7 +60,7 @@ link=( lambda item: iff( item.deleted, None, dict( operation="switch", id=item.id ) ) ), attach_popup=True ), DatasetsByStateColumn( "Datasets (by state)", ncells=4 ), - #TagsColumn( "Tags" ), + TagsColumn( "Tags"), StatusColumn( "Status", attach_popup=False ), grids.GridColumn( "Created", key="create_time", format=time_ago ), grids.GridColumn( "Last Updated", key="update_time", format=time_ago ), diff -r ebf68a725c09 -r c2e3183756da static/scripts/autocomplete_tagging.js --- a/static/scripts/autocomplete_tagging.js Sun Sep 20 11:48:51 2009 -0400 +++ b/static/scripts/autocomplete_tagging.js Mon Sep 21 10:39:58 2009 -0400 @@ -23,7 +23,7 @@ text = "Add tags"; return text; }, - tag_click_fn : function (tag) { }, + tag_click_fn : function (name, value) { }, input_size: 20, in_form: false, tags : {}, @@ -240,10 +240,11 @@ // Build tag button. var tag_name_elt = $("<span>").text(tag_str).addClass("tag-name"); tag_name_elt.click( function() - { - settings.tag_click_fn(tag_str); - return true; - }); + { + tag_name_and_value = tag_str.split(":") + settings.tag_click_fn(tag_name_and_value[0], tag_name_and_value[1]); + return true; + }); var tag_button = $("<span></span>").addClass("tag-button"); tag_button.append(tag_name_elt); diff -r ebf68a725c09 -r c2e3183756da templates/tagging_common.mako --- a/templates/tagging_common.mako Sun Sep 20 11:48:51 2009 -0400 +++ b/templates/tagging_common.mako Mon Sep 21 10:39:58 2009 -0400 @@ -1,3 +1,8 @@ +## Render a tagging element if there is a tagged_item. +%if tagged_item is not None and elt_id is not None: + ${render_tagging_element(tagged_item, elt_id=elt_id, in_form=in_form, input_size=input_size)} +%endif + ## Render the tags 'tags' as an autocomplete element. <%def name="render_tagging_element(tagged_item, elt_id, use_toggle_link='true', in_form='false', input_size='15')"> <script type="text/javascript"> @@ -73,11 +78,23 @@ return text; }; + // + // Function to handle a tag click. + // + var tag_click_fn = function(tag_name, tag_value) + { + alert(tag_name); + + // Do URL request to get histories tag. + self.location = "http://www.yahoo.com"; + }; + var options = { tags : ${h.to_json_string(tag_names_and_values)}, get_toggle_link_text_fn: get_toggle_link_text, - tag_click_fn: function(tag) { /* Do nothing. */ }, + tag_click_fn: tag_click_fn, + ##tag_click_fn: function(name, value) { /* Do nothing. */ }, <% tagged_item_id = trans.security.encode_id(tagged_item.id) %> ajax_autocomplete_tag_url: "${h.url_for( controller='tag', action='tag_autocomplete_data', id=tagged_item_id, item_class=tagged_item.__class__.__name__ )}", ajax_add_tag_url: "${h.url_for( controller='tag', action='add_tag_async', id=tagged_item_id, item_class=tagged_item.__class__.__name__ )}",

1 0

[hg] galaxy 2735: Fixed a bug in creating requests as an admin
by Nate Coraor 25 Sep '09

25 Sep '09

details: http://www.bx.psu.edu/hg/galaxy/rev/12e6192f6201 changeset: 2735:12e6192f6201 user: rc date: Mon Sep 21 10:49:17 2009 -0400 description: Fixed a bug in creating requests as an admin 2 file(s) affected in this change: lib/galaxy/web/form_builder.py scripts/galaxy_messaging/amqp_consumer.py diffs (62 lines): diff -r 63e37852f658 -r 12e6192f6201 lib/galaxy/web/form_builder.py --- a/lib/galaxy/web/form_builder.py Mon Sep 21 10:46:36 2009 -0400 +++ b/lib/galaxy/web/form_builder.py Mon Sep 21 10:49:17 2009 -0400 @@ -384,8 +384,9 @@ from galaxy import util address_html = '' add_ids = ['none'] - for a in self.user.addresses: - add_ids.append(str(a.id)) + if self.user: + for a in self.user.addresses: + add_ids.append(str(a.id)) add_ids.append('new') self.select_address = SelectField(self.name, refresh_on_change=True, @@ -394,16 +395,17 @@ self.select_address.add_option('Select one', 'none', selected=True) else: self.select_address.add_option('Select one', 'none') - for a in self.user.addresses: - if not a.deleted: - if self.value == str(a.id): - self.select_address.add_option(a.desc, str(a.id), selected=True) - # display this address - address_html = '''<div class="form-row"> - %s - </div>''' % a.get_html() - else: - self.select_address.add_option(a.desc, str(a.id)) + if self.user: + for a in self.user.addresses: + if not a.deleted: + if self.value == str(a.id): + self.select_address.add_option(a.desc, str(a.id), selected=True) + # display this address + address_html = '''<div class="form-row"> + %s + </div>''' % a.get_html() + else: + self.select_address.add_option(a.desc, str(a.id)) if self.value == 'new': self.select_address.add_option('Add a new address', 'new', selected=True) for field_name, label in self.fields(): diff -r 63e37852f658 -r 12e6192f6201 scripts/galaxy_messaging/amqp_consumer.py --- a/scripts/galaxy_messaging/amqp_consumer.py Mon Sep 21 10:46:36 2009 -0400 +++ b/scripts/galaxy_messaging/amqp_consumer.py Mon Sep 21 10:49:17 2009 -0400 @@ -1,3 +1,15 @@ +''' +Galaxy Messaging with AMQP (RabbitMQ) +Galaxy uses AMQ protocol to receive messages from external sources like +bar code scanners. Galaxy has been tested against RabbitMQ AMQP implementation. +For Galaxy to receive messages from a message queue the RabbitMQ server has +to be set up with a user account and other parameters listed in the [galaxy:amq] +section in the universe_wsgi.ini config file +Once the RabbitMQ server has been setup and started with the given parameters, +this script can be run to receive messages and update the Galaxy database accordingly +''' + + from amqplib import client_0_8 as amqp import ConfigParser import sys

1 0

[hg] galaxy 2734: Quick fix to disable partial functionality in ...
by Nate Coraor 25 Sep '09

25 Sep '09

details: http://www.bx.psu.edu/hg/galaxy/rev/63e37852f658 changeset: 2734:63e37852f658 user: jeremy goecks <jeremy.goecks(a)emory.edu> date: Mon Sep 21 10:46:36 2009 -0400 description: Quick fix to disable partial functionality in tagging 1 file(s) affected in this change: templates/tagging_common.mako diffs (16 lines): diff -r 2f848dc3e35c -r 63e37852f658 templates/tagging_common.mako --- a/templates/tagging_common.mako Mon Sep 21 10:40:25 2009 -0400 +++ b/templates/tagging_common.mako Mon Sep 21 10:46:36 2009 -0400 @@ -83,10 +83,12 @@ // var tag_click_fn = function(tag_name, tag_value) { + /* alert(tag_name); // Do URL request to get histories tag. self.location = "http://www.yahoo.com"; + */ }; var options =

1 0

[hg] galaxy 2733: merge
by Nate Coraor 25 Sep '09

25 Sep '09

details: http://www.bx.psu.edu/hg/galaxy/rev/2f848dc3e35c changeset: 2733:2f848dc3e35c user: jeremy goecks <jeremy.goecks(a)emory.edu> date: Mon Sep 21 10:40:25 2009 -0400 description: merge 0 file(s) affected in this change: diffs (196 lines): diff -r c2e3183756da -r 2f848dc3e35c tool_conf.xml.sample --- a/tool_conf.xml.sample Mon Sep 21 10:39:58 2009 -0400 +++ b/tool_conf.xml.sample Mon Sep 21 10:40:25 2009 -0400 @@ -169,29 +169,28 @@ <tool file="fastx_toolkit/fasta_nucleotide_changer.xml" /> <tool file="fastx_toolkit/fastx_collapser.xml" /> </section> - <section name="NGS: FASTQ QC and manipulation" id="cshl_library_information"> - <tool file="fastx_toolkit/fastx_quality_statistics.xml" /> - <tool file="fastx_toolkit/fastq_quality_boxplot.xml" /> - <tool file="fastx_toolkit/fastx_nucleotides_distribution.xml" /> -  - <tool file="fastx_toolkit/fastq_to_fasta.xml" /> - <tool file="fastx_toolkit/fastq_quality_converter.xml" /> -  - <tool file="fastx_toolkit/fastx_trimmer.xml" /> - <tool file="fastx_toolkit/fastx_renamer.xml" /> - <tool file="fastx_toolkit/fastx_reverse_complement.xml" /> - <tool file="fastx_toolkit/fastx_artifacts_filter.xml" /> - <tool file="fastx_toolkit/fastq_quality_filter.xml" /> -  - <tool file="metag_tools/split_paired_reads.xml" /> - </section> - <section name="454: QC" id="short_read_analysis"> - <tool file="metag_tools/short_reads_figure_score.xml" /> - <tool file="metag_tools/short_reads_trim_seq.xml" /> - </section> - <section name="SOLiD: QC" id="solid_tools"> - <tool file="solid_tools/solid_qual_stats.xml" /> - <tool file="solid_tools/solid_qual_boxplot.xml" /> + <section name="NGS: QC and manipulation" id="cshl_library_information"> + <label text="Genetic FASTQ data" id="fastq" /> + <tool file="fastx_toolkit/fastx_quality_statistics.xml" /> + <tool file="fastx_toolkit/fastq_quality_boxplot.xml" /> + <tool file="fastx_toolkit/fastx_nucleotides_distribution.xml" /> +  + <tool file="fastx_toolkit/fastq_to_fasta.xml" /> + <tool file="fastx_toolkit/fastq_quality_converter.xml" /> +  + <tool file="fastx_toolkit/fastx_trimmer.xml" /> + <tool file="fastx_toolkit/fastx_renamer.xml" /> + <tool file="fastx_toolkit/fastx_reverse_complement.xml" /> + <tool file="fastx_toolkit/fastx_artifacts_filter.xml" /> + <tool file="fastx_toolkit/fastq_quality_filter.xml" /> +  + <tool file="metag_tools/split_paired_reads.xml" /> + <label text="Roche-454 Specific" id="454" /> + <tool file="metag_tools/short_reads_figure_score.xml" /> + <tool file="metag_tools/short_reads_trim_seq.xml" /> + <label text="AB-SOLiD Specific" id="solid" /> + <tool file="solid_tools/solid_qual_stats.xml" /> + <tool file="solid_tools/solid_qual_boxplot.xml" /> </section> <section name="NGS: Mapping" id="solexa_tools">  diff -r c2e3183756da -r 2f848dc3e35c tools/samtools/pileup_parser.pl --- a/tools/samtools/pileup_parser.pl Mon Sep 21 10:39:58 2009 -0400 +++ b/tools/samtools/pileup_parser.pl Mon Sep 21 10:40:25 2009 -0400 @@ -1,6 +1,8 @@ #! /usr/bin/perl -w use strict; +use POSIX; + die "Usage: pileup_parser.pl <in_file> <ref_base_column> <read_bases_column> <base_quality_column> <coverage column> <qv cutoff> <coverage cutoff> <SNPs only?> <output bed?> <coord_column> <out_file>\n" unless @ARGV == 11; @@ -28,10 +30,12 @@ while (<IN>) { chop; + next if m/^\#/; my @fields = split /\t/; next if $fields[ $ref_base_column ] eq "*"; # skip indel lines - next if $fields[ $cvrg_column ] < $cvrg_cutoff; # skip low coverage lines - my $read_bases = $fields[ $read_bases_column ]; + my $read_bases = $fields[ $read_bases_column ]; + die "Coverage column" . ($cvrg_column+1) . " contains non-numeric values. Check your input parameters as well as format of input dataset." if ( not isdigit $fields[ $cvrg_column ] ); + next if $fields[ $cvrg_column ] < $cvrg_cutoff; my $base_quality = $fields[ $base_quality_column ]; if ($read_bases =~ m/[\$\^\+-]/) { @@ -42,11 +46,8 @@ $read_bases =~ s/[\+-]{1}$indel_len.{$indel_len}//; # remove indel info from read base field } } - if ( length($read_bases) != length($base_quality) ) { - - $first_skipped_line = $_ if $invalid_line_counter == 0; - ++$invalid_line_counter; - } + die "Error parsing read bases and qualities in line $.. Last processed line conatined these values: " . join("\t", @fields) . "\n" if ( length($read_bases) != length($base_quality) ); + # after removing read block and indel data the length of read_base # field should identical to the length of base_quality field @@ -95,4 +96,4 @@ print STDERR "Could not parse $invalid_line_counter line(s) beginning with: $first_skipped_line\n" if $invalid_line_counter > 0; close IN; -close OUT; \ No newline at end of file +close OUT; diff -r c2e3183756da -r 2f848dc3e35c tools/samtools/sam2interval.py --- a/tools/samtools/sam2interval.py Mon Sep 21 10:39:58 2009 -0400 +++ b/tools/samtools/sam2interval.py Mon Sep 21 10:40:25 2009 -0400 @@ -78,7 +78,7 @@ for line in infile: line = line.rstrip( '\r\n' ) - if line and not line.startswith( '#,@' ): + if line and not line.startswith( '#' ) and not line.startswith( '@' ) : fields = line.split( '\t' ) start = int( fields[ int( options.start_col ) - 1 ] ) - 1 end = 0 diff -r c2e3183756da -r 2f848dc3e35c tools/samtools/sam_bitwise_flag_filter.py --- a/tools/samtools/sam_bitwise_flag_filter.py Mon Sep 21 10:39:58 2009 -0400 +++ b/tools/samtools/sam_bitwise_flag_filter.py Mon Sep 21 10:40:25 2009 -0400 @@ -135,7 +135,7 @@ for line in infile: line = line.rstrip( '\r\n' ) - if line and not line.startswith( '#' ): + if line and not line.startswith( '#' ) and not line.startswith( '@' ) : fields = line.split( '\t' ) sam_states = [] sam_states.append( bool( int( fields[ int( options.flag_col ) - 1 ] ) & 0x0001 ) ) diff -r c2e3183756da -r 2f848dc3e35c tools/sr_mapping/bowtie_wrapper.xml --- a/tools/sr_mapping/bowtie_wrapper.xml Mon Sep 21 10:39:58 2009 -0400 +++ b/tools/sr_mapping/bowtie_wrapper.xml Mon Sep 21 10:40:25 2009 -0400 @@ -336,7 +336,7 @@ </param> </when> </conditional> - <param name="offrate" type="integer" value="-1" label="Override the offrate of the index to n -o)" help="-1 for default" /> + <param name="offrate" type="integer" value="-1" label="Override the offrate of the index to n (-o)" help="-1 for default" /> <param name="seed" type="integer" value="-1" label="Seed for pseudo-random number generator (--seed)" help="-1 for default" /> </when>  </conditional>  @@ -443,31 +443,34 @@ The output is in SAM format, and has the following columns:: - 1 QNAME - Query (pair) NAME - 2 FLAG - bitwise FLAG - 3 RNAME - Reference sequence NAME - 4 POS - 1-based leftmost POSition/coordinate of clipped sequence - 5 MAPQ - MAPping Quality (Phred-scaled) - 6 CIGAR - extended CIGAR string - 7 MRNM - Mate Reference sequence NaMe ('=' if same as RNAME) - 8 MPOS - 1-based Mate POSition - 9 ISIZE - Inferred insert SIZE - 10 SEQ - query SEQuence on the same strand as the reference - 11 QUAL - query QUALity (ASCII-33 gives the Phred base quality) - 12 OPT - variable OPTional fields in the format TAG:VTYPE:VALU + Column Description + -------- -------------------------------------------------------- + 1 QNAME Query (pair) NAME + 2 FLAG bitwise FLAG + 3 RNAME Reference sequence NAME + 4 POS 1-based leftmost POSition/coordinate of clipped sequence + 5 MAPQ MAPping Quality (Phred-scaled) + 6 CIGAR extended CIGAR string + 7 MRNM Mate Reference sequence NaMe ('=' if same as RNAME) + 8 MPOS 1-based Mate POSition + 9 ISIZE Inferred insert SIZE + 10 SEQ query SEQuence on the same strand as the reference + 11 QUAL query QUALity (ASCII-33 gives the Phred base quality) + 12 OPT variable OPTional fields in the format TAG:VTYPE:VALUE The flags are as follows:: - Flag - Description - 0x0001 - the read is paired in sequencing - 0x0002 - the read is mapped in a proper pair - 0x0004 - the query sequence itself is unmapped - 0x0008 - the mate is unmapped - 0x0010 - strand of the query (1 for reverse) - 0x0020 - strand of the mate - 0x0040 - the read is the first read in a pair - 0x0080 - the read is the second read in a pair - 0x0100 - the alignment is not primary + Flag Description + ------ ------------------------------------- + 0x0001 the read is paired in sequencing + 0x0002 the read is mapped in a proper pair + 0x0004 the query sequence itself is unmapped + 0x0008 the mate is unmapped + 0x0010 strand of the query (1 for reverse) + 0x0020 strand of the mate + 0x0040 the read is the first read in a pair + 0x0080 the read is the second read in a pair + 0x0100 the alignment is not primary It looks like this (scroll sideways to see the entire example)::

1 0

[hg] galaxy 2731: More SR changes
by Nate Coraor 25 Sep '09

25 Sep '09

details: http://www.bx.psu.edu/hg/galaxy/rev/8255ed4330e9 changeset: 2731:8255ed4330e9 user: Anton Nekrutenko <anton(a)bx.psu.edu> date: Mon Sep 21 10:32:48 2009 -0400 description: More SR changes 5 file(s) affected in this change: tool_conf.xml.sample tools/samtools/pileup_parser.pl tools/samtools/sam2interval.py tools/samtools/sam_bitwise_flag_filter.py tools/sr_mapping/bowtie_wrapper.xml diffs (196 lines): diff -r ebf68a725c09 -r 8255ed4330e9 tool_conf.xml.sample --- a/tool_conf.xml.sample Sun Sep 20 11:48:51 2009 -0400 +++ b/tool_conf.xml.sample Mon Sep 21 10:32:48 2009 -0400 @@ -169,29 +169,28 @@ <tool file="fastx_toolkit/fasta_nucleotide_changer.xml" /> <tool file="fastx_toolkit/fastx_collapser.xml" /> </section> - <section name="NGS: FASTQ QC and manipulation" id="cshl_library_information"> - <tool file="fastx_toolkit/fastx_quality_statistics.xml" /> - <tool file="fastx_toolkit/fastq_quality_boxplot.xml" /> - <tool file="fastx_toolkit/fastx_nucleotides_distribution.xml" /> -  - <tool file="fastx_toolkit/fastq_to_fasta.xml" /> - <tool file="fastx_toolkit/fastq_quality_converter.xml" /> -  - <tool file="fastx_toolkit/fastx_trimmer.xml" /> - <tool file="fastx_toolkit/fastx_renamer.xml" /> - <tool file="fastx_toolkit/fastx_reverse_complement.xml" /> - <tool file="fastx_toolkit/fastx_artifacts_filter.xml" /> - <tool file="fastx_toolkit/fastq_quality_filter.xml" /> -  - <tool file="metag_tools/split_paired_reads.xml" /> - </section> - <section name="454: QC" id="short_read_analysis"> - <tool file="metag_tools/short_reads_figure_score.xml" /> - <tool file="metag_tools/short_reads_trim_seq.xml" /> - </section> - <section name="SOLiD: QC" id="solid_tools"> - <tool file="solid_tools/solid_qual_stats.xml" /> - <tool file="solid_tools/solid_qual_boxplot.xml" /> + <section name="NGS: QC and manipulation" id="cshl_library_information"> + <label text="Genetic FASTQ data" id="fastq" /> + <tool file="fastx_toolkit/fastx_quality_statistics.xml" /> + <tool file="fastx_toolkit/fastq_quality_boxplot.xml" /> + <tool file="fastx_toolkit/fastx_nucleotides_distribution.xml" /> +  + <tool file="fastx_toolkit/fastq_to_fasta.xml" /> + <tool file="fastx_toolkit/fastq_quality_converter.xml" /> +  + <tool file="fastx_toolkit/fastx_trimmer.xml" /> + <tool file="fastx_toolkit/fastx_renamer.xml" /> + <tool file="fastx_toolkit/fastx_reverse_complement.xml" /> + <tool file="fastx_toolkit/fastx_artifacts_filter.xml" /> + <tool file="fastx_toolkit/fastq_quality_filter.xml" /> +  + <tool file="metag_tools/split_paired_reads.xml" /> + <label text="Roche-454 Specific" id="454" /> + <tool file="metag_tools/short_reads_figure_score.xml" /> + <tool file="metag_tools/short_reads_trim_seq.xml" /> + <label text="AB-SOLiD Specific" id="solid" /> + <tool file="solid_tools/solid_qual_stats.xml" /> + <tool file="solid_tools/solid_qual_boxplot.xml" /> </section> <section name="NGS: Mapping" id="solexa_tools">  diff -r ebf68a725c09 -r 8255ed4330e9 tools/samtools/pileup_parser.pl --- a/tools/samtools/pileup_parser.pl Sun Sep 20 11:48:51 2009 -0400 +++ b/tools/samtools/pileup_parser.pl Mon Sep 21 10:32:48 2009 -0400 @@ -1,6 +1,8 @@ #! /usr/bin/perl -w use strict; +use POSIX; + die "Usage: pileup_parser.pl <in_file> <ref_base_column> <read_bases_column> <base_quality_column> <coverage column> <qv cutoff> <coverage cutoff> <SNPs only?> <output bed?> <coord_column> <out_file>\n" unless @ARGV == 11; @@ -28,10 +30,12 @@ while (<IN>) { chop; + next if m/^\#/; my @fields = split /\t/; next if $fields[ $ref_base_column ] eq "*"; # skip indel lines - next if $fields[ $cvrg_column ] < $cvrg_cutoff; # skip low coverage lines - my $read_bases = $fields[ $read_bases_column ]; + my $read_bases = $fields[ $read_bases_column ]; + die "Coverage column" . ($cvrg_column+1) . " contains non-numeric values. Check your input parameters as well as format of input dataset." if ( not isdigit $fields[ $cvrg_column ] ); + next if $fields[ $cvrg_column ] < $cvrg_cutoff; my $base_quality = $fields[ $base_quality_column ]; if ($read_bases =~ m/[\$\^\+-]/) { @@ -42,11 +46,8 @@ $read_bases =~ s/[\+-]{1}$indel_len.{$indel_len}//; # remove indel info from read base field } } - if ( length($read_bases) != length($base_quality) ) { - - $first_skipped_line = $_ if $invalid_line_counter == 0; - ++$invalid_line_counter; - } + die "Error parsing read bases and qualities in line $.. Last processed line conatined these values: " . join("\t", @fields) . "\n" if ( length($read_bases) != length($base_quality) ); + # after removing read block and indel data the length of read_base # field should identical to the length of base_quality field @@ -95,4 +96,4 @@ print STDERR "Could not parse $invalid_line_counter line(s) beginning with: $first_skipped_line\n" if $invalid_line_counter > 0; close IN; -close OUT; \ No newline at end of file +close OUT; diff -r ebf68a725c09 -r 8255ed4330e9 tools/samtools/sam2interval.py --- a/tools/samtools/sam2interval.py Sun Sep 20 11:48:51 2009 -0400 +++ b/tools/samtools/sam2interval.py Mon Sep 21 10:32:48 2009 -0400 @@ -78,7 +78,7 @@ for line in infile: line = line.rstrip( '\r\n' ) - if line and not line.startswith( '#,@' ): + if line and not line.startswith( '#' ) and not line.startswith( '@' ) : fields = line.split( '\t' ) start = int( fields[ int( options.start_col ) - 1 ] ) - 1 end = 0 diff -r ebf68a725c09 -r 8255ed4330e9 tools/samtools/sam_bitwise_flag_filter.py --- a/tools/samtools/sam_bitwise_flag_filter.py Sun Sep 20 11:48:51 2009 -0400 +++ b/tools/samtools/sam_bitwise_flag_filter.py Mon Sep 21 10:32:48 2009 -0400 @@ -135,7 +135,7 @@ for line in infile: line = line.rstrip( '\r\n' ) - if line and not line.startswith( '#' ): + if line and not line.startswith( '#' ) and not line.startswith( '@' ) : fields = line.split( '\t' ) sam_states = [] sam_states.append( bool( int( fields[ int( options.flag_col ) - 1 ] ) & 0x0001 ) ) diff -r ebf68a725c09 -r 8255ed4330e9 tools/sr_mapping/bowtie_wrapper.xml --- a/tools/sr_mapping/bowtie_wrapper.xml Sun Sep 20 11:48:51 2009 -0400 +++ b/tools/sr_mapping/bowtie_wrapper.xml Mon Sep 21 10:32:48 2009 -0400 @@ -336,7 +336,7 @@ </param> </when> </conditional> - <param name="offrate" type="integer" value="-1" label="Override the offrate of the index to n -o)" help="-1 for default" /> + <param name="offrate" type="integer" value="-1" label="Override the offrate of the index to n (-o)" help="-1 for default" /> <param name="seed" type="integer" value="-1" label="Seed for pseudo-random number generator (--seed)" help="-1 for default" /> </when>  </conditional>  @@ -443,31 +443,34 @@ The output is in SAM format, and has the following columns:: - 1 QNAME - Query (pair) NAME - 2 FLAG - bitwise FLAG - 3 RNAME - Reference sequence NAME - 4 POS - 1-based leftmost POSition/coordinate of clipped sequence - 5 MAPQ - MAPping Quality (Phred-scaled) - 6 CIGAR - extended CIGAR string - 7 MRNM - Mate Reference sequence NaMe ('=' if same as RNAME) - 8 MPOS - 1-based Mate POSition - 9 ISIZE - Inferred insert SIZE - 10 SEQ - query SEQuence on the same strand as the reference - 11 QUAL - query QUALity (ASCII-33 gives the Phred base quality) - 12 OPT - variable OPTional fields in the format TAG:VTYPE:VALU + Column Description + -------- -------------------------------------------------------- + 1 QNAME Query (pair) NAME + 2 FLAG bitwise FLAG + 3 RNAME Reference sequence NAME + 4 POS 1-based leftmost POSition/coordinate of clipped sequence + 5 MAPQ MAPping Quality (Phred-scaled) + 6 CIGAR extended CIGAR string + 7 MRNM Mate Reference sequence NaMe ('=' if same as RNAME) + 8 MPOS 1-based Mate POSition + 9 ISIZE Inferred insert SIZE + 10 SEQ query SEQuence on the same strand as the reference + 11 QUAL query QUALity (ASCII-33 gives the Phred base quality) + 12 OPT variable OPTional fields in the format TAG:VTYPE:VALUE The flags are as follows:: - Flag - Description - 0x0001 - the read is paired in sequencing - 0x0002 - the read is mapped in a proper pair - 0x0004 - the query sequence itself is unmapped - 0x0008 - the mate is unmapped - 0x0010 - strand of the query (1 for reverse) - 0x0020 - strand of the mate - 0x0040 - the read is the first read in a pair - 0x0080 - the read is the second read in a pair - 0x0100 - the alignment is not primary + Flag Description + ------ ------------------------------------- + 0x0001 the read is paired in sequencing + 0x0002 the read is mapped in a proper pair + 0x0004 the query sequence itself is unmapped + 0x0008 the mate is unmapped + 0x0010 strand of the query (1 for reverse) + 0x0020 strand of the mate + 0x0040 the read is the first read in a pair + 0x0080 the read is the second read in a pair + 0x0100 the alignment is not primary It looks like this (scroll sideways to see the entire example)::

1 0

Feedback
by Pieter Neerincx 25 Sep '09

25 Sep '09

Hi Galaxy team, The following is just a bunch of thoughts after using Galaxy for a while and which might be interesting for future developments... 1. Interface consistency: "Save" * There are three nice icons at the top of all my dataset items in the history panel on the right for view, edit and delete. So why is there no save icon at the same location instead of a link further down? * When I edit a workflow there is a save button above the canvas and there is another on in the panel on the right when I edit the properties of a specific workflow item. As far as I can tell these buttons are not completely redundant, but why do I need two save buttons? 2. Provenance data * Reproducibility is important and it is nice that Galaxy automatically captures your analysis in histories, but if I want to have a second look at my data after let's say a few months to figure out what I did exactly and how a certain combination of data and tools produced a certain result. Hence if I for example executed a workflow once every two weeks on updated data for many months I might want to retrieve the history for a certain version of a database. So I might want to say give me the histories containing datasets tagged as Ensembl version 48, or UniProt 3 or some version of a reference assembly, etc. Or I might want to see how the results changed for a certain gene over time as result of updated databases and /or tools. So I might want to say to Galaxy show me all histories containing ENSGALG000012589 or NM_45689725. Hence, I'd love to be able to search histories. In addition to make it a bit easier to trace thing in browse mode it would be nice if the date a history was last modified would be visible. Currently I only have the age of the history in minutes, hours or days. That is convenient for recent items, but for things that are longer ago a date makes more sense to me.... * There is a fixed "Database/Build" popup that I can use to tag my data sets, but this feels artificially limited. Is there any reason why the species and database version cannot be separate items? If there would be a popup first to select a species followed by a second popup to select the genome assembly version, the lists could be a lot smaller and hence easier to navigate. In addition there are cases where I do have a species, but don't have an assembly or where there are additional version numbers to keep track of. For example I have lots of Ensembl data. Ensembl does not have a single version number, but 3 version numbers. There is one for the database schema, one for the assembly and one for the annotation/ genebuild. The curent version for mouse is for example: 55 37 h, where 55 is the release and schema version number, 37 the assembly and "h" the version of the gene build. In addition I recently moved to a proteomics group and might want to capture DB version numbers for species without a reference assembly. for example I might know the species name and the fact I'm using UniProt 15.5... but currently I cannot easily capture that in a consistent way. (I know I might add this to the "info" for a dataset, but it's free text, with all kinds of possible spelling variants as a result...) Cheers, Pi ------------------------------------------------------------- Biomolecular Mass Spectrometry and Proteomics Utrecht University Visiting address: H.R. Kruyt building room O607 Padualaan 8 3584 CH Utrecht The Netherlands Mail address: P.O. box 80.082 3508 TB Utrecht The Netherlands phone: +31 (0)6-143 66 783 email: pieter.neerincx(a)gmail.com skype: pieter.online ------------------------------------------------------------

4 4

Feature request: save as
by Chris Cole 25 Sep '09

25 Sep '09

Hi, Is it possible to have a Save As option when saving files? The default naming scheme for saved files is pretty painful, it would be nice to be able to give a sensible filename to a saved item /before/ downloading. Thanks, Chris -- Dr Chris Cole Senior Bioinformatics Research Officer School of Life Sciences Research University of Dundee Dow Street Dundee DD1 5EH Scotland, UK url: http://network.nature.com/profile/drchriscole e-mail: chris(a)compbio.dundee.ac.uk Tel: +44 (0)1382 388 721 The University of Dundee is a registered Scottish charity, No: SC015096

2 2

[hg] galaxy 2730: Template fixes: get current FormDefinition whe...
by Greg Von Kuster 21 Sep '09

21 Sep '09

details: http://www.bx.psu.edu/hg/galaxy/rev/ebf68a725c09 changeset: 2730:ebf68a725c09 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Sun Sep 20 11:48:51 2009 -0400 description: Template fixes: get current FormDefinition when inheriting templates, and use filter then retrieving FormDeinition for library templates. A few other miscellaneous bug fixes as well. Functional tests should now pass. 7 file(s) affected in this change: lib/galaxy/model/__init__.py lib/galaxy/web/controllers/library.py lib/galaxy/web/controllers/library_admin.py lib/galaxy/web/controllers/library_dataset.py templates/library/browse_library.mako templates/library/new_dataset.mako test/functional/test_security_and_libraries.py diffs (247 lines): diff -r 8cbdd73ab68c -r ebf68a725c09 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Sat Sep 19 00:25:14 2009 -0400 +++ b/lib/galaxy/model/__init__.py Sun Sep 20 11:48:51 2009 -0400 @@ -693,7 +693,10 @@ return None, inherited def get_template_widgets( self, trans, get_contents=True ): # See if we have any associated templates - the returned value for - # inherited is not applicable at the library level + # inherited is not applicable at the library level. The get_contents + # param is passed by callers that are inheriting a template - these + # are usually new library datsets for which we want to include template + # fields on the upload form. info_association, inherited = self.get_info_association() if info_association: template = info_association.template @@ -738,10 +741,16 @@ return self.library_root[0].get_info_association( inherited=True ) return None, inherited def get_template_widgets( self, trans, get_contents=True ): - # See if we have any associated templates + # See if we have any associated templates. The get_contents + # param is passed by callers that are inheriting a template - these + # are usually new library datsets for which we want to include template + # fields on the upload form. info_association, inherited = self.get_info_association() if info_association: - template = info_association.template + if inherited: + template = info_association.template.current.latest_form + else: + template = info_association.template # See if we have any field contents, but only if the info_association was # not inherited ( we do not want to display the inherited contents ). if not inherited and get_contents: @@ -881,10 +890,16 @@ return None, inherited return self.library_dataset.folder.get_info_association( inherited=True ) def get_template_widgets( self, trans, get_contents=True ): - # See if we have any associated templates + # See if we have any associated templatesThe get_contents + # param is passed by callers that are inheriting a template - these + # are usually new library datsets for which we want to include template + # fields on the upload form. info_association, inherited = self.get_info_association() if info_association: - template = info_association.template + if inherited: + template = info_association.template.current.latest_form + else: + template = info_association.template # See if we have any field contents, but only if the info_association was # not inherited ( we do not want to display the inherited contents ). if not inherited and get_contents: diff -r 8cbdd73ab68c -r ebf68a725c09 lib/galaxy/web/controllers/library.py --- a/lib/galaxy/web/controllers/library.py Sat Sep 19 00:25:14 2009 -0400 +++ b/lib/galaxy/web/controllers/library.py Sun Sep 20 11:48:51 2009 -0400 @@ -717,11 +717,21 @@ if trans.app.security_agent.can_add_library_item( user, roles, folder ) or \ ( replace_dataset and trans.app.security_agent.can_modify_library_item( user, roles, replace_dataset ) ): if params.get( 'new_dataset_button', False ): + # See if we have any inherited templates, but do not inherit contents. + info_association, inherited = folder.get_info_association( inherited=True ) + if info_association: + template_id = str( info_association.template.id ) + widgets = folder.get_template_widgets( trans, get_contents=False ) + else: + template_id = None + widgets = [] upload_option = params.get( 'upload_option', 'upload_file' ) created_ldda_ids = trans.webapp.controllers[ 'library_dataset' ].upload_dataset( trans, controller='library', library_id=library_id, folder_id=folder_id, + template_id=template_id, + widgets=widgets, replace_dataset=replace_dataset, **kwd ) if created_ldda_ids: @@ -1049,6 +1059,17 @@ library_item_desc = 'library' response_action = 'browse_library' response_id = library_id + forms = get_all_forms( trans, + filter=dict( deleted=False ), + form_type=trans.app.model.FormDefinition.types.LIBRARY_INFO_TEMPLATE ) + if not forms: + msg = "There are no forms on which to base the template, so create a form and " + msg += "try again to add the information template to the %s." % library_item_desc + trans.response.send_redirect( web.url_for( controller='forms', + action='new', + new=True, + msg=msg, + messagetype='done' ) ) if params.get( 'add', False ): if params.get( 'add_info_template_button', False ): form = trans.app.model.FormDefinition.get( int( kwd[ 'form_id' ] ) ) @@ -1068,14 +1089,13 @@ id=response_id, msg=msg, message_type='done' ) ) - return trans.fill_template( '/library/select_info_template.mako', + return trans.fill_template( '/admin/library/select_info_template.mako', library_item_name=library_item.name, library_item_desc=library_item_desc, library_id=library_id, folder_id=folder_id, ldda_id=ldda_id, - forms=get_all_forms( trans, filter=dict(deleted=False), - form_type=trans.app.model.FormDefinition.types.LIBRARY_INFO_TEMPLATE ), + forms=forms, msg=msg, messagetype=messagetype ) @web.expose diff -r 8cbdd73ab68c -r ebf68a725c09 lib/galaxy/web/controllers/library_admin.py --- a/lib/galaxy/web/controllers/library_admin.py Sat Sep 19 00:25:14 2009 -0400 +++ b/lib/galaxy/web/controllers/library_admin.py Sun Sep 20 11:48:51 2009 -0400 @@ -429,11 +429,21 @@ ldatatypes = [ dtype_name for dtype_name, dtype_value in trans.app.datatypes_registry.datatypes_by_extension.iteritems() if dtype_value.allow_datatype_change ] ldatatypes.sort() if params.get( 'new_dataset_button', False ): + # See if we have any inherited templates, but do not inherit contents. + info_association, inherited = folder.get_info_association( inherited=True ) + if info_association: + template_id = str( info_association.template.id ) + widgets = folder.get_template_widgets( trans, get_contents=False ) + else: + template_id = None + widgets = [] upload_option = params.get( 'upload_option', 'upload_file' ) created_ldda_ids = trans.webapp.controllers[ 'library_dataset' ].upload_dataset( trans, controller='library_admin', library_id=library_id, folder_id=folder_id, + template_id=template_id, + widgets=widgets, replace_dataset=replace_dataset, **kwd ) if created_ldda_ids: @@ -915,7 +925,9 @@ library_item_desc = 'library' response_action = 'browse_library' response_id = library_id - forms = get_all_forms( trans, filter=dict(deleted=False) ) + forms = get_all_forms( trans, + filter=dict( deleted=False ), + form_type=trans.app.model.FormDefinition.types.LIBRARY_INFO_TEMPLATE ) if not forms: msg = "There are no forms on which to base the template, so create a form and " msg += "try again to add the information template to the %s." % library_item_desc diff -r 8cbdd73ab68c -r ebf68a725c09 lib/galaxy/web/controllers/library_dataset.py --- a/lib/galaxy/web/controllers/library_dataset.py Sat Sep 19 00:25:14 2009 -0400 +++ b/lib/galaxy/web/controllers/library_dataset.py Sun Sep 20 11:48:51 2009 -0400 @@ -203,16 +203,20 @@ err_redirect = False # See if we have any template field contents template_field_contents = [] - template = None + template_id = params.get( 'template_id', None ) folder = trans.app.model.LibraryFolder.get( folder_id ) - info_association, inherited = folder.get_info_association() - if info_association: - template = info_association.template + # We are inheriting the folder's info_association, so we did not + # receive any inherited contents, but we may have redirected here + # after the user entered template contents ( due to errors ). + if template_id: + template = trans.app.model.FormDefinition.get( template_id ) for field_index in range( len( template.fields ) ): field_name = 'field_%i' % field_index if params.get( field_name, False ): field_value = util.restore_text( params.get( field_name, '' ) ) template_field_contents.append( field_value ) + else: + template = None if upload_option == 'upload_file' and data_file == '' and url_paste == '': msg = 'Select a file, enter a URL or enter text' err_redirect = True diff -r 8cbdd73ab68c -r ebf68a725c09 templates/library/browse_library.mako --- a/templates/library/browse_library.mako Sat Sep 19 00:25:14 2009 -0400 +++ b/templates/library/browse_library.mako Sun Sep 20 11:48:51 2009 -0400 @@ -226,15 +226,15 @@ <h2>Data Library “${library.name}”</h2> <% -can_add = trans.app.security_agent.can_add_library_item( user, roles, library ) -can_modify = trans.app.security_agent.can_modify_library_item( user, roles, library ) -can_manage = trans.app.security_agent.can_manage_library_item( user, roles, library ) + can_add = trans.app.security_agent.can_add_library_item( user, roles, library ) + can_modify = trans.app.security_agent.can_modify_library_item( user, roles, library ) + can_manage = trans.app.security_agent.can_manage_library_item( user, roles, library ) %> <ul class="manage-table-actions"> - %if can_add and not_deleted: + %if can_add: <li> - <a class="action-button" href="${h.url_for( controller='library', action='library_dataset_dataset_association', library_id=library.id, folder_id=library.root_folder.id )}"><span>Add datasets to this library</span></a> + <a class="action-button" href="${h.url_for( controller='library', action='library_dataset_dataset_association', library_id=library.id, folder_id=library.root_folder.id )}"><span>Add datasets to this data library</span></a> </li> <li> <a class="action-button" href="${h.url_for( controller='library', action='folder', new=True, id=library.root_folder.id, library_id=library.id )}">Add a folder to this library</a> @@ -246,7 +246,7 @@ <li><a class="action-button" href="${h.url_for( controller='library', action='library', information=True, id=library.id )}">View this library's information</a></li> %endif %if can_add and forms and not library.info_association: - <a class="action-button" href="${h.url_for( controller='library', action='info_template', library_id=library.id, add=True )}">Add an information template to this library</a> + <a class="action-button" href="${h.url_for( controller='library', action='info_template', library_id=library.id, add=True )}">Add an information template to this data library</a> %endif %if can_manage: <li><a class="action-button" href="${h.url_for( controller='library', action='library', permissions=True, id=library.id )}">Edit this library's permissions</a></li> diff -r 8cbdd73ab68c -r ebf68a725c09 templates/library/new_dataset.mako --- a/templates/library/new_dataset.mako Sat Sep 19 00:25:14 2009 -0400 +++ b/templates/library/new_dataset.mako Sun Sep 20 11:48:51 2009 -0400 @@ -169,9 +169,18 @@ </div> </div> <div style="clear: both"></div> - <% folder = trans.app.model.LibraryFolder.get( folder_id ) %> - %if upload_option == 'upload_file' and widgets: - ${render_template_info( folder, library_id, widgets )} + %if widgets: + <p/> + %for i, field in enumerate( widgets ): + <div class="form-row"> + <label>${field[ 'label' ]}</label> + ${field[ 'widget' ].get_html()} + <div class="toolParamHelp" style="clear: both;"> + ${field[ 'helptext' ]} + </div> + <div style="clear: both"></div> + </div> + %endfor %endif <div class="form-row"> <input type="submit" class="primary-button" name="new_dataset_button" value="Upload to library"/> diff -r 8cbdd73ab68c -r ebf68a725c09 test/functional/test_security_and_libraries.py --- a/test/functional/test_security_and_libraries.py Sat Sep 19 00:25:14 2009 -0400 +++ b/test/functional/test_security_and_libraries.py Sun Sep 20 11:48:51 2009 -0400 @@ -488,7 +488,7 @@ .order_by( galaxy.model.FormDefinitionCurrent.table.c.create_time.desc() ) \ .all() for fdc in fdcs: - if form_name == fdc.latest_form.name: + if form_name == fdc.latest_form.name and form_type == fdc.latest_form.type: form_one = fdc.latest_form break assert form_one is not None, 'Problem retrieving form named (%s) from the database' % form_name

1 0