- galaxy-commits - lists.galaxyproject.org

galaxy-dist commit cdd6ce1f38b0: Enable filter tool to operate correctly on files with a variable number of columns (such as SAM files). Tool prepares and processes only columns up to and including the right-most column (i.e. column with the largest index) in the filtering condition, and subsequent columns are ignored. New functional test added for this functionality as well.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User jeremy goecks <jeremy.goecks(a)emory.edu> # Date 1288902634 14400 # Node ID cdd6ce1f38b0e22802ed63e3e0cb4f6a58cb93b2 # Parent f372ea5a601443a9b9cd1d3f7e04ddc24db5b57a Enable filter tool to operate correctly on files with a variable number of columns (such as SAM files). Tool prepares and processes only columns up to and including the right-most column (i.e. column with the largest index) in the filtering condition, and subsequent columns are ignored. New functional test added for this functionality as well. --- a/tools/stats/filtering.py +++ b/tools/stats/filtering.py @@ -60,10 +60,18 @@ for operand in operands: except: if operand in secured: stop_err( "Illegal value '%s' in condition '%s'" % ( operand, cond_text ) ) + +# Find the largest column used in the filter. +largest_col_index = -1 +for match in re.finditer( 'c(\d)+', cond_text ): + col_index = int( match.group()[1:] ) + if col_index > largest_col_index: + largest_col_index = col_index -# Prepare the column variable names and wrappers for column data types +# Prepare the column variable names and wrappers for column data types. Only +# prepare columns up to largest column in condition. cols, type_casts = [], [] -for col in range( 1, in_columns + 1 ): +for col in range( 1, largest_col_index + 1 ): col_name = "c%d" % col cols.append( col_name ) col_type = in_column_types[ col - 1 ] @@ -72,7 +80,7 @@ for col in range( 1, in_columns + 1 ): col_str = ', '.join( cols ) # 'c1, c2, c3, c4' type_cast_str = ', '.join( type_casts ) # 'str(c1), int(c2), int(c3), str(c4)' -assign = "%s = line.split( '\\t' )" % col_str +assign = "%s, = line.split( '\\t' )[:%i]" % ( col_str, largest_col_index ) wrap = "%s = %s" % ( col_str, type_cast_str ) skipped_lines = 0 first_invalid_line = 0 --- a/tools/stats/filtering.xml +++ b/tools/stats/filtering.xml @@ -23,6 +23,13 @@ <param name="cond" value="c1=='chr1' and c3-c2>=2000 and c6=='+'"/><output name="out_file1" file="filter1_test2.bed"/></test> +  + <test> + <param name="input" value="filter1_in3.sam"/> + <param name="cond" value="c3=='chr1' and c5>5"/> + <output name="out_file1" file="filter1_test3.sam"/> + </test> + </tests><help> --- /dev/null +++ b/test-data/filter1_in3.sam @@ -0,0 +1,100 @@ +HWI-EAS269B:8:34:797:623 145 chr1 16632182 255 27M = 16631977 0 CCATTTCCTGTATGCTGTAAAGTACAA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:40:39:1184 145 chr1 24620331 3 27M = 24620115 0 ATTTATGTGGTTTCGTTTACCTTCTAT IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:2 CC:Z:chrM CP:i:9060 +HWI-EAS269B:8:58:533:1198 81 chr1 88426079 255 27M = 88423429 0 GAAGAGGAAGAAGGTGGGGAGGAAGAG IIIG?IIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:89:1776:1815 97 chr1 134085638 255 27M = 134085824 0 GAATGATTCTCTGGGTGTTACTTTGCA IIIIIIIIIIIIIIIDIII:IIII>F5 NM:i:0 NH:i:1 +HWI-EAS269B:8:74:1134:1670 161 chr1 138166886 255 27M = 138167084 0 TTACTAGTGTCTCTCTTACCATCATAT .IIIIIIIIIIIIIIIIIIIIIIIIII NM:i:1 NH:i:1 +HWI-EAS269:3:59:1321:1427 147 chr1 173149715 255 27M = 173149555 0 AAGGGCTAGGGTGACAGGCAGGGGACG -C<CID?IIIIIIIDIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:8:164:1678 145 chr1 178660716 255 27M = 178660493 0 CAATTGGTGTTTTTCTTAAGAGACTCA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:1:1048:638 137 chr10 12456478 1 27M * 0 0 TAAAANAAATAAAACAAAACAATAAAA $'&(+")'%$(&*&#&$$#%%$$%%$$ NM:i:2 NH:i:4 CC:Z:chr16 CP:i:21639623 +HWI-EAS269:3:62:1211:798 147 chr10 28135294 255 27M = 28135117 0 ACATGGTGTGGGGACAGAGATGTGAAG I;IIIIIIAIIIIIIIIIIIIIIIIII NM:i:1 NH:i:1 +HWI-EAS269:3:27:410:424 145 chr10 76950070 255 27M = 76949871 0 GAGTCTGTGTCCAGGCCAATTCACTAT 25-8I+.6B.IIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:48:1180:1122 73 chr10 83772157 0 27M * 0 0 GATCATCTTTTCTAAAACAATAAAGAC /IIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:6 CC:Z:chr11 CP:i:93811140 +HWI-EAS269:3:77:654:131 99 chr10 93419810 255 27M = 93419993 0 TTGCATCCCTAGGAACTGGAGTTATAG IIIIIIGIHIIIED@CIIH5I3D9G6: NM:i:0 NH:i:1 +HWI-EAS269B:8:98:895:1810 177 chr11 3371472 255 27M = 37952212 0 CTACATAGTGGGAACCGGCGACCGCTG IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:2 NH:i:1 +HWI-EAS269B:8:58:1126:883 97 chr11 30080486 255 27M = 30080702 0 TACTTCCACCAGGCTCCAGTTTTGTGA IIIIIIIIIIIIIIIIIIIIIIIIII5 NM:i:0 NH:i:1 +HWI-EAS269B:8:89:1763:1446 81 chr11 59589605 255 27M = 59589400 0 GTACCTGGCCACTGATGCAGCTTAGAA II<IIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:6:739:1329 161 chr11 68629628 3 27M = 68629835 0 GTTTTTGGTTGATAGTTGAGCAAACTG IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:2 CC:Z:chr4 CP:i:126661877 +HWI-EAS269:3:19:1683:1350 81 chr11 70846915 255 27M = 70846725 0 CACGGGCAGATAAGCTGCTGAGACTAA H50BB:IF=IIGIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:9:1691:1087 97 chr11 84640783 255 27M = 84641010 0 ACATTATCCATCTCTCTGTCATTGTCC IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:48:1350:424 81 chr11 94150331 255 27M = 94150113 0 TTGCAAGCCAACCCTGAGTGAAGTGTC IIDHIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:22:1176:1405 97 chr11 98629843 255 27M = 98630030 0 CCCTCGGAGGGCAACGGGAAAGGAGAA IIIIIIIIIIIDIIIII?III/79E14 NM:i:0 NH:i:1 +HWI-EAS269:3:67:1311:1512 97 chr11 100488190 255 27M = 100488374 0 CATGCCCTGAGACTTAGCAAGACTCTT IIIIIIIIIIIIIIIEIA3,I57GBI@ NM:i:0 NH:i:1 +HWI-EAS269B:8:41:1142:1307 161 chr11 101320424 255 27M = 101320625 0 GCCTCCCTACATAGCAAAGGAAAGAAA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:31:1101:21 97 chr11 116711287 255 27M = 116711485 0 CATAAGCAAAAGATACACCATGTTTTA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:84:637:1203 145 chr12 35829424 255 27M = 35829229 0 CTCCCTCAAGGATCTTGTGGGGCATCA </'>I,I5B26$II;EB=%IIII1III NM:i:0 NH:i:1 +HWI-EAS269:3:65:1158:1081 163 chr12 52234332 255 27M = 52234505 0 TTTTTTTTTTAAGACAGGGAGTTTTTT IIIIIIIIIIIC4,,*>II%-,%III> NM:i:1 NH:i:1 +HWI-EAS269B:8:65:1325:1287 137 chr12 52234513 3 27M * 0 0 GCAGCACCTTGTGACATTTAATTTAGT IIIII+IIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:2 CC:Z:chr15 CP:i:98780443 +HWI-EAS269:3:38:488:1093 97 chr12 80471642 255 27M = 80471839 0 TGGAGAGATGACTCCTTGTTTGAGACA IIIIIIIIIIIIIIIIIIIBAE0B<7D NM:i:0 NH:i:1 +HWI-EAS269:3:76:290:1451 99 chr12 117745566 0 27M = 117745739 0 CAATCTGAAGATCCACAATCTTTTATA IIIIIIIIIIIIIIFII5I9IIII+I7 NM:i:0 NH:i:5 CC:Z:chr4 CP:i:133520935 +HWI-EAS269B:8:39:353:1526 65 chr13 18122558 255 27M = 18122761 0 ACTCTACTCAAAACCACACTAAGCCTC @IIIIII6IIIIIDII9IIIIII<III NM:i:0 NH:i:1 +HWI-EAS269:3:63:1260:365 99 chr13 41278147 255 27M = 41278319 0 AAGACAAGAACTTATCCACCAATATGT IIIIIIIIIIIIIII<IIA<CIIDII> NM:i:0 NH:i:1 +HWI-EAS269:3:70:152:1609 161 chr13 55421426 255 27M = 55421687 0 CGCTTGACCATTCCAGCCCAGACAAGA IIIIIIIII8IIII3IIII-C(3-%B' NM:i:0 NH:i:1 +HWI-EAS269:3:9:1343:1414 99 chr13 83721797 255 27M = 83721967 0 AACTACAGCTGAGGCAGCCTCCTGCCT IIIIIIIIIIIIIIIII;E+?7&819& NM:i:0 NH:i:1 +HWI-EAS269:3:4:1480:1956 137 chr14 46703849 0 27M * 0 0 GTGGAGCCCAGTGACACCATCGAGAAC IIIIIIIIIIIII?C3IH?1@I@@=27 NM:i:0 NH:i:8 CC:Z:= CP:i:46704077 +HWI-EAS269:3:86:616:1519 137 chr14 56244763 255 27M * 0 0 CATGGCCTGAAGTTCCTGAGCTTTATC IIIIIIIII3DIBI9II3)73BIG'G+ NM:i:0 NH:i:1 +HWI-EAS269:3:42:656:1216 97 chr14 100271567 255 27M = 100271751 0 ATAAGTCTCAGTTCTTGGGCCAGATCA IIIIIIIIICIIIIII3IA854I1C+) NM:i:0 NH:i:1 +HWI-EAS269:3:87:95:767 73 chr16 7368880 255 27M * 0 0 AATATCTGAAACATTCAAATGAGCAAT F>II/4I@9H=II?IIIIIIIIIIII- NM:i:0 NH:i:1 +HWI-EAS269B:8:50:1572:711 145 chr16 32230400 255 27M = 32230196 0 TACAGCAACAACAAATTCAACGACACG IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:56:1436:121 147 chr16 49887529 255 27M = 49887356 0 TTAAGGCCCAGCTCTACATAAAACACT IIIIAHIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:53:584:1696 99 chr17 45705815 255 27M = 45705987 0 CCTTCTTGTCCAAGATCTCCTTCATGA IIIIIIIIIIIIBI6IIII=II:GI@< NM:i:0 NH:i:1 +HWI-EAS269B:8:16:643:1950 177 chr18 38000738 255 27M = 38000570 0 ATTCTGGTTCAGCCTGGGCAGCTTGGG III@==I0<IIGIII;IIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:66:1760:1539 137 chr18 38624681 3 27M * 0 0 TTTGGAACTTTTGAGAGGATCCCTAGC IIIIIIIIIIIIIII1II.IIIII1II NM:i:0 NH:i:2 +HWI-EAS269:3:2:1723:1277 163 chr19 16341705 255 27M = 16341870 0 ATGGGCTCGTCGCAGCTCAGCGGCTGG IIIIIIIIIIIIICI=8<&I7:F(+;2 NM:i:1 NH:i:1 +HWI-EAS269B:8:66:317:1676 97 chr19 47099316 255 27M = 47099535 0 ATCTGGTCAATCAACACCACCAGCAGA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:10:1428:1315 81 chr2 22444224 3 27M = 22444039 0 GGAATTGCGATAATTATAGTGGCTGAT ?6>CI>GIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:2 CC:Z:chrM CP:i:6245 +HWI-EAS269:3:72:152:785 97 chr2 22796881 255 27M = 22797074 0 TGAGTACTCCAGATAATCGTTACACAA IIIIIIII6IIIIIIII0IA@?,7752 NM:i:0 NH:i:1 +HWI-EAS269:3:79:321:1095 99 chr2 62083760 255 27M = 62083938 0 CAAAAATTGAGATATCAAAAAGCTCTT IIIIIIIII2IGIBIIC8:4IB1H4I; NM:i:0 NH:i:1 +HWI-EAS269:3:93:1529:881 97 chr2 129998362 255 27M = 129998827 0 CGCATGCCAGGAGGGGGCATGCCCATT IIIFIIIIFIIIIIFII?72I.,020* NM:i:0 NH:i:1 +HWI-EAS269B:8:20:813:376 113 chr2 130534278 255 27M = 130534454 0 TGTATTAGCAGGAGGTGGGGAGGCTGA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:89:1221:1082 83 chr2 151285071 0 27M = 151284944 0 ACCAGTGCACAGGTCTCCAGGGCTTCT -IC=B+8IIIEIIFI5IIIIIIIIIII NM:i:0 NH:i:5 CC:Z:chr9 CP:i:31743980 +HWI-EAS269:3:66:46:336 163 chr2 156679837 255 27M = 156680016 0 TATTTTCCTTTTGCTGTGGTTTGTGTT IIIIIIIIIIIIIEI@E-I?GI&B%3I NM:i:0 NH:i:1 +HWI-EAS269B:8:52:1139:1381 73 chr2 174953004 0 27M * 0 0 AATGCTCAACTCTTAGTTTCTTATTCA IIIIIIIIIIIIIIIIB@?IIIII?+0 NM:i:0 NH:i:19 CC:Z:= CP:i:175018018 +HWI-EAS269:3:98:585:19 161 chr3 26005087 255 27M = 26005271 0 ACCTAACACATGTAAACTTAAATTCAT I:IIIIIIII<IIE==DDI):,D1--8 NM:i:0 NH:i:1 +HWI-EAS269:3:74:447:1309 83 chr3 80492396 255 27M = 80492220 0 TCGGATGCCTCTCACCACTTTGACAAT )H():F=85IIIIEIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:74:425:1427 161 chr3 96447603 255 27M = 96447820 0 TGGAGTAGGAGTCTCAGGAGGAGTAGA IIIIIIIIIIIIIIIIIIIII:I?II< NM:i:0 NH:i:1 +HWI-EAS269:3:10:739:1474 163 chr3 124089371 255 27M = 124089535 0 AAAAAAACAATCTTATTCCGAGCATTC IIIIIIIIIIIIIIIIIIIIIIIGIII NM:i:0 NH:i:1 +HWI-EAS269:3:20:357:1709 65 chr4 128895987 255 27M = 128895822 0 CCTACCTTCTTCCCTTGGCAGCTGACT IIII>IIIBIIA:I18):,*3&,/*'+ NM:i:0 NH:i:1 +HWI-EAS269B:8:22:623:129 113 chr4 135492867 255 27M = 135490613 0 CCACTTTCCTGTACTGGCCAGAAAATG IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:48:824:702 65 chr4 139584010 255 27M = 139582899 0 CTGGGCAGTGCAGCGGTACATGGAGCC IIIIIIIIIIIIIIIIII<AIIB;I>: NM:i:0 NH:i:1 +HWI-EAS269B:8:33:527:1211 73 chr5 37627410 255 27M * 0 0 GGGAAGGTGTGAGTACAACAGCCAAAG IIIIIIIIIIIIIIIIIIIIIIII@II NM:i:0 NH:i:1 +HWI-EAS269:3:2:1518:599 163 chr5 136908928 255 27M = 136909101 0 CTATTGCCAAAAAACTATGTTCACAAA IIIIIIIIIIIIIIIIIIIIIIIII=I NM:i:0 NH:i:1 +HWI-EAS269B:8:62:1493:1455 161 chr5 138715332 255 27M = 138715534 0 CCCAAATGAAAAAATAAATATTATGAA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:69:384:1894 147 chr5 143665243 255 27M = 143665064 0 TGTTTGCTCCAACCAACTGCTGTCGCC &++*,2H-IGI+IFIGI?IIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:56:1358:762 113 chr6 15024174 255 7M12235N20M = 51485453 0 TGGGTACTTTCTCTAGCTCCTCCATTG /@)3I?IIIIIIIIIIIIIIIIIIIII NM:i:2 XS:A:- NH:i:1 +HWI-EAS269:3:100:228:1799 163 chr6 37756378 3 27M = 37756558 0 GTGTGGGGCTGCGTGGCCTGGCTGGTG DII4A+>5EI.)F634820&1(0%&&& NM:i:2 NH:i:2 +HWI-EAS269B:8:50:188:1253 161 chr6 52296684 1 27M = 52296901 0 ACTTTTCAGGGTTTTCAATAGTCACAC IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:1 NH:i:3 CC:Z:= CP:i:140695441 +HWI-EAS269B:8:19:440:1687 161 chr6 52850377 255 27M = 52850580 0 TACAAAGATGGACTTTTAAAATTCATT IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:47:1555:1520 161 chr6 67231655 255 27M = 67233919 0 ATTCAGTTATGGACCATCATTTCCGGA IIIIIIIIIIIIIIIIIIIIII<AII1 NM:i:0 NH:i:1 +HWI-EAS269B:8:79:1563:1318 161 chr6 89267414 255 27M = 89267614 0 GTGCGTGTTCCAGGCAGAGCTGGAAGA IIIIIIIIIIIIIIIIIIIIIII=;IA NM:i:0 NH:i:1 +HWI-EAS269B:8:54:954:565 161 chr6 120882513 255 27M = 120882716 0 ACGTCATGGCTGACCAGGACAGAGGTG IIIIIIIIIIIIIII?IEG;II<II*8 NM:i:0 NH:i:1 +HWI-EAS269B:8:16:570:1775 145 chr7 26081268 255 27M = 26080313 0 GAAAGAGTGACACAAATCAATAGTAAA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:13:280:189 161 chr7 29184615 255 27M = 29184814 0 CTTTTCCAACAGCGAGAAAAATGTACA IIIIIIIIIIIIIIIIIIIIIIIIDII NM:i:0 NH:i:1 +HWI-EAS269B:8:19:72:273 137 chr7 52258521 255 27M * 0 0 ATCAGAGGCACAGGGACAGGGTAAGGA &&2IIIII;E(IIIIIIIIIIIIIIII NM:i:1 NH:i:1 +HWI-EAS269B:8:40:235:869 161 chr7 52631974 255 27M = 52632179 0 GGCGCTGACTCCATCAGATATCCATTC IIIIIII7IIIIIII6IIIIIIIBIII NM:i:1 NH:i:1 +HWI-EAS269B:8:18:304:346 129 chr7 107789557 255 27M = 33158390 0 CACGTACTGTCACCTTGTAACATTTGG IIIIIIIIIIIIIIIIIIIIIIIIIIF NM:i:0 NH:i:1 +HWI-EAS269B:8:3:698:297 161 chr7 110976065 255 27M = 110976385 0 GCAGTTATCACTTTCTTGCCATGGGCC IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:50:1269:965 145 chr7 118217605 255 27M = 118216239 0 ACCTGTAGATCCACATGATCATGAAGA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:38:1381:1429 83 chr7 136141344 255 27M = 136141280 0 ATCTGAAGTATCCCACATGTTGAGCTC <III@IIIIIIIII>IIIIIIIIIIII NM:i:1 NH:i:1 +HWI-EAS269:3:25:781:386 83 chr8 29844005 255 27M = 29843841 0 TAAGGGAGGAAAGTGTTTCAGAGTGTA ;83<159<<III@IIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:25:979:679 163 chr8 72664237 255 27M = 72664413 0 CTGATGGGAGCCCTGCGTGGTAAGAGG IIIIIIIIIII=III@I(II$27I>I4 NM:i:0 NH:i:1 +HWI-EAS269B:8:23:1069:816 145 chr8 74704433 255 27M = 74704222 0 TGTTCTCAGTTGTGGACAAGTGACAGC I<GII@IIIIIIII;IIIIIIIIII0I NM:i:0 NH:i:1 +HWI-EAS269B:8:34:435:1679 73 chr8 87366211 3 27M * 0 0 AAGCCTAGGGCTTCTCCTCTACACCCC I556I;FCAIIIFI<IIIIIIIIIIII NM:i:0 NH:i:2 +HWI-EAS269B:8:32:1486:294 97 chr8 124121215 255 27M = 124121411 0 TGTTACCATACGCCCTTCTGCTGAGGC IIIIIIIIIIIIIIIIIIDIIIIGIII NM:i:0 NH:i:1 +HWI-EAS269B:8:32:142:1471 81 chr8 124496752 3 27M = 124496536 0 GGATCTGCTTCATGAGTTGCCACATTG IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:1 NH:i:2 +HWI-EAS269B:8:60:82:1806 97 chr8 125945215 255 27M = 125945423 0 AGATGCTGGCCATCCAGAGTAAGAACA IIIIIIIIIIIIIIIIIIIHIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:16:1162:495 81 chr8 125945584 255 27M = 125945381 0 GTTGCTCGCAGCTGGGGTGTGGGGCCA <CIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:1:165:1889 81 chr8 126016885 255 27M = 126016469 0 TGAGCAGGAAAACACTTTAAACCAGAT IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:66:939:1154 145 chr9 17537149 255 27M = 17536959 0 CTCCTTCAAGTACGCCTGGGTCTTAGA IDIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:68:1189:285 81 chr9 61249807 255 27M = 61242460 0 AGTCAAGCTCACTTGGCGGTGAAGGAT @.0;;A,?-F5I;7IIIIGIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:15:784:1754 163 chr9 74846937 255 27M = 74847108 0 GCACAGCACTGAGGAAAGGATCATCTC IIIIIIIIIIIIIIIIIIGII1CI7II NM:i:0 NH:i:1 +HWI-EAS269B:8:26:126:1382 145 chrM 3413 255 27M = 3190 0 GTTATTCTTTATAGCAGAGTACACTAA IIII8IB328I9IIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:90:320:1336 73 chrM 6326 255 27M * 0 0 ATGATCTCCAGCTATCCTATGAGCCTT IIIIIIIIA.ICI.4'(=,C>7-*&@8 NM:i:1 NH:i:1 +HWI-EAS269B:8:23:469:1215 161 chrM 7472 3 27M = 7676 0 ATTTCATCTGAAGACGTCCTCCACTCA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:2 +HWI-EAS269:3:73:36:783 161 chrM 7958 3 27M = 8152 0 CCCCAACAATAATAGGATTCCCAATCG IIIIIIIIIIIIIIIIIIIII09I>>I NM:i:0 NH:i:2 +HWI-EAS269:3:33:1528:954 99 chrM 8012 3 27M = 8186 0 TCCTATTCCCATCCTCAAAACGCCTAA IIIIIIIIIIIIIIIIIIIIIIIDII: NM:i:0 NH:i:2 +HWI-EAS269:3:2:192:1456 161 chrM 9071 3 27M = 9255 0 ACGAAACCACATAAATCAAGCCCTACT III;IEIIDI7III+III*?I@CH+5I NM:i:0 NH:i:2 +HWI-EAS269:3:63:192:470 97 chrM 14787 255 27M = 14982 0 GCAGATAAAATTCCATTTCACCCCTAC IIIIIIIIIIIIIIFIII=B5042:4E NM:i:0 NH:i:1 +HWI-EAS269B:8:49:528:63 83 chrM 14963 255 27M = 14829 0 TTTCCTATTTGCATACGCCATTCTACG IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:98:856:1516 145 chrX 20257960 255 27M = 20256746 0 TACCCGGATTTAAGATGTACCCCATTG IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:28:1174:110 145 chrX 53916348 255 27M = 53916153 0 TGAATGTCAGCATCATTGACCCACAAA IIIIFIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:58:663:410 147 chrX 75006902 255 27M = 75006834 0 TCAGGTGGTTTACAGTGTTCTGACAAA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 --- /dev/null +++ b/test-data/filter1_test3.sam @@ -0,0 +1,6 @@ +HWI-EAS269B:8:34:797:623 145 chr1 16632182 255 27M = 16631977 0 CCATTTCCTGTATGCTGTAAAGTACAA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:58:533:1198 81 chr1 88426079 255 27M = 88423429 0 GAAGAGGAAGAAGGTGGGGAGGAAGAG IIIG?IIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:89:1776:1815 97 chr1 134085638 255 27M = 134085824 0 GAATGATTCTCTGGGTGTTACTTTGCA IIIIIIIIIIIIIIIDIII:IIII>F5 NM:i:0 NH:i:1 +HWI-EAS269B:8:74:1134:1670 161 chr1 138166886 255 27M = 138167084 0 TTACTAGTGTCTCTCTTACCATCATAT .IIIIIIIIIIIIIIIIIIIIIIIIII NM:i:1 NH:i:1 +HWI-EAS269:3:59:1321:1427 147 chr1 173149715 255 27M = 173149555 0 AAGGGCTAGGGTGACAGGCAGGGGACG -C<CID?IIIIIIIDIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:8:164:1678 145 chr1 178660716 255 27M = 178660493 0 CAATTGGTGTTTTTCTTAAGAGACTCA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1

1 0

galaxy-dist commit 702f4717a8f3: Bugfix for the abstracted job runner
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Nate Coraor <nate(a)bx.psu.edu> # Date 1288986871 14400 # Node ID 702f4717a8f3fe38758c54136f81935f7679c937 # Parent 6d94416b2ccf9205283ec197154d99522615cbf8 Bugfix for the abstracted job runner --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -26,7 +26,7 @@ class BaseJobRunner( object ): exec_dir = os.path.abspath( os.getcwd() ), tmp_dir = self.app.config.new_file_path, dataset_files_path = self.app.model.Dataset.file_path, - output_fnames = output_fnames, + output_fnames = job_wrapper.get_output_fnames(), set_extension = False, kwds = { 'overwrite' : False } ) return commands

1 0

galaxy-dist commit 32e5efb7a7d5: Remove the need for setup.sh - All configuration is done in run.sh/galaxy.config.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Nate Coraor <nate(a)bx.psu.edu> # Date 1288983302 14400 # Node ID 32e5efb7a7d5606a4dc1c8808fe4888ade2892e8 # Parent eb79ab327351e4789a89748269fc34428f791463 Remove the need for setup.sh - All configuration is done in run.sh/galaxy.config. --- a/run.sh +++ b/run.sh @@ -2,6 +2,53 @@ cd `dirname $0` +python ./scripts/check_python.py +[ $? -ne 0 ] && exit 1 + +FROM_SAMPLE=" + datatypes_conf.xml + reports_wsgi.ini + tool_conf.xml + tool_data_table_conf.xml + universe_wsgi.ini + tool-data/add_scores.loc + tool-data/alignseq.loc + tool-data/annotation_profiler_options.xml + tool-data/annotation_profiler_valid_builds.txt + tool-data/bfast_indexes.loc + tool-data/binned_scores.loc + tool-data/blastdb.loc + tool-data/blastdb_p.loc + tool-data/bowtie_indices.loc + tool-data/bowtie_indices_color.loc + tool-data/codingSnps.loc + tool-data/encode_datasets.loc + tool-data/funDo.loc + tool-data/lastz_seqs.loc + tool-data/liftOver.loc + tool-data/maf_index.loc + tool-data/maf_pairwise.loc + tool-data/microbial_data.loc + tool-data/phastOdds.loc + tool-data/perm_base_index.loc + tool-data/perm_color_index.loc + tool-data/quality_scores.loc + tool-data/regions.loc + tool-data/sam_fa_indices.loc + tool-data/sift_db.loc + tool-data/srma_index.loc + tool-data/twobit.loc + tool-data/shared/ucsc/builds.txt +" + +# Create any missing config/location files +for file in $FROM_SAMPLE; do + if [ ! -f "$file" -a -f "$file.sample" ]; then + echo "Initializing $file from `basename $file`.sample" + cp $file.sample $file + fi +done + # explicitly attempt to fetch eggs before running FETCH_EGGS=1 for arg in "$@"; do @@ -21,10 +68,4 @@ if [ $FETCH_EGGS -eq 1 ]; then fi fi -# Temporary: since builds.txt is now removed from source control, create it -# from the sample if necessary -if [ ! -f "tool-data/shared/ucsc/builds.txt" ]; then - cp tool-data/shared/ucsc/builds.txt.sample tool-data/shared/ucsc/builds.txt -fi - python ./scripts/paster.py serve universe_wsgi.ini $@ --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -94,14 +94,8 @@ class Configuration( object ): self.blog_url = kwargs.get( 'blog_url', None ) self.screencasts_url = kwargs.get( 'screencasts_url', None ) self.library_import_dir = kwargs.get( 'library_import_dir', None ) - if self.library_import_dir is not None and not os.path.exists( self.library_import_dir ): - raise ConfigurationError( "library_import_dir specified in config (%s) does not exist" % self.library_import_dir ) self.user_library_import_dir = kwargs.get( 'user_library_import_dir', None ) - if self.user_library_import_dir is not None and not os.path.exists( self.user_library_import_dir ): - raise ConfigurationError( "user_library_import_dir specified in config (%s) does not exist" % self.user_library_import_dir ) self.ftp_upload_dir = kwargs.get( 'ftp_upload_dir', None ) - if self.ftp_upload_dir is not None and not os.path.exists( self.ftp_upload_dir ): - os.makedirs( self.ftp_upload_dir ) self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None ) self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False ) self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' ) @@ -159,9 +153,16 @@ class Configuration( object ): return default def check( self ): # Check that required directories exist - for path in self.root, self.file_path, self.tool_path, self.tool_data_path, self.template_path, self.job_working_directory, self.cluster_files_directory: + for path in self.root, self.tool_path, self.tool_data_path, self.template_path: if not os.path.isdir( path ): raise ConfigurationError("Directory does not exist: %s" % path ) + # Create the directories that it makes sense to create + for path in self.file_path, self.new_file_path, self.job_working_directory, self.cluster_files_directory, self.template_cache, self.ftp_upload_dir, self.library_import_dir, self.user_library_import_dir, self.nginx_upload_store, './static/genetrack/plots', os.path.join( self.tool_data_path, 'shared', 'jars' ): + if path not in [ None, False ] and not os.path.isdir( path ): + try: + os.makedirs( path ) + except Exception, e: + raise ConfigurationError( "Unable to create missing directory: %s\n%s" % ( path, e ) ) # Check that required files exist for path in self.tool_config, self.datatypes_config: if not os.path.isfile(path):

1 0

galaxy-dist commit 1e8d7c6ad88b: Add support for tool dependency injection to all cluster runners
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User James Taylor <james(a)jamestaylor.org> # Date 1288983797 14400 # Node ID 1e8d7c6ad88b074b33b2029f3baa15c2eb929e70 # Parent 32e5efb7a7d5606a4dc1c8808fe4888ade2892e8 Add support for tool dependency injection to all cluster runners --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -51,7 +51,7 @@ class DependencyManager( object ): if os.path.exists( script ): return script, path, version else: - return None, None + return None, None, None def _find_dep_default( self, name ): version = None @@ -64,6 +64,6 @@ class DependencyManager( object ): if os.path.exists( script ): return script, real_path, real_version else: - return None, None + return None, None, None --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -0,0 +1,32 @@ +import os, os.path + +class BaseJobRunner( object ): + + def build_command_line( self, job_wrapper, include_metadata=False ): + """ + Compose the sequence of commands neccesary to execute a job. This will + currently include: + - environment settings corresponding to any requirement tags + - command line taken from job wrapper + - commands to set metadata (if include_metadata is True) + """ + commands = job_wrapper.get_command_line() + # All job runners currently handle this case which should never + # occur + if not commands: + return None + # Prepend dependency injection + if job_wrapper.dependency_shell_commands: + commands = "; ".join( job_wrapper.dependency_shell_commands + [ commands ] ) + # Append metadata setting commands, we don't want to overwrite metadata + # that was copied over in init_meta(), as per established behavior + if include_metadata and self.app.config.set_metadata_externally: + commands += "; cd %s; " % os.path.abspath( os.getcwd() ) + commands += job_wrapper.setup_external_metadata( + exec_dir = os.path.abspath( os.getcwd() ), + tmp_dir = self.app.config.new_file_path, + dataset_files_path = self.app.model.Dataset.file_path, + output_fnames = output_fnames, + set_extension = False, + kwds = { 'overwrite' : False } ) + return commands --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -5,13 +5,14 @@ import threading from galaxy import model from galaxy.datatypes.data import nice_size +from galaxy.jobs.runners import BaseJobRunner import os, errno from time import sleep log = logging.getLogger( __name__ ) -class LocalJobRunner( object ): +class LocalJobRunner( BaseJobRunner ): """ Job runner backed by a finite pool of worker threads. FIFO scheduling """ @@ -53,9 +54,7 @@ class LocalJobRunner( object ): # Prepare the job to run try: job_wrapper.prepare() - command_line = job_wrapper.get_command_line() - if job_wrapper.dependency_shell_commands: - command_line = "; ".join( job_wrapper.dependency_shell_commands + [ command_line ] ) + command_line = self.build_command_line( job_wrapper ) except: job_wrapper.fail( "failure preparing job", exception=True ) log.exception("failure running job %d" % job_wrapper.job_id) --- a/lib/galaxy/jobs/runners/sge.py +++ b/lib/galaxy/jobs/runners/sge.py @@ -2,6 +2,8 @@ import os, logging, threading, time from Queue import Queue, Empty from galaxy import model +from galaxy.jobs.runners import BaseJobRunner + from paste.deploy.converters import asbool import pkg_resources @@ -58,7 +60,7 @@ class SGEJobState( object ): self.efile = None self.runner_url = None -class SGEJobRunner( object ): +class SGEJobRunner( BaseJobRunner ): """ Job runner backed by a finite pool of worker threads. FIFO scheduling """ @@ -144,7 +146,7 @@ class SGEJobRunner( object ): try: job_wrapper.prepare() - command_line = job_wrapper.get_command_line() + command_line = self.build_command_line( job_wrapper, include_metadata = True ) except: job_wrapper.fail( "failure preparing job", exception=True ) log.exception("failure running job %d" % job_wrapper.job_id) @@ -191,14 +193,7 @@ class SGEJobRunner( object ): jt.nativeSpecification = ' '.join(nativeSpec) script = sge_template % (job_wrapper.galaxy_lib_dir, os.path.abspath( job_wrapper.working_directory ), command_line) - if self.app.config.set_metadata_externally: - script += "cd %s\n" % os.path.abspath( os.getcwd() ) - script += "%s\n" % job_wrapper.setup_external_metadata( exec_dir = os.path.abspath( os.getcwd() ), - tmp_dir = self.app.config.new_file_path, - dataset_files_path = self.app.model.Dataset.file_path, - output_fnames = job_wrapper.get_output_fnames(), - set_extension = False, - kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior + fh = file( jt.remoteCommand, "w" ) fh.write( script ) fh.close() --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -2,6 +2,8 @@ import os, logging, threading, time from Queue import Queue, Empty from galaxy import model +from galaxy.jobs.runners import BaseJobRunner + from paste.deploy.converters import asbool import pkg_resources @@ -58,7 +60,7 @@ class DRMAAJobState( object ): self.efile = None self.runner_url = None -class DRMAAJobRunner( object ): +class DRMAAJobRunner( BaseJobRunner ): """ Job runner backed by a finite pool of worker threads. FIFO scheduling """ @@ -120,7 +122,7 @@ class DRMAAJobRunner( object ): try: job_wrapper.prepare() - command_line = job_wrapper.get_command_line() + command_line = self.build_command_line( job_wrapper, include_metadata=True ) except: job_wrapper.fail( "failure preparing job", exception=True ) log.exception("failure running job %d" % job_wrapper.job_id) @@ -154,14 +156,6 @@ class DRMAAJobRunner( object ): jt.nativeSpecification = native_spec script = drm_template % (job_wrapper.galaxy_lib_dir, os.path.abspath( job_wrapper.working_directory ), command_line) - if self.app.config.set_metadata_externally: - script += "cd %s\n" % os.path.abspath( os.getcwd() ) - script += "%s\n" % job_wrapper.setup_external_metadata( exec_dir = os.path.abspath( os.getcwd() ), - tmp_dir = self.app.config.new_file_path, - dataset_files_path = self.app.model.Dataset.file_path, - output_fnames = job_wrapper.get_output_fnames(), - set_extension = False, - kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior fh = file( jt.remoteCommand, "w" ) fh.write( script ) fh.close() --- a/lib/galaxy/jobs/runners/pbs.py +++ b/lib/galaxy/jobs/runners/pbs.py @@ -5,6 +5,7 @@ from Queue import Queue, Empty from galaxy import model from galaxy.datatypes.data import nice_size from galaxy.util.bunch import Bunch +from galaxy.jobs.runners import BaseJobRunner from paste.deploy.converters import asbool @@ -80,7 +81,7 @@ class PBSJobState( object ): self.check_count = 0 self.stop_job = False -class PBSJobRunner( object ): +class PBSJobRunner( BaseJobRunner ): """ Job runner backed by a finite pool of worker threads. FIFO scheduling """ @@ -183,7 +184,7 @@ class PBSJobRunner( object ): try: job_wrapper.prepare() - command_line = job_wrapper.get_command_line() + command_line = self.build_command_line( job_wrapper, include_metadata=not( self.app.config.pbs_stage_path ) ) except: job_wrapper.fail( "failure preparing job", exception=True ) log.exception("failure running job %d" % job_wrapper.job_id) @@ -253,14 +254,6 @@ class PBSJobRunner( object ): script = pbs_symlink_template % (job_wrapper.galaxy_lib_dir, " ".join(job_wrapper.get_input_fnames() + output_files), self.app.config.pbs_stage_path, exec_dir, command_line) else: script = pbs_template % ( job_wrapper.galaxy_lib_dir, exec_dir, command_line ) - if self.app.config.set_metadata_externally: - script += "cd %s\n" % os.path.abspath( os.getcwd() ) - script += "%s\n" % job_wrapper.setup_external_metadata( exec_dir = os.path.abspath( os.getcwd() ), - tmp_dir = self.app.config.new_file_path, - dataset_files_path = self.app.model.Dataset.file_path, - output_fnames = output_fnames, - set_extension = False, - kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior job_file = "%s/%s.sh" % (self.app.config.cluster_files_directory, job_wrapper.job_id) fh = file(job_file, "w") fh.write(script)

1 0

galaxy-dist commit 7e9ae4605105: Add a new "Files uploaded via FTP" grid to the upload form and related parameter types, form fields, etc.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Nate Coraor <nate(a)bx.psu.edu> # Date 1288969490 14400 # Node ID 7e9ae46051057d822aff82a9b1f3aec0200086f2 # Parent cdd6ce1f38b0e22802ed63e3e0cb4f6a58cb93b2 Add a new "Files uploaded via FTP" grid to the upload form and related parameter types, form fields, etc. --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -1,6 +1,6 @@ <?xml version="1.0"?> -<tool name="Upload File" id="upload1" version="1.1.1"> +<tool name="Upload File" id="upload1" version="1.1.2"><description> from your computer </description> @@ -29,6 +29,7 @@ <validator type="expression" message="You will need to reselect the file you specified (%s)." substitute_value_in_message="True">not ( ( isinstance( value, unicode ) or isinstance( value, str ) ) and value != "" )</validator></param><param name="url_paste" type="text" area="true" size="5x35" label="URL/Text" help="Here you may specify a list of URLs (one per line) or paste the contents of a file."/> + <param name="ftp_files" type="ftpfile" label="Files uploaded via FTP"/><param name="space_to_tab" type="select" display="checkboxes" multiple="True" label="Convert spaces to tabs" help="Use this option if you are entering intervals by hand."><option value="Yes">Yes</option></param> --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -2,9 +2,10 @@ Classes for generating HTML forms """ -import logging,sys +import logging, sys, os, time from cgi import escape -from galaxy.util import restore_text +from galaxy.util import restore_text, relpath, nice_size +from galaxy.web import url_for log = logging.getLogger(__name__) @@ -145,6 +146,68 @@ class FileField(BaseField): ajax_text = ' galaxy-ajax-upload="true"' return '<input type="file" name="%s%s"%s%s>' % ( prefix, self.name, ajax_text, value_text ) +class FTPFileField(BaseField): + """ + An FTP file upload input. + """ + thead = ''' + <table id="grid-table" class="grid"> + <thead id="grid-table-header"> + <tr> + <th id="select-header"></th> + <th id="name-header"> + File + </th> + <th id="size-header"> + Size + </th> + <th id="date-header"> + Date + </th> + </tr> + </thead> + <tbody id="grid-table-body"> + ''' + trow = ''' + <tr> + <td><input type="checkbox" name="%s%s" value="%s"/></td> + <td>%s</td> + <td>%s</td> + <td>%s</td> + </tr> + ''' + tfoot = ''' + </tbody> + </table> + ''' + def __init__( self, name, dir, ftp_site, value = None ): + self.name = name + self.dir = dir + self.ftp_site = ftp_site + self.value = value + def get_html( self, prefix="" ): + rval = FTPFileField.thead + if self.dir is None: + rval += '<tr><td colspan="3"><em>Please <a href="%s">create</a> or <a href="%s">log in to</a> a Galaxy account to view files uploaded via FTP.</em></td></tr>' % ( url_for( controller='user', action='create', referer=url_for( controller='root' ) ), url_for( controller='user', action='login', referer=url_for( controller='root' ) ) ) + elif not os.path.exists( self.dir ): + rval += '<tr><td colspan="3"><em>Your FTP upload directory contains no files.</em></td></tr>' + else: + uploads = [] + for ( dirpath, dirnames, filenames ) in os.walk( self.dir ): + for filename in filenames: + path = relpath( os.path.join( dirpath, filename ), self.dir ) + statinfo = os.lstat( os.path.join( dirpath, filename ) ) + uploads.append( dict( path=path, + size=nice_size( statinfo.st_size ), + ctime=time.strftime( "%m/%d/%Y %I:%M:%S %p", time.localtime( statinfo.st_ctime ) ) ) ) + if not uploads: + rval += '<tr><td colspan="3"><em>Your FTP upload directory contains no files.</em></td></tr>' + for upload in uploads: + rval += FTPFileField.trow % ( prefix, self.name, upload['path'], upload['path'], upload['size'], upload['ctime'] ) + rval += FTPFileField.tfoot + rval += '<div class="toolParamHelp">This Galaxy server allows you to upload files via FTP. To upload some files, log in to the FTP server at <strong>%s</strong> using your Galaxy credentials (email address and password).</div>' % self.ftp_site + return rval + class HiddenField(BaseField): """ A hidden field. --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -40,6 +40,11 @@ class ToolParameter( object ): for elem in param.findall("validator"): self.validators.append( validation.Validator.from_element( self, elem ) ) + @property + def visible( self ): + """Return true if the parameter should be rendered on the form""" + return True + def get_label( self ): """Return user friendly name for the parameter""" if self.label: return self.label @@ -362,6 +367,41 @@ class FileToolParameter( ToolParameter ) def get_initial_value( self, trans, context ): return None +class FTPFileToolParameter( ToolParameter ): + """ + Parameter that takes a file uploaded via FTP as a value. + """ + def __init__( self, tool, elem ): + """ + Example: C{<param name="bins" type="file" />} + """ + ToolParameter.__init__( self, tool, elem ) + @property + def visible( self ): + if self.tool.app.config.ftp_upload_dir is None or self.tool.app.config.ftp_upload_site is None: + return False + return True + def get_html_field( self, trans=None, value=None, other_values={} ): + if trans is None or trans.user is None: + user_ftp_dir = None + else: + user_ftp_dir = os.path.join( trans.app.config.ftp_upload_dir, trans.user.email ) + return form_builder.FTPFileField( self.name, user_ftp_dir, trans.app.config.ftp_upload_site, value = value ) + def from_html( self, value, trans=None, other_values={} ): + return util.listify( value ) + def to_string( self, value, app ): + if value in [ None, '' ]: + return None + elif isinstance( value, unicode ) or isinstance( value, str ) or isinstance( value, list ): + return value + def to_python( self, value, app ): + if value is None: + return None + elif isinstance( value, unicode ) or isinstance( value, str ) or isinstance( value, list ): + return value + def get_initial_value( self, trans, context ): + return None + class HiddenToolParameter( ToolParameter ): """ Parameter that takes one of two values. @@ -1427,6 +1467,7 @@ parameter_types = dict( text = Te hidden = HiddenToolParameter, baseurl = BaseURLToolParameter, file = FileToolParameter, + ftpfile = FTPFileToolParameter, data = DataToolParameter, drill_down = DrillDownSelectToolParameter ) --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -480,6 +480,32 @@ def umask_fix_perms( path, umask, unmask current_group, e ) ) +def nice_size(size): + """ + Returns a readably formatted string with the size + + >>> nice_size(100) + '100.0 bytes' + >>> nice_size(10000) + '9.8 Kb' + >>> nice_size(1000000) + '976.6 Kb' + >>> nice_size(100000000) + '95.4 Mb' + """ + words = [ 'bytes', 'Kb', 'Mb', 'Gb' ] + try: + size = float( size ) + except: + return '??? bytes' + for ind, word in enumerate(words): + step = 1024 ** (ind + 1) + if step > size: + size = size / float(1024 ** ind) + out = "%.1f %s" % (size, word) + return out + return '??? bytes' + galaxy_root_path = os.path.join(__path__[0], "..","..","..") # The dbnames list is used in edit attributes and the upload tool dbnames = read_dbnames( os.path.join( galaxy_root_path, "tool-data", "shared", "ucsc", "builds.txt" ) ) --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -324,6 +324,18 @@ use_interactive = True # Enable the (experimental! beta!) Web API. Documentation forthcoming. #enable_api = False +# Enable Galaxy's "Upload via FTP" interface. You'll need to install and +# configure an FTP server (we've used ProFTPd since it can use Galaxy's +# database for authentication) and set the following two options. + +# This should point to a directory containing subdirectories matching users' +# email addresses, where Galaxy will look for files. +#ftp_upload_dir = None + +# This should be the hostname of your FTP server, which will be provided to +# users in the help text. +#ftp_upload_site = None + # -- Job Execution # If running multiple Galaxy processes, one can be designated as the job --- a/templates/tool_form.mako +++ b/templates/tool_form.mako @@ -51,7 +51,9 @@ function checkUncheckAll( name, check ) <%def name="do_inputs( inputs, tool_state, errors, prefix, other_values=None )"><% other_values = ExpressionContext( tool_state, other_values ) %> %for input_index, input in enumerate( inputs.itervalues() ): - %if input.type == "repeat": + %if not input.visible: + <% pass %> + %elif input.type == "repeat": <div class="repeat-group"><div class="form-title-row"><b>${input.title_plural}</b></div><% repeat_state = tool_state[input.name] %> --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -929,7 +929,11 @@ class Tool: assert isinstance( out_data, odict ) return 'tool_executed.mako', dict( out_data=out_data ) except: - return 'message.mako', dict( status='error', message='odict not returned from tool execution', refresh_frames=[] ) + if isinstance( out_data, str ): + message = out_data + else: + message = 'Failure executing tool (odict not returned from tool execution)' + return 'message.mako', dict( status='error', message=message, refresh_frames=[] ) # Otherwise move on to the next page else: state.page += 1 --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -12,11 +12,14 @@ import StringIO, os, urllib from galaxy.datatypes import sniff from galaxy.util.bunch import Bunch from galaxy.util.odict import odict -from galaxy.util import json +from galaxy.util import json, relpath class Group( object ): def __init__( self ): self.name = None + @property + def visible( self ): + return True def value_to_basic( self, value, app ): """ Convert value to a (possibly nested) representation using only basic @@ -267,6 +270,7 @@ class UploadDataset( Group ): rval = [] data_file = context['file_data'] url_paste = context['url_paste'] + ftp_files = context['ftp_files'] name = context.get( 'NAME', None ) info = context.get( 'INFO', None ) space_to_tab = False @@ -281,6 +285,31 @@ class UploadDataset( Group ): if file_bunch.path: file_bunch.space_to_tab = space_to_tab rval.append( file_bunch ) + # look for files uploaded via FTP + valid_files = [] + if ftp_files: + if trans.user is None: + log.warning( 'Anonymous user passed values in ftp_files: %s' % ftp_files ) + ftp_files = [] + # TODO: warning to the user (could happen if session has become invalid) + else: + user_ftp_dir = os.path.join( trans.app.config.ftp_upload_dir, trans.user.email ) + for ( dirpath, dirnames, filenames ) in os.walk( user_ftp_dir ): + for filename in filenames: + path = relpath( os.path.join( dirpath, filename ), user_ftp_dir ) + if not os.path.islink( os.path.join( dirpath, filename ) ): + valid_files.append( path ) + for ftp_file in ftp_files: + if ftp_file not in valid_files: + log.warning( 'User passed an invalid file path in ftp_files: %s' % ftp_file ) + continue + # TODO: warning to the user (could happen if file is already imported) + ftp_data_file = { 'local_filename' : os.path.abspath( os.path.join( user_ftp_dir, ftp_file ) ), + 'filename' : os.path.basename( ftp_file ) } + file_bunch = get_data_file_filename( ftp_data_file, override_name = name, override_info = info ) + if file_bunch.path: + file_bunch.space_to_tab = space_to_tab + rval.append( file_bunch ) return rval file_type = self.get_file_type( context ) d_type = self.get_datatype( trans, context ) --- a/static/june_2007_style/blue/base.css +++ b/static/june_2007_style/blue/base.css @@ -39,8 +39,7 @@ div.form-title-row{padding:5px 10px;} div.repeat-group-item{border-left:solid #d8b365 5px;margin-left:10px;margin-bottom:10px;} div.form-row-error{background:#FFCCCC;} div.form-row label{font-weight:bold;display:block;margin-bottom:.2em;} -div.form-row-input{float:left;width:300px;} -div.form-row-input > input{max-width:300px;} +div.form-row-input{float:left;} div.form-row-input label{font-weight:normal;display:inline;} div.form-row-error-message{width:300px;float:left;color:red;font-weight:bold;padding:3px 0 0 1em;} select,input,textarea{font:inherit;font-size:115%;} --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -419,36 +419,16 @@ class Newick( Text ): # ------------- Utility methods -------------- +# nice_size used to be here, but to resolve cyclical dependencies it's been +# moved to galaxy.util. It belongs there anyway since it's used outside +# datatypes. +nice_size = util.nice_size + def get_test_fname( fname ): """Returns test data filename""" path, name = os.path.split(__file__) full_path = os.path.join( path, 'test', fname ) return full_path -def nice_size(size): - """ - Returns a readably formatted string with the size - - >>> nice_size(100) - '100.0 bytes' - >>> nice_size(10000) - '9.8 Kb' - >>> nice_size(1000000) - '976.6 Kb' - >>> nice_size(100000000) - '95.4 Mb' - """ - words = [ 'bytes', 'Kb', 'Mb', 'Gb' ] - try: - size = float( size ) - except: - return '??? bytes' - for ind, word in enumerate(words): - step = 1024 ** (ind + 1) - if step > size: - size = size / float(1024 ** ind) - out = "%.1f %s" % (size, word) - return out - return '??? bytes' def get_file_peek( file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5 ): """ Returns the first LINE_COUNT lines wrapped to WIDTH --- a/lib/galaxy/tools/actions/upload.py +++ b/lib/galaxy/tools/actions/upload.py @@ -21,7 +21,7 @@ class UploadToolAction( ToolAction ): upload_common.cleanup_unused_precreated_datasets( precreated_datasets ) if not uploaded_datasets: - return 'No data was entered in the upload form, please go back and choose data to upload.' + return None, 'No data was entered in the upload form, please go back and choose data to upload.' json_file_path = upload_common.create_paramfile( trans, uploaded_datasets ) data_list = [ ud.data for ud in uploaded_datasets ] --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -99,6 +99,10 @@ class Configuration( object ): self.user_library_import_dir = kwargs.get( 'user_library_import_dir', None ) if self.user_library_import_dir is not None and not os.path.exists( self.user_library_import_dir ): raise ConfigurationError( "user_library_import_dir specified in config (%s) does not exist" % self.user_library_import_dir ) + self.ftp_upload_dir = kwargs.get( 'ftp_upload_dir', None ) + if self.ftp_upload_dir is not None and not os.path.exists( self.ftp_upload_dir ): + os.makedirs( self.ftp_upload_dir ) + self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None ) self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False ) self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' ) # Location for dependencies --- a/static/june_2007_style/base.css.tmpl +++ b/static/june_2007_style/base.css.tmpl @@ -212,11 +212,6 @@ div.form-row label { div.form-row-input { float: left; - width: 300px; -} - -div.form-row-input > input { - max-width: 300px; } div.form-row-input label {

1 0

galaxy-dist commit 9545e8675a5d: Fix uploads when ftp_file is not defined.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Nate Coraor <nate(a)bx.psu.edu> # Date 1288978480 14400 # Node ID 9545e8675a5d43408d75c66fcf3115cbdaf4af8a # Parent 7e9ae46051057d822aff82a9b1f3aec0200086f2 Fix uploads when ftp_file is not defined. --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -287,7 +287,7 @@ class UploadDataset( Group ): rval.append( file_bunch ) # look for files uploaded via FTP valid_files = [] - if ftp_files: + if ftp_files is not None: if trans.user is None: log.warning( 'Anonymous user passed values in ftp_files: %s' % ftp_files ) ftp_files = [] @@ -299,6 +299,8 @@ class UploadDataset( Group ): path = relpath( os.path.join( dirpath, filename ), user_ftp_dir ) if not os.path.islink( os.path.join( dirpath, filename ) ): valid_files.append( path ) + else: + ftp_files = [] for ftp_file in ftp_files: if ftp_file not in valid_files: log.warning( 'User passed an invalid file path in ftp_files: %s' % ftp_file )

1 0

galaxy-dist commit eb79ab327351: Remove missing hapmapmart tool from sample tool conf
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Nate Coraor <nate(a)bx.psu.edu> # Date 1288983276 14400 # Node ID eb79ab327351e4789a89748269fc34428f791463 # Parent 11abfc8eed60c0bdd231d06992f72001a2c785fa Remove missing hapmapmart tool from sample tool conf --- a/tool_conf.xml.sample +++ b/tool_conf.xml.sample @@ -319,7 +319,6 @@ <label text="Data: Import and upload" id="rgimport" /><tool file="data_source/upload.xml"/><tool file="data_source/access_libraries.xml" /> - <tool file="data_source/hapmapmart.xml" /><label text="Data: Filter and Clean" id="rgfilter" /><tool file="rgenetics/rgClean.xml"/><tool file="rgenetics/rgPedSub.xml"/>

1 0

galaxy-dist commit 11abfc8eed60: Update cufflinks and cuffdiff wrappers to indicate that, as of version 0.9, setting parameters for paired-end reads is not recommended.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User jeremy goecks <jeremy.goecks(a)emory.edu> # Date 1288979059 14400 # Node ID 11abfc8eed60c0bdd231d06992f72001a2c785fa # Parent 9545e8675a5d43408d75c66fcf3115cbdaf4af8a Update cufflinks and cuffdiff wrappers to indicate that, as of version 0.9, setting parameters for paired-end reads is not recommended. --- a/tools/ngs_rna/cuffdiff_wrapper.xml +++ b/tools/ngs_rna/cuffdiff_wrapper.xml @@ -25,6 +25,11 @@ --inputA=$gtf_input --input1=$aligned_reads1 --input2=$aligned_reads2 + + #if $singlePaired.sPaired == "Yes": + -m $singlePaired.mean_inner_distance + -s $singlePaired.inner_distance_std_dev + #end if </command><inputs><param format="gtf" name="gtf_input" type="data" label="Transcripts" help="A transcript GTF file produced by cufflinks, cuffcompare, or other source."/> @@ -34,12 +39,12 @@ <param name="min_mapqual" type="integer" value="0" label="Min SAM Mapping Quality" help="Instructs Cufflinks to ignore alignments with a SAM mapping quality lower than this number."/><param name="min_alignment_count" type="integer" value="0" label="Min Alignment Count" help="The minimum number of alignments in a locus for needed to conduct significance testing on changes in that locus observed between samples."/><conditional name="singlePaired"> - <param name="sPaired" type="select" label="Is this library mate-paired?"> - <option value="single">Single-end</option> - <option value="paired">Paired-end</option> + <param name="sPaired" type="select" label="Set Parameters for Paired-end Reads? (not recommended)"> + <option value="No">No</option> + <option value="Yes">Yes</option></param> - <when value="single"></when> - <when value="paired"> + <when value="No"></when> + <when value="Yes"><param name="mean_inner_distance" type="integer" value="20" label="Mean Inner Distance between Mate Pairs"/><param name="inner_distance_std_dev" type="integer" value="20" label="Standard Deviation for Inner Distance between Mate Pairs"/></when> --- a/tools/ngs_rna/cufflinks_wrapper.xml +++ b/tools/ngs_rna/cufflinks_wrapper.xml @@ -17,7 +17,7 @@ #if $reference_annotation.use_ref == "Yes": -G $reference_annotation.reference_annotation_file #end if - #if $singlePaired.sPaired == "paired": + #if $singlePaired.sPaired == "Yes": -m $singlePaired.mean_inner_distance -s $singlePaired.inner_distance_std_dev #end if @@ -39,12 +39,12 @@ </when></conditional><conditional name="singlePaired"> - <param name="sPaired" type="select" label="Is this library mate-paired?"> - <option value="single">Single-end</option> - <option value="paired">Paired-end</option> + <param name="sPaired" type="select" label="Set Parameters for Paired-end Reads? (not recommended)"> + <option value="No">No</option> + <option value="Yes">Yes</option></param> - <when value="single"></when> - <when value="paired"> + <when value="No"></when> + <when value="Yes"><param name="mean_inner_distance" type="integer" value="20" label="Mean Inner Distance between Mate Pairs"/><param name="inner_distance_std_dev" type="integer" value="20" label="Standard Deviation for Inner Distance between Mate Pairs"/></when>

1 0

galaxy-dist commit cf8bef68e8be: Correction to docstring for Tool.build_param_dict() -- from peter@maubp.freeserve.co.uk
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User jeremy goecks <jeremy.goecks(a)emory.edu> # Date 1288810958 14400 # Node ID cf8bef68e8befb8b6631578b71ff69f701fc0d4d # Parent b1ec8342053f3cdcd0a081ff28f077a76bd188cc Correction to docstring for Tool.build_param_dict() -- from peter(a)maubp.freeserve.co.uk --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1333,7 +1333,7 @@ class Tool: line. Each value is wrapped in a `InputValueWrapper`, which allows all the attributes of the value to be used in the template, *but* when the __str__ method is called it actually calls the - `to_param_dict_value` method of the associated input. + `to_param_dict_string` method of the associated input. """ param_dict = dict() # All parameters go into the param_dict

1 0

galaxy-dist commit f372ea5a6014: Fix interpreter for the FASTX Barcode Splitter.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Nate Coraor <nate(a)bx.psu.edu> # Date 1288902351 14400 # Node ID f372ea5a601443a9b9cd1d3f7e04ddc24db5b57a # Parent cdf8d61425cce011f98b1f24bd9fe19dc59b10ab Fix interpreter for the FASTX Barcode Splitter. --- a/tools/fastx_toolkit/fastx_barcode_splitter.xml +++ b/tools/fastx_toolkit/fastx_barcode_splitter.xml @@ -1,7 +1,7 @@ <tool id="cshl_fastx_barcode_splitter" name="Barcode Splitter"><description></description><requirements><requirement type="package">fastx_toolkit</requirement></requirements> - <command interpreter="sh">fastx_barcode_splitter_galaxy_wrapper.sh $BARCODE $input "$input.name" "$output.files_path" --mismatches $mismatches --partial $partial $EOL > $output </command> + <command interpreter="bash">fastx_barcode_splitter_galaxy_wrapper.sh $BARCODE $input "$input.name" "$output.files_path" --mismatches $mismatches --partial $partial $EOL > $output </command><inputs><param format="txt" name="BARCODE" type="data" label="Barcodes to use" />

1 0