galaxy-commits May 2013

galaxy-commits@lists.galaxyproject.org

1 participants
218 discussions

commit/galaxy-central: inithello: Limit when tests are explicitly removed from the tests dict.
by commits-noreply＠bitbucket.org 06 May '13

06 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f10bc351f2ca/ Changeset: f10bc351f2ca User: inithello Date: 2013-05-06 20:53:46 Summary: Limit when tests are explicitly removed from the tests dict. Affected #: 1 file diff -r 01cfd0c5af0fa5555697d14283404af2f5ef496c -r f10bc351f2ca97577a3a823dc4113f965164cbd5 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -704,8 +704,6 @@ os.environ[ 'GALAXY_TOOL_SHED_TEST_FILE' ] = galaxy_shed_tools_dict os.environ[ 'GALAXY_TEST_HOST' ] = galaxy_test_host os.environ[ 'GALAXY_TEST_PORT' ] = galaxy_test_port - # Explicitly clear tests from twill's test environment. - remove_generated_tests( app ) # Set the module-level variable 'toolbox', so that test.functional.test_toolbox will generate the appropriate test methods. test_toolbox.toolbox = app.toolbox # Generate the test methods for this installed repository. We need to pass in True here, or it will look Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Tophat2 wrapper update: (a) remove unneeded wrapper script; and (b) use stdio tags to capture and show output; and (c) update functional tests for most recent version of Tophat.
by commits-noreply＠bitbucket.org 06 May '13

06 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/01cfd0c5af0f/ Changeset: 01cfd0c5af0f User: jgoecks Date: 2013-05-06 20:40:44 Summary: Tophat2 wrapper update: (a) remove unneeded wrapper script; and (b) use stdio tags to capture and show output; and (c) update functional tests for most recent version of Tophat. Affected #: 3 files diff -r 10a0ad2f0280cb044bde1e6424b7a7a93365d5ce -r 01cfd0c5af0fa5555697d14283404af2f5ef496c test-data/tophat2_out2j.bed --- a/test-data/tophat2_out2j.bed +++ b/test-data/tophat2_out2j.bed @@ -1,3 +1,3 @@ track name=junctions description="TopHat junctions" -test_chromosome 179 400 JUNC00000001 45 + 179 400 255,0,0 2 71,50 0,171 -test_chromosome 350 550 JUNC00000002 38 + 350 550 255,0,0 2 50,50 0,150 +test_chromosome 179 400 JUNC00000001 37 + 179 400 255,0,0 2 71,50 0,171 +test_chromosome 350 550 JUNC00000002 37 + 350 550 255,0,0 2 50,50 0,150 diff -r 10a0ad2f0280cb044bde1e6424b7a7a93365d5ce -r 01cfd0c5af0fa5555697d14283404af2f5ef496c tools/ngs_rna/tophat2_wrapper.py --- a/tools/ngs_rna/tophat2_wrapper.py +++ /dev/null @@ -1,270 +0,0 @@ -#!/usr/bin/env python - -import optparse, os, shutil, subprocess, sys, tempfile, fileinput - -def stop_err( msg ): - sys.stderr.write( "%s\n" % msg ) - sys.exit() - -def __main__(): - #Parse Command Line - parser = optparse.OptionParser() - parser.add_option( '-p', '--num-threads', dest='num_threads', help='Use this many threads to align reads. The default is 1.' ) - parser.add_option( '-C', '--color-space', dest='color_space', action='store_true', help='This indicates color-space data' ) - parser.add_option( '-J', '--junctions-output', dest='junctions_output_file', help='Junctions output file; formate is BED.' ) - parser.add_option( '-H', '--hits-output', dest='accepted_hits_output_file', help='Accepted hits output file; formate is BAM.' ) - parser.add_option( '', '--own-file', dest='own_file', help='' ) - parser.add_option( '-D', '--indexes-path', dest='index_path', help='Indexes directory; location of .ebwt and .fa files.' ) - parser.add_option( '-r', '--mate-inner-dist', dest='mate_inner_dist', help='This is the expected (mean) inner distance between mate pairs. \ - For, example, for paired end runs with fragments selected at 300bp, \ - where each end is 50bp, you should set -r to be 200. There is no default, \ - and this parameter is required for paired end runs.') - parser.add_option( '', '--mate-std-dev', dest='mate_std_dev', help='Standard deviation of distribution on inner distances between male pairs.' ) - parser.add_option( '', '--read-mismatches', dest='read_mismatches' ) - parser.add_option( '', '--bowtie-n', action="store_true", dest='bowtie_n' ) - parser.add_option( '', '--no-discordant', action="store_true", dest='report_concordant_pairs_only' ) - parser.add_option( '-a', '--min-anchor-length', dest='min_anchor_length', - help='The "anchor length". TopHat will report junctions spanned by reads with at least this many bases on each side of the junction.' ) - parser.add_option( '-m', '--splice-mismatches', dest='splice_mismatches', help='The maximum number of mismatches that can appear in the anchor region of a spliced alignment.' ) - parser.add_option( '-i', '--min-intron-length', dest='min_intron_length', - help='The minimum intron length. TopHat will ignore donor/acceptor pairs closer than this many bases apart.' ) - parser.add_option( '-I', '--max-intron-length', dest='max_intron_length', - help='The maximum intron length. When searching for junctions ab initio, TopHat will ignore donor/acceptor pairs farther than this many bases apart, except when such a pair is supported by a split segment alignment of a long read.' ) - parser.add_option( '-g', '--max_multihits', dest='max_multihits', help='Maximum number of alignments to be allowed' ) - parser.add_option( '', '--seg-mismatches', dest='seg_mismatches', help='Number of mismatches allowed in each segment alignment for reads mapped independently' ) - parser.add_option( '', '--seg-length', dest='seg_length', help='Minimum length of read segments' ) - parser.add_option( '', '--library-type', dest='library_type', help='TopHat will treat the reads as strand specific. Every read alignment will have an XS attribute tag. Consider supplying library type options below to select the correct RNA-seq protocol.' ) - parser.add_option( '', '--allow-indels', action="store_true", help='Allow indel search. Indel search is disabled by default.(Not used since version 1.3.0)' ) - parser.add_option( '', '--max-insertion-length', dest='max_insertion_length', help='The maximum insertion length. The default is 3.' ) - parser.add_option( '', '--max-deletion-length', dest='max_deletion_length', help='The maximum deletion length. The default is 3.' ) - - # Options for supplying own junctions - parser.add_option( '-G', '--GTF', dest='gene_model_annotations', help='Supply TopHat with a list of gene model annotations. \ - TopHat will use the exon records in this file to build \ - a set of known splice junctions for each gene, and will \ - attempt to align reads to these junctions even if they \ - would not normally be covered by the initial mapping.') - parser.add_option( '-j', '--raw-juncs', dest='raw_juncs', help='Supply TopHat with a list of raw junctions. Junctions are \ - specified one per line, in a tab-delimited format. Records \ - look like: <chrom><left><right><+/-> left and right are \ - zero-based coordinates, and specify the last character of the \ - left sequenced to be spliced to the first character of the right \ - sequence, inclusive.') - parser.add_option( '', '--no-novel-juncs', action="store_true", dest='no_novel_juncs', help="Only look for junctions indicated in the \ - supplied GFF file. (ignored without -G)") - parser.add_option( '', '--no-novel-indels', action="store_true", dest='no_novel_indels', help="Skip indel search. Indel search is enabled by default.") - # Types of search. - parser.add_option( '', '--microexon-search', action="store_true", dest='microexon_search', help='With this option, the pipeline will attempt to find alignments incident to microexons. Works only for reads 50bp or longer.') - parser.add_option( '', '--coverage-search', action="store_true", dest='coverage_search', help='Enables the coverage based search for junctions. Use when coverage search is disabled by default (such as for reads 75bp or longer), for maximum sensitivity.') - parser.add_option( '', '--no-coverage-search', action="store_false", dest='coverage_search' ) - parser.add_option( '', '--min-segment-intron', dest='min_segment_intron', help='Minimum intron length that may be found during split-segment search' ) - parser.add_option( '', '--max-segment-intron', dest='max_segment_intron', help='Maximum intron length that may be found during split-segment search' ) - parser.add_option( '', '--min-coverage-intron', dest='min_coverage_intron', help='Minimum intron length that may be found during coverage search' ) - parser.add_option( '', '--max-coverage-intron', dest='max_coverage_intron', help='Maximum intron length that may be found during coverage search' ) - - # Bowtie2 options. - parser.add_option( '', '--b2-very-fast', action='store_true', dest='b2_very_fast') - parser.add_option( '', '--b2-fast', action='store_true', dest='b2_fast') - parser.add_option( '', '--b2-very-sensitive', action='store_true', dest='b2_very_sensitive') - parser.add_option( '', '--b2-sensitive', action='store_true', dest='b2_sensitive') - - # Fusion search options. - parser.add_option( '', '--fusion-search', action='store_true', dest='fusion_search' ) - parser.add_option( '', '--fusion-anchor-length', dest='fusion_anchor_length' ) - parser.add_option( '', '--fusion-min-dist', dest='fusion_min_dist' ) - parser.add_option( '', '--fusion-read-mismatches', dest='fusion_read_mismatches' ) - parser.add_option( '', '--fusion-multireads', dest='fusion_multireads' ) - parser.add_option( '', '--fusion-multipairs', dest='fusion_multipairs' ) - parser.add_option( '', '--fusion-ignore-chromosomes', dest='fusion_ignore_chromosomes' ) - - # Wrapper options. - parser.add_option( '-1', '--input1', dest='input1', help='The (forward or single-end) reads file in Sanger FASTQ format' ) - parser.add_option( '-2', '--input2', dest='input2', help='The reverse reads file in Sanger FASTQ format' ) - parser.add_option( '', '--single-paired', dest='single_paired', help='' ) - parser.add_option( '', '--settings', dest='settings', help='' ) - - # Read group options. - parser.add_option( '', '--rgid', dest='rgid', help='Read group identifier' ) - parser.add_option( '', '--rglb', dest='rglb', help='Library name' ) - parser.add_option( '', '--rgpl', dest='rgpl', help='Platform/technology used to produce the reads' ) - parser.add_option( '', '--rgsm', dest='rgsm', help='Sample' ) - - (options, args) = parser.parse_args() - - # Color or base space - space = '' - if options.color_space: - space = '-C' - - # Creat bowtie index if necessary. - tmp_index_dir = tempfile.mkdtemp() - if options.own_file: - index_path = os.path.join( tmp_index_dir, '.'.join( os.path.split( options.own_file )[1].split( '.' )[:-1] ) ) - try: - os.link( options.own_file, index_path + '.fa' ) - except: - # Tophat prefers (but doesn't require) fasta file to be in same directory, with .fa extension - pass - cmd_index = 'bowtie2-build %s -f %s %s' % ( space, options.own_file, index_path ) - try: - tmp = tempfile.NamedTemporaryFile( dir=tmp_index_dir ).name - tmp_stderr = open( tmp, 'wb' ) - proc = subprocess.Popen( args=cmd_index, shell=True, cwd=tmp_index_dir, stderr=tmp_stderr.fileno() ) - returncode = proc.wait() - tmp_stderr.close() - # get stderr, allowing for case where it's very large - tmp_stderr = open( tmp, 'rb' ) - stderr = '' - buffsize = 1048576 - try: - while True: - stderr += tmp_stderr.read( buffsize ) - if not stderr or len( stderr ) % buffsize != 0: - break - except OverflowError: - pass - tmp_stderr.close() - if returncode != 0: - raise Exception, stderr - except Exception, e: - if os.path.exists( tmp_index_dir ): - shutil.rmtree( tmp_index_dir ) - stop_err( 'Error indexing reference sequence\n' + str( e ) ) - else: - index_path = options.index_path - - # Build tophat command. - cmd = 'tophat2 --keep-fasta-order %s %s %s' - reads = options.input1 - if options.input2: - reads += ' ' + options.input2 - opts = '-p %s %s' % ( options.num_threads, space ) - if options.single_paired == 'paired': - opts += ' -r %s' % options.mate_inner_dist - if options.report_concordant_pairs_only: - opts += ' --no-discordant' - # Read group options. - if options.rgid: - if not options.rglb or not options.rgpl or not options.rgsm: - stop_err( 'If you want to specify read groups, you must include the ID, LB, PL, and SM tags.' ) - opts += ' --rg-id %s' % options.rgid - opts += ' --rg-library %s' % options.rglb - opts += ' --rg-platform %s' % options.rgpl - opts += ' --rg-sample %s' % options.rgsm - - if options.settings == 'preSet': - cmd = cmd % ( opts, index_path, reads ) - else: - try: - if int( options.min_anchor_length ) >= 3: - opts += ' -a %s' % options.min_anchor_length - else: - raise Exception, 'Minimum anchor length must be 3 or greater' - opts += ' -m %s' % options.splice_mismatches - opts += ' -i %s' % options.min_intron_length - opts += ' -I %s' % options.max_intron_length - opts += ' -g %s' % options.max_multihits - # Custom junctions options. - if options.gene_model_annotations: - opts += ' -G %s' % options.gene_model_annotations - if options.raw_juncs: - opts += ' -j %s' % options.raw_juncs - if options.no_novel_juncs: - opts += ' --no-novel-juncs' - if options.library_type: - opts += ' --library-type %s' % options.library_type - if options.no_novel_indels: - opts += ' --no-novel-indels' - else: - if options.max_insertion_length: - opts += ' --max-insertion-length %i' % int( options.max_insertion_length ) - if options.max_deletion_length: - opts += ' --max-deletion-length %i' % int( options.max_deletion_length ) - # Max options do not work for Tophat v1.2.0, despite documentation to the contrary. (Fixed in version 1.3.1) - # need to warn user of this fact - #sys.stdout.write( "Max insertion length and max deletion length options don't work in Tophat v1.2.0\n" ) - - if options.read_mismatches: - opts += ' --read-mismatches %i' % int( options.read_mismatches ) - if options.bowtie_n: - opts += ' --bowtie-n' - - # Search type options. - if options.coverage_search: - opts += ' --coverage-search --min-coverage-intron %s --max-coverage-intron %s' % ( options.min_coverage_intron, options.max_coverage_intron ) - else: - opts += ' --no-coverage-search' - if options.microexon_search: - opts += ' --microexon-search' - if options.single_paired == 'paired' and options.mate_std_dev: - opts += ' --mate-std-dev %s' % options.mate_std_dev - if options.seg_mismatches: - opts += ' --segment-mismatches %d' % int( options.seg_mismatches ) - if options.seg_length: - opts += ' --segment-length %d' % int( options.seg_length ) - if options.min_segment_intron: - opts += ' --min-segment-intron %d' % int( options.min_segment_intron ) - if options.max_segment_intron: - opts += ' --max-segment-intron %d' % int( options.max_segment_intron ) - - # Fusion search options. - if options.fusion_search: - opts += ' --fusion-search --fusion-anchor-length %i --fusion-min-dist %i --fusion-read-mismatches %i --fusion-multireads %i --fusion-multipairs %i' % \ - ( int( options.fusion_anchor_length ), int( options.fusion_min_dist ), - int( options.fusion_read_mismatches ), int( options.fusion_multireads ), - int( options.fusion_multipairs ) ) - if options.fusion_ignore_chromosomes: - opts += ' --fusion-ignore-chromosomes %s' % options.fusion_ignore_chromosomes - - # Bowtie2 options. - if options.b2_very_fast: - opts += ' --b2-very-fast' - if options.b2_fast: - opts += ' --b2-fast' - if options.b2_sensitive: - opts += ' --b2-sensitive' - if options.b2_very_sensitive: - opts += ' --b2-very-sensitive' - - cmd = cmd % ( opts, index_path, reads ) - - except Exception, e: - # Clean up temp dirs - if os.path.exists( tmp_index_dir ): - shutil.rmtree( tmp_index_dir ) - stop_err( 'Something is wrong with the alignment parameters and the alignment could not be run\n' + str( e ) ) - print cmd - - # Run - try: - tmp_out = tempfile.NamedTemporaryFile().name - tmp_stdout = open( tmp_out, 'wb' ) - tmp_err = tempfile.NamedTemporaryFile().name - tmp_stderr = open( tmp_err, 'wb' ) - proc = subprocess.Popen( args=cmd, shell=True, cwd=".", stdout=tmp_stdout, stderr=tmp_stderr ) - returncode = proc.wait() - tmp_stderr.close() - # get stderr, allowing for case where it's very large - tmp_stderr = open( tmp_err, 'rb' ) - stderr = '' - buffsize = 1048576 - try: - while True: - stderr += tmp_stderr.read( buffsize ) - if not stderr or len( stderr ) % buffsize != 0: - break - except OverflowError: - pass - tmp_stdout.close() - tmp_stderr.close() - if returncode != 0: - raise Exception, stderr - - except Exception, e: - stop_err( 'Error in tophat:\n' + str( e ) ) - - # Clean up temp dirs - if os.path.exists( tmp_index_dir ): - shutil.rmtree( tmp_index_dir ) - -if __name__=="__main__": __main__() diff -r 10a0ad2f0280cb044bde1e6424b7a7a93365d5ce -r 01cfd0c5af0fa5555697d14283404af2f5ef496c tools/ngs_rna/tophat2_wrapper.xml --- a/tools/ngs_rna/tophat2_wrapper.xml +++ b/tools/ngs_rna/tophat2_wrapper.xml @@ -7,49 +7,38 @@ <requirement type="package">bowtie2</requirement><requirement type="package">tophat2</requirement></requirements> - <command interpreter="python"> - tophat2_wrapper.py + + <command> + ## + ## Set path to index, building the reference if necessary. + ## + + #set index_path = '' + #if $refGenomeSource.genomeSource == "history": + bowtie2-build "$refGenomeSource.ownFile" genome ; ln -s "$refGenomeSource.ownFile" genome.fa ; + #set index_path = 'genome' + #else: + #set index_path = $refGenomeSource.index.fields.path + #end if + + ## + ## Run tophat. + ## + + tophat2 ## Change this to accommodate the number of threads you have available. - --num-threads="4" - - ## Provide outputs. - --junctions-output=$junctions - --hits-output=$accepted_hits - - ## Handle reference file. - #if $refGenomeSource.genomeSource == "history": - --own-file=$refGenomeSource.ownFile - #else: - --indexes-path="${refGenomeSource.index.fields.path}" - #end if - - ## Are reads single-end or paired? - --single-paired=$singlePaired.sPaired - - ## First input file always required. - --input1=$input1 - - ## Second input only if input is paired-end. - ## Also set parameters specific to paired data. - #if $singlePaired.sPaired == "paired" - --input2=$singlePaired.input2 - -r $singlePaired.mate_inner_distance - --mate-std-dev=$singlePaired.mate_std_dev - - #if str($singlePaired.report_discordant_pairs) == "No": - --no-discordant - #end if - #end if + --num-threads 4 ## Set params. - --settings=$params.settingsType #if $params.settingsType == "full": --read-mismatches $params.read_mismatches #if str($params.bowtie_n) == "Yes": --bowtie-n #end if + --read-edit-dist $params.read_edit_dist + --read-realign-edit-dist $params.read_realign_edit_dist -a $params.anchor_length -m $params.splice_mismatches -i $params.min_intron_length @@ -57,9 +46,9 @@ -g $params.max_multihits --min-segment-intron $params.min_segment_intron --max-segment-intron $params.max_segment_intron - --seg-mismatches=$params.seg_mismatches - --seg-length=$params.seg_length - --library-type=$params.library_type + --segment-mismatches $params.seg_mismatches + --segment-length $params.seg_length + --library-type $params.library_type ## Indel search. #if $params.indel_search.allow_indel_search == "Yes": @@ -78,7 +67,6 @@ #if $params.own_junctions.raw_juncs.use_juncs == "Yes": -j $params.own_junctions.raw_juncs.raw_juncs #end if - ## TODO: No idea why a string cast is necessary, but it is: #if str($params.own_junctions.no_novel_juncs) == "Yes": --no-novel-juncs #end if @@ -91,7 +79,7 @@ #else: --no-coverage-search #end if - ## TODO: No idea why the type conversion is necessary, but it seems to be. + #if str($params.microexon_search) == "Yes": --microexon-search #end if @@ -116,12 +104,27 @@ ## Read group information. #if $readGroup.specReadGroup == "yes" - --rgid="$readGroup.rgid" - --rglb="$readGroup.rglb" - --rgpl="$readGroup.rgpl" - --rgsm="$readGroup.rgsm" + --rg-id "$readGroup.rgid" + --rg-library "$readGroup.rglb" + --rg-platform "$readGroup.rgpl" + --rg-sample "$readGroup.rgsm" + #end if + + ## Set index path, inputs and parameters specific to paired data. + #if $singlePaired.sPaired == "paired" + -r $singlePaired.mate_inner_distance + --mate-std-dev=$singlePaired.mate_std_dev + + #if str($singlePaired.report_discordant_pairs) == "No": + --no-discordant + #end if + + ${index_path} $singlePaired.input1 $singlePaired.input2 + #else + ${index_path} $singlePaired.input1 #end if </command> + <inputs><conditional name="singlePaired"><param name="sPaired" type="select" label="Is this library mate-paired?"> @@ -157,6 +160,10 @@ <when value="preSet" /><when value="full"> + <param name="read_realign_edit_dist" type="integer" value="1000" label="Max realign edit distance" help="Some of the reads spanning multiple exons may be mapped incorrectly as a contiguous alignment to the genome even though the correct alignment should be a spliced one - this can happen in the presence of processed pseudogenes that are rarely (if at all) transcribed or expressed. This option can direct TopHat to re-align reads for which the edit distance of an alignment obtained in a previous mapping step is above or equal to this option value. If you set this option to 0, TopHat will map every read in all the mapping steps (transcriptome if you provided gene annotations, genome, and finally splice variants detected by TopHat), reporting the best possible alignment found in any of these mapping steps. This may greatly increase the mapping accuracy at the expense of an increase in running time. The default value for this option is set such that TopHat will not try to realign reads already mapped in earlier steps." /> + + <param name="read_edit_dist" type="integer" value="2" label="Max edit distance" help="Final read alignments having more than these many edit distance are discarded." /> + <param name="library_type" type="select" label="Library Type" help="TopHat will treat the reads as strand specific. Every read alignment will have an XS attribute tag. Consider supplying library type options below to select the correct RNA-seq protocol."><option value="fr-unstranded">FR Unstranded</option><option value="fr-firststrand">FR First Strand</option> @@ -260,6 +267,11 @@ </conditional></inputs> + <stdio> + <regex match="Exception" source="both" level="error" description="Tool exception"/> + <regex match=".*" source="both" level="log" description="tool progress"/> + </stdio> + <outputs><data format="tabular" name="fusions" label="${tool.name} on ${on_string}: fusions" from_work_dir="tophat_out/fusions.out"><filter>(params['settingsType'] == 'full' and params['fusion_search']['do_search'] == 'Yes')</filter> @@ -277,6 +289,7 @@ <expand macro="dbKeyActions" /></data></outputs> + <macros><import>tophat_macros.xml</import><macro name="dbKeyActions"> @@ -312,7 +325,7 @@ <param name="index" value="tophat_test" /><param name="settingsType" value="preSet" /><param name="specReadGroup" value="No" /> - <output name="junctions" file="tophat_out1j.bed" /> + <output name="junctions" file="tophat2_out1j.bed" /><output name="accepted_hits" file="tophat_out1h.bam" compare="sim_size" /></test> @@ -369,6 +382,7 @@ <param name="min_coverage_intron" value="50" /><param name="max_coverage_intron" value="20000" /><param name="microexon_search" value="Yes" /> + <param name="b2_settings" value="No" /><param name="do_search" value="Yes" /><param name="anchor_len" value="21" /> @@ -418,6 +432,7 @@ <param name="report_discordant_pairs" value="Yes" /><param name="use_search" value="No" /><param name="microexon_search" value="Yes" /> + <param name="b2_settings" value="No" /><param name="do_search" value="Yes" /><param name="anchor_len" value="21" /> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Remove set_metadata_externally from the universe sample.
by commits-noreply＠bitbucket.org 06 May '13

06 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/10a0ad2f0280/ Changeset: 10a0ad2f0280 User: dannon Date: 2013-05-06 19:07:31 Summary: Remove set_metadata_externally from the universe sample. Affected #: 1 file diff -r b72a06cf0247ac266fb7b2fc204cef7fb4691cb0 -r 10a0ad2f0280cb044bde1e6424b7a7a93365d5ce universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -625,12 +625,6 @@ # running more than one Galaxy server using the same database. #enable_job_recovery = True -# Setting metadata on job outputs to in a separate process (or if using a -# cluster, on the cluster). Thanks to Python's Global Interpreter Lock and the -# hefty expense that setting metadata incurs, your Galaxy process may become -# unresponsive when this operation occurs internally. -#set_metadata_externally = False - # Although it is fairly reliable, setting metadata can occasionally fail. In # these instances, you can choose to retry setting it internally or leave it in # a failed state (since retrying internally may cause the Galaxy process to be Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 5 new changesets
by commits-noreply＠bitbucket.org 06 May '13

06 May '13

5 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/bb5771403508/ Changeset: bb5771403508 User: dannon Date: 2013-05-05 17:54:28 Summary: Workflow controller: remove unused variables. Affected #: 1 file diff -r 02822f28dc93348264a18f9f30abfedb53b80728 -r bb5771403508fbd75829960f089e6f35267716cf lib/galaxy/webapps/galaxy/controllers/workflow.py --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -134,7 +134,7 @@ @web.require_login( "use Galaxy workflows" ) def list_grid( self, trans, **kwargs ): """ List user's stored workflows. """ - status = message = None + # status = message = None if 'operation' in kwargs: operation = kwargs['operation'].lower() if operation == "rename": @@ -462,7 +462,7 @@ return trans.show_error_message( "The specified workflow does not exist." ) # Rate workflow. - stored_rating = self.rate_item( trans.sa_session, trans.get_user(), stored, rating ) + self.rate_item( trans.sa_session, trans.get_user(), stored, rating ) return self.get_ave_item_rating_data( trans.sa_session, stored ) @@ -507,8 +507,6 @@ @web.require_login( "use Galaxy workflows" ) def gen_image( self, trans, id ): stored = self.get_stored_workflow( trans, id, check_ownership=True ) - session = trans.sa_session - trans.response.set_content_type("image/svg+xml") return self._workflow_to_svg_canvas( trans, stored ).standalone_xml() @@ -942,7 +940,6 @@ """ # Load encoded workflow from database - user = trans.get_user() id = trans.security.decode_id( id ) trans.workflow_building_mode = True stored = trans.sa_session.query( model.StoredWorkflow ).get( id ) @@ -1034,6 +1031,7 @@ trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy-Workflow-%s.ga"' % ( sname ) trans.response.set_content_type( 'application/galaxy-archive' ) return stored_dict + @web.expose def import_workflow( self, trans, cntrller='workflow', **kwd ): """ @@ -1097,7 +1095,7 @@ else: uploaded_file = file_data.file uploaded_file_name = uploaded_file.name - uploaded_file_filename = file_data.filename + # uploaded_file_filename = file_data.filename if os.path.getsize( os.path.abspath( uploaded_file_name ) ) > 0: # We're reading the file as text so we can re-use the existing code below. # This may not be ideal... @@ -1198,6 +1196,7 @@ status=status, use_panels=True, myexperiment_target_url=myexperiment_target_url ) + @web.json def get_datatypes( self, trans ): ext_to_class_name = dict() @@ -1335,7 +1334,8 @@ # Build the state for each step errors = {} has_upgrade_messages = False - has_errors = False + # has_errors is never used + # has_errors = False saved_history = None if history_id is not None: saved_history = trans.get_history(); @@ -1498,7 +1498,8 @@ step.state = step.module.state # Error dict if step.tool_errors: - has_errors = True + # has_errors is never used. + # has_errors = True errors[step.id] = step.tool_errors else: ## Non-tool specific stuff? @@ -1553,7 +1554,8 @@ # Build the state for each step errors = {} has_upgrade_messages = False - has_errors = False + # has_errors is never used + # has_errors = False if kwargs: # If kwargs were provided, the states for each step should have # been POSTed @@ -1591,7 +1593,8 @@ step.state = step.module.state # Error dict if step.tool_errors: - has_errors = True + # has_errors is never used + # has_errors = True errors[step.id] = step.tool_errors else: ## Non-tool specific stuff? @@ -1825,7 +1828,7 @@ # Unpack and add post-job actions. post_job_actions = step_dict.get( 'post_job_actions', {} ) for name, pja_dict in post_job_actions.items(): - pja = model.PostJobAction( pja_dict[ 'action_type' ], + model.PostJobAction( pja_dict[ 'action_type' ], step, pja_dict[ 'output_name' ], pja_dict[ 'action_arguments' ] ) # Second pass to deal with connections between steps @@ -2064,7 +2067,6 @@ but track the associations. """ associations = [] - names_to_clean = [] # dbkey is pushed in by the framework if 'dbkey' in values: del values['dbkey'] https://bitbucket.org/galaxy/galaxy-central/commits/e3b8df9f9ecd/ Changeset: e3b8df9f9ecd User: dannon Date: 2013-05-05 18:12:19 Summary: Workflow controller - trim trailing whitespace, add spacing between classes/methods. Affected #: 1 file diff -r bb5771403508fbd75829960f089e6f35267716cf -r e3b8df9f9ecd003a9b2c0441987ce1b79b6179a4 lib/galaxy/webapps/galaxy/controllers/workflow.py --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -37,8 +37,8 @@ from galaxy.workflow.modules import module_factory, ToolModule +class StoredWorkflowListGrid( grids.Grid ): -class StoredWorkflowListGrid( grids.Grid ): class StepsColumn( grids.GridColumn ): def get_value(self, trans, grid, workflow): return len( workflow.latest_workflow.steps ) @@ -73,6 +73,7 @@ def apply_query_filter( self, trans, query, **kwargs ): return query.filter_by( user=trans.user, deleted=False ) + class StoredWorkflowAllPublishedGrid( grids.Grid ): title = "Published Workflows" model_class = model.StoredWorkflow @@ -94,13 +95,16 @@ key="free-text-search", visible=False, filterable="standard" ) ) operations = [] + def build_initial_query( self, trans, **kwargs ): # Join so that searching stored_workflow.user makes sense. return trans.sa_session.query( self.model_class ).join( model.User.table ) + def apply_query_filter( self, trans, query, **kwargs ): # A public workflow is published, has a slug, and is not deleted. return query.filter( self.model_class.published==True ).filter( self.model_class.slug != None ).filter( self.model_class.deleted == False ) + # Simple SGML parser to get all content in a single tag. class SingleTagContentsParser( sgmllib.SGMLParser ): @@ -119,6 +123,7 @@ if self.cur_tag == self.target_tag: self.tag_content += text + class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMixin, UsesAnnotations, UsesItemRatings ): stored_list_grid = StoredWorkflowListGrid() published_list_grid = StoredWorkflowAllPublishedGrid() @@ -847,7 +852,7 @@ for input_name, conns in step.temp_input_connections.iteritems(): if conns: conn_dicts = conns if isinstance(conns,list) else [conns] - for conn_dict in conn_dicts: + for conn_dict in conn_dicts: conn = model.WorkflowStepConnection() conn.input_step = step conn.input_name = input_name @@ -883,6 +888,7 @@ """ stored = self.get_stored_workflow( trans, id, check_ownership=False, check_accessible=True ) return trans.fill_template( "/workflow/export.mako", item=stored, use_panels=True ) + @web.expose @web.require_login( "use workflows" ) def import_from_myexp( self, trans, myexp_id, **kwd ): @@ -932,6 +938,7 @@ return trans.show_warn_message( "Imported, but this workflow contains cycles. %s" % workflow_list_str ) else: return trans.show_message( "Workflow '%s' imported. %s" % (workflow.name, workflow_list_str) ) + @web.expose @web.require_login( "use workflows" ) def export_to_myexp( self, trans, id, myexp_username, myexp_password ): @@ -1767,7 +1774,7 @@ # tools. This should be removed at some point. Mirrored # hack in _workflow_from_dict should never be removed so # existing workflow exports continue to function. - for input_name, input_conn in dict(input_conn_dict).iteritems(): + for input_name, input_conn in dict(input_conn_dict).iteritems(): if len(input_conn) == 1: input_conn_dict[input_name] = input_conn[0] step_dict['input_connections'] = input_conn_dict @@ -1776,6 +1783,7 @@ # Add to return value data['steps'][step.order_index] = step_dict return data + def _workflow_from_dict( self, trans, data, source=None, add_to_menu=False ): """ Creates a workflow from a dict. Created workflow is stored in the database and returned. @@ -1869,16 +1877,13 @@ return stored, missing_tool_tups def _workflow_to_svg_canvas( self, trans, stored ): - workflow = stored.latest_workflow data = [] - canvas = svgfig.canvas(style="stroke:black; fill:none; stroke-width:1px; stroke-linejoin:round; text-anchor:left") text = svgfig.SVG("g") connectors = svgfig.SVG("g") boxes = svgfig.SVG("g") svgfig.Text.defaults["font-size"] = "10px" - in_pos = {} out_pos = {} margin = 5 @@ -2021,6 +2026,7 @@ except CycleError: return None + class FakeJob( object ): """ Fake job object for datasets that have no creating_job_associations, https://bitbucket.org/galaxy/galaxy-central/commits/16fbd8c387a0/ Changeset: 16fbd8c387a0 User: dannon Date: 2013-05-05 19:57:21 Summary: Add docstring comment for workflow controller 'imp' Affected #: 1 file diff -r e3b8df9f9ecd003a9b2c0441987ce1b79b6179a4 -r 16fbd8c387a0cb7eb7607b6d37c6ea26fa9c6d50 lib/galaxy/webapps/galaxy/controllers/workflow.py --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -356,6 +356,7 @@ @web.expose @web.require_login( "to import a workflow", use_panels=True ) def imp( self, trans, id, **kwargs ): + """Imports a workflow shared by other users.""" # Set referer message. referer = trans.request.referer if referer is not "": https://bitbucket.org/galaxy/galaxy-central/commits/c01b7cf7257a/ Changeset: c01b7cf7257a User: dannon Date: 2013-05-06 18:50:06 Summary: Metadata setting: Always set externally. Affected #: 16 files diff -r 16fbd8c387a0cb7eb7607b6d37c6ea26fa9c6d50 -r c01b7cf7257af79616730a73886711b6ecf0310a lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -86,7 +86,6 @@ self.galaxy_data_manager_data_path = kwargs.get( 'galaxy_data_manager_data_path', self.tool_data_path ) self.tool_secret = kwargs.get( "tool_secret", "" ) self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" ) - self.set_metadata_externally = string_as_bool( kwargs.get( "set_metadata_externally", "False" ) ) self.retry_metadata_internally = string_as_bool( kwargs.get( "retry_metadata_internally", "True" ) ) self.use_remote_user = string_as_bool( kwargs.get( "use_remote_user", "False" ) ) self.remote_user_maildomain = kwargs.get( "remote_user_maildomain", None ) diff -r 16fbd8c387a0cb7eb7607b6d37c6ea26fa9c6d50 -r c01b7cf7257af79616730a73886711b6ecf0310a lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -28,8 +28,6 @@ STATEMENTS = "__galaxy_statements__" #this is the name of the property in a Datatype class where new metadata spec element Statements are stored -DATABASE_CONNECTION_AVAILABLE = False #When False, certain metadata parameter types (see FileParameter) will behave differently - class Statement( object ): """ This class inserts its target into a list in the surrounding @@ -442,19 +440,9 @@ return None if isinstance( value, galaxy.model.MetadataFile ) or isinstance( value, MetadataTempFile ): return value - if DATABASE_CONNECTION_AVAILABLE: - try: - # FIXME: this query requires a monkey patch in assignmapper.py since - # MetadataParameters do not have a handle to the sqlalchemy session - # DBTODO this is problematic now. - return galaxy.model.MetadataFile.get( value ) - except: - #value was not a valid id - return None - else: - mf = galaxy.model.MetadataFile() - mf.id = value #we assume this is a valid id, since we cannot check it - return mf + mf = galaxy.model.MetadataFile() + mf.id = value #we assume this is a valid id, since we cannot check it + return mf def make_copy( self, value, target_context, source_context ): value = self.wrap( value ) @@ -499,13 +487,13 @@ return value def new_file( self, dataset = None, **kwds ): - if DATABASE_CONNECTION_AVAILABLE: + if object_session( dataset ): mf = galaxy.model.MetadataFile( name = self.spec.name, dataset = dataset, **kwds ) object_session( dataset ).add( mf ) object_session( dataset ).flush() #flush to assign id return mf else: - #we need to make a tmp file that is accessable to the head node, + #we need to make a tmp file that is accessable to the head node, #we will be copying its contents into the MetadataFile objects filename after restoring from JSON #we do not include 'dataset' in the kwds passed, as from_JSON_value() will handle this for us return MetadataTempFile( **kwds ) diff -r 16fbd8c387a0cb7eb7607b6d37c6ea26fa9c6d50 -r c01b7cf7257af79616730a73886711b6ecf0310a lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -397,7 +397,7 @@ except KeyError: builder = data.Text() return builder - def change_datatype(self, data, ext, set_meta = True ): + def change_datatype(self, data, ext): data.extension = ext # call init_meta and copy metadata from itself. The datatype # being converted *to* will handle any metadata copying and @@ -405,10 +405,6 @@ if data.has_data(): data.set_size() data.init_meta( copy_from=data ) - if set_meta: - #metadata is being set internally - data.set_meta( overwrite = False ) - data.set_peek() return data def old_change_datatype(self, data, ext): """Creates and returns a new datatype based on an existing data and an extension""" diff -r 16fbd8c387a0cb7eb7607b6d37c6ea26fa9c6d50 -r c01b7cf7257af79616730a73886711b6ecf0310a lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -953,9 +953,7 @@ #either use the metadata from originating output dataset, or call set_meta on the copies #it would be quicker to just copy the metadata from the originating output dataset, #but somewhat trickier (need to recurse up the copied_from tree), for now we'll call set_meta() - if not self.app.config.set_metadata_externally or \ - ( not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) \ - and self.app.config.retry_metadata_internally ): + if ( not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) and self.app.config.retry_metadata_internally ): dataset.datatype.set_meta( dataset, overwrite = False ) #call datatype.set_meta directly for the initial set_meta call during dataset creation elif not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) and job.states.ERROR != final_job_state: dataset._state = model.Dataset.states.FAILED_METADATA @@ -1226,8 +1224,7 @@ try: for fname in self.extra_filenames: os.remove( fname ) - if self.app.config.set_metadata_externally: - self.external_output_metadata.cleanup_external_metadata( self.sa_session ) + self.external_output_metadata.cleanup_external_metadata( self.sa_session ) galaxy.tools.imp_exp.JobExportHistoryArchiveWrapper( self.job_id ).cleanup_after_job( self.sa_session ) galaxy.tools.imp_exp.JobImportHistoryArchiveWrapper( self.app, self.job_id ).cleanup_after_job() galaxy.tools.genome_index.GenomeIndexToolWrapper( self.job_id ).postprocessing( self.sa_session, self.app ) diff -r 16fbd8c387a0cb7eb7607b6d37c6ea26fa9c6d50 -r c01b7cf7257af79616730a73886711b6ecf0310a lib/galaxy/jobs/deferred/data_transfer.py --- a/lib/galaxy/jobs/deferred/data_transfer.py +++ b/lib/galaxy/jobs/deferred/data_transfer.py @@ -136,15 +136,11 @@ if name not in [ 'name', 'info', 'dbkey', 'base_name' ]: if spec.get( 'default' ): setattr( ldda.metadata, name, spec.unwrap( spec.get( 'default' ) ) ) - if self.app.config.set_metadata_externally: - self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( self.app.datatypes_registry.set_external_metadata_tool, - FakeTrans( self.app, - history=sample.history, - user=sample.request.user ), - incoming = { 'input1':ldda } ) - else: - ldda.set_meta() - ldda.datatype.after_setting_metadata( ldda ) + self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( self.app.datatypes_registry.set_external_metadata_tool, + FakeTrans( self.app, + history=sample.history, + user=sample.request.user ), + incoming = { 'input1':ldda } ) ldda.state = ldda.states.OK # TODO: not sure if this flush is necessary self.sa_session.add( ldda ) diff -r 16fbd8c387a0cb7eb7607b6d37c6ea26fa9c6d50 -r c01b7cf7257af79616730a73886711b6ecf0310a lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -174,7 +174,7 @@ # Append metadata setting commands, we don't want to overwrite metadata # that was copied over in init_meta(), as per established behavior - if include_metadata and self.app.config.set_metadata_externally: + if include_metadata: commands += "; cd %s; " % os.path.abspath( os.getcwd() ) commands += job_wrapper.setup_external_metadata( exec_dir = os.path.abspath( os.getcwd() ), diff -r 16fbd8c387a0cb7eb7607b6d37c6ea26fa9c6d50 -r c01b7cf7257af79616730a73886711b6ecf0310a lib/galaxy/jobs/runners/local.py --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -83,14 +83,14 @@ stdout_file.close() stderr_file.close() log.debug('execution finished: %s' % command_line) - except Exception, exc: + except Exception: job_wrapper.fail( "failure running job", exception=True ) log.exception("failure running job %d" % job_wrapper.job_id) return #run the metadata setting script here #this is terminate-able when output dataset/job is deleted #so that long running set_meta()s can be canceled without having to reboot the server - if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and self.app.config.set_metadata_externally and job_wrapper.output_paths: + if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and job_wrapper.output_paths: external_metadata_script = job_wrapper.setup_external_metadata( output_fnames = job_wrapper.get_output_fnames(), set_extension = True, tmp_dir = job_wrapper.working_directory, diff -r 16fbd8c387a0cb7eb7607b6d37c6ea26fa9c6d50 -r c01b7cf7257af79616730a73886711b6ecf0310a lib/galaxy/jobs/runners/lwr.py --- a/lib/galaxy/jobs/runners/lwr.py +++ b/lib/galaxy/jobs/runners/lwr.py @@ -143,7 +143,7 @@ #run the metadata setting script here #this is terminate-able when output dataset/job is deleted #so that long running set_meta()s can be canceled without having to reboot the server - if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and self.app.config.set_metadata_externally and job_wrapper.output_paths: + if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and job_wrapper.output_paths: external_metadata_script = job_wrapper.setup_external_metadata( output_fnames = job_wrapper.get_output_fnames(), set_extension = True, kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior diff -r 16fbd8c387a0cb7eb7607b6d37c6ea26fa9c6d50 -r c01b7cf7257af79616730a73886711b6ecf0310a lib/galaxy/jobs/runners/tasks.py --- a/lib/galaxy/jobs/runners/tasks.py +++ b/lib/galaxy/jobs/runners/tasks.py @@ -123,7 +123,7 @@ #run the metadata setting script here #this is terminate-able when output dataset/job is deleted #so that long running set_meta()s can be canceled without having to reboot the server - if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and self.app.config.set_metadata_externally and job_wrapper.output_paths: + if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and job_wrapper.output_paths: external_metadata_script = job_wrapper.setup_external_metadata( output_fnames = job_wrapper.get_output_fnames(), set_extension = True, kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior diff -r 16fbd8c387a0cb7eb7607b6d37c6ea26fa9c6d50 -r c01b7cf7257af79616730a73886711b6ecf0310a lib/galaxy/tools/imp_exp/__init__.py --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -189,15 +189,10 @@ # Although metadata is set above, need to set metadata to recover BAI for BAMs. if hda.extension == 'bam': - if self.app.config.set_metadata_externally: - self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute_via_app( - self.app.datatypes_registry.set_external_metadata_tool, self.app, jiha.job.session_id, - new_history.id, jiha.job.user, incoming={ 'input1': hda }, overwrite=False - ) - else: - message = 'Attributes updated' - hda.set_meta() - hda.datatype.after_setting_metadata( hda ) + self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute_via_app( + self.app.datatypes_registry.set_external_metadata_tool, self.app, jiha.job.session_id, + new_history.id, jiha.job.user, incoming={ 'input1': hda }, overwrite=False + ) # # Create jobs. diff -r 16fbd8c387a0cb7eb7607b6d37c6ea26fa9c6d50 -r c01b7cf7257af79616730a73886711b6ecf0310a lib/galaxy/webapps/galaxy/api/tools.py --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -365,15 +365,9 @@ # Set metadata. # TODO: set meta internally if dataset is small enough? - if trans.app.config.set_metadata_externally: - trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, - trans, incoming = { 'input1':new_dataset }, - overwrite=False, job_params={ "source" : "trackster" } ) - else: - message = 'Attributes updated' - new_dataset.set_meta() - new_dataset.datatype.after_setting_metadata( new_dataset ) - + trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, + trans, incoming = { 'input1':new_dataset }, + overwrite=False, job_params={ "source" : "trackster" } ) # Add HDA subset association. subset_association = trans.app.model.HistoryDatasetAssociationSubset( hda=input_dataset, subset=new_dataset, location=regions_str ) trans.sa_session.add( subset_association ) diff -r 16fbd8c387a0cb7eb7607b6d37c6ea26fa9c6d50 -r c01b7cf7257af79616730a73886711b6ecf0310a lib/galaxy/webapps/galaxy/controllers/dataset.py --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -379,10 +379,9 @@ message = "This dataset is currently being used as input or output. You cannot change datatype until the jobs have completed or you have canceled them." error = True else: - trans.app.datatypes_registry.change_datatype( data, params.datatype, set_meta = not trans.app.config.set_metadata_externally ) + trans.app.datatypes_registry.change_datatype( data, params.datatype ) trans.sa_session.flush() - if trans.app.config.set_metadata_externally: - trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming = { 'input1':data }, overwrite = False ) #overwrite is False as per existing behavior + trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming = { 'input1':data }, overwrite = False ) #overwrite is False as per existing behavior message = "Changed the type of dataset '%s' to %s" % ( to_unicode( data.name ), params.datatype ) refresh_frames=['history'] else: @@ -436,13 +435,8 @@ if name not in [ 'name', 'info', 'dbkey', 'base_name' ]: if spec.get( 'default' ): setattr( data.metadata, name, spec.unwrap( spec.get( 'default' ) ) ) - if trans.app.config.set_metadata_externally: - message = 'Attributes have been queued to be updated' - trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming = { 'input1':data } ) - else: - message = 'Attributes updated' - data.set_meta() - data.datatype.after_setting_metadata( data ) + message = 'Attributes have been queued to be updated' + trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming = { 'input1':data } ) trans.sa_session.flush() refresh_frames=['history'] elif params.convert_data: diff -r 16fbd8c387a0cb7eb7607b6d37c6ea26fa9c6d50 -r c01b7cf7257af79616730a73886711b6ecf0310a lib/galaxy/webapps/galaxy/controllers/user.py --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -1339,9 +1339,6 @@ continue # Get chrom count file. - # NOTE: this conversion doesn't work well with set_metadata_externally=False - # because the conversion occurs before metadata can be set; the - # dataset is marked as deleted and a subsequent conversion is run. chrom_count_dataset = len_dataset.get_converted_dataset( trans, "linecount" ) if not chrom_count_dataset or chrom_count_dataset.state != trans.app.model.Job.states.OK: # No valid linecount dataset. diff -r 16fbd8c387a0cb7eb7607b6d37c6ea26fa9c6d50 -r c01b7cf7257af79616730a73886711b6ecf0310a scripts/extract_dataset_part.py --- a/scripts/extract_dataset_part.py +++ b/scripts/extract_dataset_part.py @@ -24,7 +24,6 @@ # This junk is here to prevent loading errors import galaxy.model.mapping #need to load this before we unpickle, in order to setup properties assigned by the mappers galaxy.model.Job() #this looks REAL stupid, but it is REQUIRED in order for SA to insert parameters into the classes defined by the mappers --> it appears that instantiating ANY mapper'ed class would suffice here -galaxy.datatypes.metadata.DATABASE_CONNECTION_AVAILABLE = False #Let metadata know that there is no database connection, and to just assume object ids are valid def __main__(): """ diff -r 16fbd8c387a0cb7eb7607b6d37c6ea26fa9c6d50 -r c01b7cf7257af79616730a73886711b6ecf0310a scripts/functional_tests.py --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -185,7 +185,6 @@ cluster_files_directory = cluster_files_directory, job_working_directory = job_working_directory, outputs_to_working_directory = 'True', - set_metadata_externally = 'True', static_enabled = 'False', debug = 'False', track_jobs_in_database = 'True', diff -r 16fbd8c387a0cb7eb7607b6d37c6ea26fa9c6d50 -r c01b7cf7257af79616730a73886711b6ecf0310a scripts/set_metadata.py --- a/scripts/set_metadata.py +++ b/scripts/set_metadata.py @@ -28,7 +28,6 @@ import simplejson import galaxy.model.mapping #need to load this before we unpickle, in order to setup properties assigned by the mappers galaxy.model.Job() #this looks REAL stupid, but it is REQUIRED in order for SA to insert parameters into the classes defined by the mappers --> it appears that instantiating ANY mapper'ed class would suffice here -galaxy.datatypes.metadata.DATABASE_CONNECTION_AVAILABLE = False #Let metadata know that there is no database connection, and to just assume object ids are valid from galaxy.util import stringify_dictionary_keys from galaxy.util.json import from_json_string from sqlalchemy.orm import clear_mappers https://bitbucket.org/galaxy/galaxy-central/commits/b72a06cf0247/ Changeset: b72a06cf0247 User: dannon Date: 2013-05-06 18:53:32 Summary: Merge Affected #: 4 files Diff not available. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: james_taylor: Password security: use PBKDF2 scheme with sha256 for any new/changed passwords. Scheme is encoded in the database field to allow other schemes to be used in the future. If no recognized prefix is seen, sha1 is assumed.
by commits-noreply＠bitbucket.org 06 May '13

06 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a6720cff69b1/ Changeset: a6720cff69b1 User: james_taylor Date: 2013-05-05 18:17:33 Summary: Password security: use PBKDF2 scheme with sha256 for any new/changed passwords. Scheme is encoded in the database field to allow other schemes to be used in the future. If no recognized prefix is seen, sha1 is assumed. Affected #: 4 files diff -r 02822f28dc93348264a18f9f30abfedb53b80728 -r a6720cff69b1a4f142c9010c5d19f07c32b4475a lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -21,6 +21,7 @@ import galaxy.datatypes import galaxy.datatypes.registry +import galaxy.security.passwords from galaxy.datatypes.metadata import MetadataCollection from galaxy.model.item_attrs import APIItem, UsesAnnotations from galaxy.security import get_permitted_actions @@ -86,13 +87,13 @@ """ Set user password to the digest of `cleartext`. """ - self.password = new_secure_hash( text_type=cleartext ) + self.password = galaxy.security.passwords.hash_password( cleartext ) def check_password( self, cleartext ): """ Check if `cleartext` matches user password when hashed. """ - return self.password == new_secure_hash( text_type=cleartext ) + return galaxy.security.passwords.check_password( cleartext, self.password ) def all_roles( self ): """ diff -r 02822f28dc93348264a18f9f30abfedb53b80728 -r a6720cff69b1a4f142c9010c5d19f07c32b4475a lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -51,7 +51,7 @@ Column( "update_time", DateTime, default=now, onupdate=now ), Column( "email", TrimmedString( 255 ), index=True, nullable=False ), Column( "username", TrimmedString( 255 ), index=True, unique=True ), - Column( "password", TrimmedString( 40 ), nullable=False ), + Column( "password", TrimmedString( 255 ), nullable=False ), Column( "external", Boolean, default=False ), Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ), Column( "deleted", Boolean, index=True, default=False ), diff -r 02822f28dc93348264a18f9f30abfedb53b80728 -r a6720cff69b1a4f142c9010c5d19f07c32b4475a lib/galaxy/model/migrate/versions/0115_longer_user_password_field.py --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0115_longer_user_password_field.py @@ -0,0 +1,25 @@ +""" +Expand the length of the password fields in the galaxy_user table to allow for other hasing schemes +""" + +from sqlalchemy import * +from migrate import * + +import logging +log = logging.getLogger( __name__ ) + +def upgrade( migrate_engine ): + meta = MetaData( bind=migrate_engine ) + user = Table( 'galaxy_user', meta, autoload=True ) + try: + user.c.password.alter(type=String(255)) + except: + log.exception( "Altering password column failed" ) + +def downgrade(migrate_engine): + meta = MetaData( bind=migrate_engine ) + user = Table( 'galaxy_user', meta, autoload=True ) + try: + user.c.password.alter(type=String(40)) + except: + log.exception( "Altering password column failed" ) \ No newline at end of file diff -r 02822f28dc93348264a18f9f30abfedb53b80728 -r a6720cff69b1a4f142c9010c5d19f07c32b4475a lib/galaxy/security/passwords.py --- /dev/null +++ b/lib/galaxy/security/passwords.py @@ -0,0 +1,83 @@ +import hmac +import hashlib +from struct import Struct +from operator import xor +from itertools import izip, starmap +from os import urandom +from base64 import b64encode + +SALT_LENGTH = 12 +KEY_LENGTH = 24 +HASH_FUNCTION = 'sha256' +COST_FACTOR = 10000 + +def hash_password( password ): + """ + Hash a password, currently will use the PBKDF2 scheme. + """ + return hash_password_PBKDF2( password ) + +def check_password( guess, hashed ): + """ + Check a hashed password. Supports either PBKDF2 if the hash is + prefixed with that string, or sha1 otherwise. + """ + if hashed.startswith( "PBKDF2" ): + if check_password_PBKDF2( guess, hashed ): + return True + else: + # Passwords were originally encoded with sha1 and hexed + if hashlib.sha1( guess ).hexdigest() == hashed: + return True + # Password does not match + return False + +def hash_password_PBKDF2( password ): + # Generate a random salt + salt = b64encode( urandom( SALT_LENGTH ) ) + # Apply the pbkdf2 encoding + hashed = pbkdf2_bin( bytes( password ), salt, COST_FACTOR, KEY_LENGTH, getattr( hashlib, HASH_FUNCTION ) ) + # Format + return 'PBKDF2${0}${1}${2}${3}'.format( HASH_FUNCTION, COST_FACTOR, salt, b64encode( hashed ) ) + +def check_password_PBKDF2( guess, hashed ): + # Split the database representation to extract cost_factor and salt + name, hash_function, cost_factor, salt, encoded_original = hashed.split( '$', 5 ) + # Hash the guess using the same parameters + hashed_guess = pbkdf2_bin( bytes( guess ), salt, int( cost_factor ), KEY_LENGTH, getattr( hashlib, hash_function ) ) + encoded_guess = b64encode( hashed_guess ) + return safe_str_cmp( encoded_original, encoded_guess ) + +## Taken from https://github.com/mitsuhiko/python-pbkdf2/blob/master/pbkdf2.py +## (c) Copyright 2011 by Armin Ronacher, BSD LICENSE + +_pack_int = Struct('>I').pack + +def pbkdf2_bin( data, salt, iterations=1000, keylen=24, hashfunc=None ): + """Returns a binary digest for the PBKDF2 hash algorithm of `data` + with the given `salt`. It iterates `iterations` time and produces a + key of `keylen` bytes. By default SHA-1 is used as hash function, + a different hashlib `hashfunc` can be provided. + """ + hashfunc = hashfunc or hashlib.sha1 + mac = hmac.new(data, None, hashfunc) + def _pseudorandom(x, mac=mac): + h = mac.copy() + h.update(x) + return map(ord, h.digest()) + buf = [] + for block in xrange(1, -(-keylen // mac.digest_size) + 1): + rv = u = _pseudorandom(salt + _pack_int(block)) + for i in xrange(iterations - 1): + u = _pseudorandom(''.join(map(chr, u))) + rv = starmap(xor, izip(rv, u)) + buf.extend(rv) + return ''.join(map(chr, buf))[:keylen] + +def safe_str_cmp(a, b): + if len(a) != len(b): + return False + rv = 0 + for x, y in izip(a, b): + rv |= ord(x) ^ ord(y) + return rv == 0 \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Temporarily disable database_connection in metadata.py until a better solution is found
by commits-noreply＠bitbucket.org 05 May '13

05 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/02822f28dc93/ Changeset: 02822f28dc93 User: dannon Date: 2013-05-05 17:05:50 Summary: Temporarily disable database_connection in metadata.py until a better solution is found Affected #: 1 file diff -r c823b1404fea2350d9c0b2d0ebc4965d7f9ffcb8 -r 02822f28dc93348264a18f9f30abfedb53b80728 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -28,7 +28,7 @@ STATEMENTS = "__galaxy_statements__" #this is the name of the property in a Datatype class where new metadata spec element Statements are stored -DATABASE_CONNECTION_AVAILABLE = True #When False, certain metadata parameter types (see FileParameter) will behave differently +DATABASE_CONNECTION_AVAILABLE = False #When False, certain metadata parameter types (see FileParameter) will behave differently class Statement( object ): """ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Remove debug filtering.
by commits-noreply＠bitbucket.org 04 May '13

04 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c823b1404fea/ Changeset: c823b1404fea User: inithello Date: 2013-05-04 15:07:50 Summary: Remove debug filtering. Affected #: 1 file diff -r d207aa915b9d89167858439df97bf385669b4a2d -r c823b1404fea2350d9c0b2d0ebc4965d7f9ffcb8 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -572,14 +572,7 @@ # it will record the result of the tests, and if any failed, the traceback and captured output of the tool that was run. # After all tests have completed, the repository is uninstalled, so that the previous test cases don't interfere with # the next repository's functional tests. - keep_skipping = True for repository_info_dict in detailed_repository_list: - if repository_info_dict[ 'name' ] == 'simsearch' and repository_info_dict[ 'changeset_revision' ] == '430f55978191': - keep_skipping = False - if keep_skipping: - continue - if repository_info_dict[ 'name' ] == 'sortmerna': - continue """ Each repository_info_dict looks something like: { Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: When deleting a history and marking associated jobs deleted, first check that there is a job to mark.
by commits-noreply＠bitbucket.org 03 May '13

03 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d207aa915b9d/ Changeset: d207aa915b9d User: jgoecks Date: 2013-05-04 00:02:57 Summary: When deleting a history and marking associated jobs deleted, first check that there is a job to mark. Affected #: 1 file diff -r e7feb8e93b6e862b8b450dc5a9636297cea28516 -r d207aa915b9d89167858439df97bf385669b4a2d lib/galaxy/webapps/galaxy/controllers/history.py --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -563,11 +563,14 @@ self.sa_session.add( history ) self.sa_session.flush() for hda in history.datasets: - job = hda.creating_job_associations[0].job - if job.history_id == history.id and job.state in [ trans.app.model.Job.states.QUEUED, trans.app.model.Job.states.RUNNING, trans.app.model.Job.states.NEW ]: - # No need to check other outputs since the job's parent history is this history - job.mark_deleted( trans.app.config.track_jobs_in_database ) - trans.app.job_manager.job_stop_queue.put( job.id ) + # Not all datasets have jobs associated with them (e.g., datasets imported from libraries). + if hda.creating_job_associations: + # HDA has associated job, so try marking it deleted. + job = hda.creating_job_associations[0].job + if job.history_id == history.id and job.state in [ trans.app.model.Job.states.QUEUED, trans.app.model.Job.states.RUNNING, trans.app.model.Job.states.NEW ]: + # No need to check other outputs since the job's parent history is this history + job.mark_deleted( trans.app.config.track_jobs_in_database ) + trans.app.job_manager.job_stop_queue.put( job.id ) # Regardless of whether it was previously deleted, we make a new history active trans.new_history() return trans.show_ok_message( "History deleted, a new history is active", refresh_frames=['history'] ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 03 May '13

03 May '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/45a70edb67ba/ Changeset: 45a70edb67ba User: dan Date: 2013-05-03 22:02:39 Summary: Provide user to tools as __user__. Affected #: 2 files diff -r b5ea0e100bdaf47d53d11fa2beb39f4218fa8136 -r 45a70edb67baae53fa97de95b4e39004687a0c23 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -678,13 +678,16 @@ # These can be passed on the command line if wanted as $__user_*__ if job.history and job.history.user: + user = job.history.user user_id = '%d' % job.history.user.id user_email = str(job.history.user.email) user_name = str(job.history.user.username) else: + user = None user_id = 'Anonymous' user_email = 'Anonymous' user_name = 'Anonymous' + incoming['__user__'] = user incoming['__user_id__'] = incoming['userId'] = user_id incoming['__user_email__'] = incoming['userEmail'] = user_email incoming['__user_name__'] = user_name diff -r b5ea0e100bdaf47d53d11fa2beb39f4218fa8136 -r 45a70edb67baae53fa97de95b4e39004687a0c23 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2546,6 +2546,7 @@ param_dict['__root_dir__'] = param_dict['GALAXY_ROOT_DIR'] = os.path.abspath( self.app.config.root ) param_dict['__datatypes_config__'] = param_dict['GALAXY_DATATYPES_CONF_FILE'] = self.app.datatypes_registry.integrated_datatypes_configs param_dict['__admin_users__'] = self.app.config.admin_users + param_dict['__user__'] = RawObjectWrapper( param_dict.get( '__user__', None ) ) # Return the dictionary of parameters return param_dict def build_param_file( self, param_dict, directory=None ): @@ -3243,6 +3244,8 @@ """ def __init__( self, obj ): self.obj = obj + def __nonzero__( self ): + return bool( self.obj ) #FIXME: would it be safe/backwards compatible to rename .obj to .value, so that we can just inherit this method? def __str__( self ): return "%s:%s" % (self.obj.__module__, self.obj.__class__.__name__) def __getattr__( self, key ): https://bitbucket.org/galaxy/galaxy-central/commits/e7feb8e93b6e/ Changeset: e7feb8e93b6e User: dan Date: 2013-05-03 22:02:40 Summary: Update GenomeSpace tools to use __user__ as needed. Affected #: 2 files diff -r 45a70edb67baae53fa97de95b4e39004687a0c23 -r e7feb8e93b6e862b8b450dc5a9636297cea28516 tools/genomespace/genomespace_exporter.xml --- a/tools/genomespace/genomespace_exporter.xml +++ b/tools/genomespace/genomespace_exporter.xml @@ -3,10 +3,9 @@ <description> - send data to GenomeSpace</description><command interpreter="python">genomespace_exporter.py --genomespace_site "prod" - #assert $__user_id__ != 'Anonymous', Exception( 'You must be logged in to use this tool.' ) - #set $user = $__app__.model.User.get( $__user_id__ ) - #set $username = $user.preferences.get( 'genomespace_username', None ) - #set $token = $user.preferences.get( 'genomespace_token', None ) + #assert $__user__, Exception( 'You must be logged in to use this tool.' ) + #set $username = $__user__.preferences.get( 'genomespace_username', None ) + #set $token = $__user__.preferences.get( 'genomespace_token', None ) #assert None not in ( $username, $token ), Exception( 'You must associate a GenomeSpace OpenID with your account and log in with it.' ) #import binascii --username "${username}" @@ -36,7 +35,7 @@ <inputs><param format="data" name="input1" type="data" label="Send this dataset to GenomeSpace" /><param name="base_url" type="baseurl" /> - <param name="subdirectory" type="drill_down" display="radio" hierarchy="exact" multiple="False" label="Choose Target Directory" dynamic_options="galaxy_code_get_genomespace_folders( genomespace_site = 'prod', trans=__trans__, value=__value__, input_dataset=input1 )" help="Leave blank to generate automatically"/> + <param name="subdirectory" type="drill_down" display="radio" hierarchy="exact" multiple="False" optional="True" label="Choose Target Directory" dynamic_options="galaxy_code_get_genomespace_folders( genomespace_site = 'prod', trans=__trans__, value=__value__, input_dataset=input1 )" help="Leave blank to generate automatically"/><param name="filename" type="text" size="80" help="Leave blank to generate automatically" /></inputs><outputs> diff -r 45a70edb67baae53fa97de95b4e39004687a0c23 -r e7feb8e93b6e862b8b450dc5a9636297cea28516 tools/genomespace/genomespace_importer.xml --- a/tools/genomespace/genomespace_importer.xml +++ b/tools/genomespace/genomespace_importer.xml @@ -3,10 +3,9 @@ <description> - receive data from GenomeSpace</description><command interpreter="python">genomespace_importer.py --genomespace_site "prod" - #assert $__user_id__ != 'Anonymous', Exception( 'You must be logged in to use this tool.' ) - #set $user = $__app__.model.User.get( $__user_id__ ) - #set $username = $user.preferences.get( 'genomespace_username', None ) - #set $token = $user.preferences.get( 'genomespace_token', None ) + #assert $__user__, Exception( 'You must be logged in to use this tool.' ) + #set $username = $__user__.preferences.get( 'genomespace_username', None ) + #set $token = $__user__.preferences.get( 'genomespace_token', None ) #assert None not in ( $username, $token ), Exception( 'You must associate a GenomeSpace OpenID with your account and log in with it.' ) --username "${username}" --token "${token}" Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Rename install_repository_tools to install_tool_shed_repositories. Use shed_util_common.url_join to generate the API url.
by commits-noreply＠bitbucket.org 03 May '13

03 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b5ea0e100bda/ Changeset: b5ea0e100bda User: inithello Date: 2013-05-03 21:55:56 Summary: Rename install_repository_tools to install_tool_shed_repositories. Use shed_util_common.url_join to generate the API url. Affected #: 1 file diff -r 75e7383c5c794e02503dc03e0d5daf3694323b7d -r b5ea0e100bdaf47d53d11fa2beb39f4218fa8136 scripts/api/install_tool_shed_repositories.py --- /dev/null +++ b/scripts/api/install_tool_shed_repositories.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +""" +Install a specified repository revision from a specified tool shed into Galaxy. This example demonstrates installation of a repository that contains +valid tools, loading them into a section of the Galaxy tool panel or creating a new tool panel section. +You can choose if tool dependencies or repository dependencies should be installed, use --repository-deps or --tool-deps. + +This example requires a tool panel config file (e.g., tool_conf.xml, shed_tool_conf.xml, etc) to contain a tool panel section like the following: + +<section id="from_test_tool_shed" name="From Test Tool Shed" version=""> +</section> + +Here is a working example of how to use this script to install a repository from the test tool shed. +./install_repository_tools.py --api <api key> --local <galaxy base url> --url http://testtoolshed.g2.bx.psu.edu --name gregs_filter --owner greg --revision f28d5018f9cb --tool-deps +""" + +import os +import sys +import argparse +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +def main( options ): + """Collect all user data and install the tools via the Galaxy API.""" + data = {} + data[ 'tool_shed_url' ] = options.tool_shed_url + data[ 'name' ] = options.name + data[ 'owner' ] = options.owner + data[ 'changeset_revision' ] = options.changeset_revision + if options.tool_panel_section_id: + data[ 'tool_panel_section_id' ] = options.tool_panel_section_id + elif options.new_tool_panel_section_label: + data[ 'new_tool_panel_section_label' ] = options.new_tool_panel_section_label + if options.install_repository_dependencies: + data[ 'install_repository_dependencies' ] = options.install_repository_dependencies + if options.install_tool_dependencies: + data[ 'install_tool_dependencies' ] = options.install_tool_dependencies + submit( options.api, '%s%s' % ( options.local_url.strip('/'), '/api/tool_shed_repositories/new/install_repository_revision' ), data ) + +if __name__ == '__main__': + parser = argparse.ArgumentParser( description='Installation of tool shed repositories via the Galaxy API.' ) + parser.add_argument( "-u", "--url", dest="tool_shed_url", required=True, help="Tool Shed URL" ) + parser.add_argument( "-a", "--api", dest="api", required=True, help="API Key" ) + parser.add_argument( "-l", "--local", dest="local_url", required=True, help="URL of the galaxy instance." ) + parser.add_argument( "-n", "--name", required=True, help="Repository name." ) + parser.add_argument( "-o", "--owner", required=True, help="Repository owner." ) + parser.add_argument( "-r", "--revision", dest="changeset_revision", required=True, help="Repository owner." ) + parser.add_argument( "--panel-section-id", dest="tool_panel_section_id", help="Tool panel section id if you want to add your repository to an existing tool section." ) + parser.add_argument( "--panel-section-name", dest="new_tool_panel_section_label", help="New tool panel section label. If specified a new tool section will be created." ) + parser.add_argument( "--repository-deps", dest="install_repository_dependencies", action="store_true", default=False, help="Install repository dependencies. [False]") + parser.add_argument( "--tool-deps", dest="install_tool_dependencies", action="store_true", default=False, help="Install tool dependencies. [False]" ) + options = parser.parse_args() + main( options ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

Jump to page:

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

galaxy-commits May 2013