December 2011 - galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: natefoo: Add control for deleting job files after job completion.
by Bitbucket 06 Dec '11

06 Dec '11

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/5f044d41a8a7/ changeset: 5f044d41a8a7 user: natefoo date: 2011-12-06 18:41:10 summary: Add control for deleting job files after job completion. affected #: 5 files diff -r c831a584e46e1995ab2b673139ffda42147bef8b -r 5f044d41a8a74af74f36d020206f9807c55f0d34 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -80,6 +80,7 @@ self.job_queue_cleanup_interval = int( kwargs.get("job_queue_cleanup_interval", "5") ) self.cluster_files_directory = os.path.abspath( kwargs.get( "cluster_files_directory", "database/pbs" ) ) self.job_working_directory = resolve_path( kwargs.get( "job_working_directory", "database/job_working_directory" ), self.root ) + self.cleanup_job = kwargs.get( "cleanup_job", "always" ) self.outputs_to_working_directory = string_as_bool( kwargs.get( 'outputs_to_working_directory', False ) ) self.output_size_limit = int( kwargs.get( 'output_size_limit', 0 ) ) self.job_walltime = kwargs.get( 'job_walltime', None ) diff -r c831a584e46e1995ab2b673139ffda42147bef8b -r 5f044d41a8a74af74f36d020206f9807c55f0d34 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -482,7 +482,8 @@ # If the job was deleted, call tool specific fail actions (used for e.g. external metadata) and clean up if self.tool: self.tool.job_failed( self, message, exception ) - self.cleanup() + if self.app.config.cleanup_job == 'always': + self.cleanup() def change_state( self, state, info = False ): job = self.get_job() @@ -525,7 +526,8 @@ job = self.get_job() # if the job was deleted, don't finish it if job.state == job.states.DELETED: - self.cleanup() + if self.app.config.cleanup_job in ( 'always', 'onsuccess' ): + self.cleanup() return elif job.state == job.states.ERROR: # Job was deleted by an administrator @@ -698,7 +700,8 @@ util.umask_fix_perms( path, self.app.config.umask, 0666, self.app.config.gid ) self.sa_session.flush() log.debug( 'job %d ended' % self.job_id ) - self.cleanup() + if self.app.config.cleanup_job in ( 'always', 'onsuccess' ): + self.cleanup() def cleanup( self ): # remove temporary files @@ -1045,7 +1048,8 @@ task = self.get_task() # if the job was deleted, don't finish it if task.state == task.states.DELETED: - self.cleanup() + if self.app.config.cleanup_job in ( 'always', 'onsuccess' ): + self.cleanup() return elif task.state == task.states.ERROR: # Job was deleted by an administrator diff -r c831a584e46e1995ab2b673139ffda42147bef8b -r 5f044d41a8a74af74f36d020206f9807c55f0d34 lib/galaxy/jobs/runners/drmaa.py --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -137,7 +137,8 @@ # Check for deletion before we change state if job_wrapper.get_state() == model.Job.states.DELETED: log.debug( "Job %s deleted by user before it entered the queue" % job_wrapper.get_id_tag() ) - job_wrapper.cleanup() + if self.app.config.cleanup_job in ( "always", "onsuccess" ): + job_wrapper.cleanup() return # Change to queued state immediately @@ -168,8 +169,9 @@ # job was deleted while we were preparing it if job_wrapper.get_state() == model.Job.states.DELETED: log.debug( "Job %s deleted by user before it entered the queue" % job_wrapper.get_id_tag() ) - self.cleanup( ( ofile, efile, jt.remoteCommand ) ) - job_wrapper.cleanup() + if self.app.config.cleanup_job in ( "always", "onsuccess" ): + self.cleanup( ( ofile, efile, jt.remoteCommand ) ) + job_wrapper.cleanup() return # wrapper.get_id_tag() instead of job_id for compatibility with TaskWrappers. @@ -288,7 +290,8 @@ log.exception("Job wrapper finish method failed") # clean up the drm files - self.cleanup( ( ofile, efile, job_file ) ) + if self.app.config.cleanup_job == "always" or ( not stderr and self.app.config.cleanup_job == "onsuccess" )): + self.cleanup( ( ofile, efile, job_file ) ) def fail_job( self, drm_job_state ): """ @@ -296,13 +299,15 @@ """ self.stop_job( self.sa_session.query( self.app.model.Job ).get( drm_job_state.job_wrapper.job_id ) ) drm_job_state.job_wrapper.fail( drm_job_state.fail_message ) - self.cleanup( ( drm_job_state.ofile, drm_job_state.efile, drm_job_state.job_file ) ) + if self.app.config.cleanup_job == "always": + self.cleanup( ( drm_job_state.ofile, drm_job_state.efile, drm_job_state.job_file ) ) def cleanup( self, files ): - if not asbool( self.app.config.get( 'debug', False ) ): - for file in files: - if os.access( file, os.R_OK ): - os.unlink( file ) + for file in files: + try: + os.unlink( file ) + except Exception, e: + log.warning( "Unable to cleanup: %s" % str( e ) ) def put( self, job_wrapper ): """Add a job to the queue (by job identifier)""" diff -r c831a584e46e1995ab2b673139ffda42147bef8b -r 5f044d41a8a74af74f36d020206f9807c55f0d34 lib/galaxy/jobs/runners/pbs.py --- a/lib/galaxy/jobs/runners/pbs.py +++ b/lib/galaxy/jobs/runners/pbs.py @@ -214,7 +214,8 @@ # Check for deletion before we change state if job_wrapper.get_state() == model.Job.states.DELETED: log.debug( "Job %s deleted by user before it entered the PBS queue" % job_wrapper.job_id ) - job_wrapper.cleanup() + if self.app.config.cleanup_job in ( "always", "onsuccess" ): + job_wrapper.cleanup() return ( pbs_server_name, runner_url ) = self.determine_pbs_server( runner_url, rewrite = True ) @@ -277,8 +278,9 @@ if job_wrapper.get_state() == model.Job.states.DELETED: log.debug( "Job %s deleted by user before it entered the PBS queue" % job_wrapper.job_id ) pbs.pbs_disconnect(c) - self.cleanup( ( ofile, efile, job_file ) ) - job_wrapper.cleanup() + if self.app.config.cleanup_job in ( "always", "onsuccess" ): + self.cleanup( ( ofile, efile, job_file ) ) + job_wrapper.cleanup() return # submit @@ -517,7 +519,8 @@ pbs_job_state.job_wrapper.fail("Unable to finish job", exception=True) # clean up the pbs files - self.cleanup( ( ofile, efile, job_file ) ) + if self.app.config.cleanup_job == "always" or ( not stderr and self.app.config.cleanup_job == "onsuccess" )): + self.cleanup( ( ofile, efile, job_file ) ) def fail_job( self, pbs_job_state ): """ @@ -526,13 +529,15 @@ if pbs_job_state.stop_job: self.stop_job( self.sa_session.query( self.app.model.Job ).get( pbs_job_state.job_wrapper.job_id ) ) pbs_job_state.job_wrapper.fail( pbs_job_state.fail_message ) - self.cleanup( ( pbs_job_state.ofile, pbs_job_state.efile, pbs_job_state.job_file ) ) + if self.app.config.cleanup_job == "always": + self.cleanup( ( pbs_job_state.ofile, pbs_job_state.efile, pbs_job_state.job_file ) ) def cleanup( self, files ): - if not asbool( self.app.config.get( 'debug', False ) ): - for file in files: - if os.access( file, os.R_OK ): - os.unlink( file ) + for file in files: + try: + os.unlink( file ) + except Exception, e: + log.warning( "Unable to cleanup: %s" % str( e ) ) def put( self, job_wrapper ): """Add a job to the queue (by job identifier)""" diff -r c831a584e46e1995ab2b673139ffda42147bef8b -r 5f044d41a8a74af74f36d020206f9807c55f0d34 universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -514,6 +514,12 @@ # Galaxy server after the job completes. #outputs_to_working_directory = False +# Clean up various bits of jobs left on the filesystem after completion. These +# bits include the job working directory, external metadata temporary files, +# and DRM stdout and stderr files (if using a DRM). Possible values are: +# always, onsuccess, never +#cleanup_job = always + # Number of concurrent jobs to run (local job runner) #local_job_queue_workers = 5 Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: natefoo: The upload tool requires samtools to upload BAM files, the Pileup tool requires samtools <= 0.1.16.
by Bitbucket 06 Dec '11

06 Dec '11

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/c831a584e46e/ changeset: c831a584e46e user: natefoo date: 2011-12-06 17:42:12 summary: The upload tool requires samtools to upload BAM files, the Pileup tool requires samtools <= 0.1.16. affected #: 2 files diff -r ded3aac16a24060d04c34da015958a1aee94610b -r c831a584e46e1995ab2b673139ffda42147bef8b tools/data_source/upload.xml --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -5,6 +5,9 @@ from your computer </description><action module="galaxy.tools.actions.upload" class="UploadToolAction"/> + <requirements> + <requirement type="package">samtools</requirement> + </requirements><command interpreter="python"> upload.py $GALAXY_ROOT_DIR $GALAXY_DATATYPES_CONF_FILE $paramfile #set $outnum = 0 diff -r ded3aac16a24060d04c34da015958a1aee94610b -r c831a584e46e1995ab2b673139ffda42147bef8b tools/samtools/sam_pileup.xml --- a/tools/samtools/sam_pileup.xml +++ b/tools/samtools/sam_pileup.xml @@ -1,7 +1,7 @@ <tool id="sam_pileup" name="Generate pileup" version="1.1.1"><description>from BAM dataset</description><requirements> - <requirement type="package">samtools</requirement> + <requirement type="package" version="0.1.16">samtools</requirement></requirements><command interpreter="python"> sam_pileup.py Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: natefoo: Missing template changes from the last commit.
by Bitbucket 06 Dec '11

06 Dec '11

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/ded3aac16a24/ changeset: ded3aac16a24 user: natefoo date: 2011-12-06 17:41:32 summary: Missing template changes from the last commit. affected #: 1 file diff -r 3c9f7bfc1bcf2be295c61857b836eb44b7dc8982 -r ded3aac16a24060d04c34da015958a1aee94610b templates/show_params.mako --- a/templates/show_params.mako +++ b/templates/show_params.mako @@ -50,6 +50,8 @@ <tr><td>Dbkey:</td><td>${hda.dbkey}</td></tr><tr><td>Format:</td><td>${hda.ext}</td></tr><tr><td>Tool Version:</td><td>${hda.tool_version}</td></tr> + <tr><td>Tool Standard Output:</td><td><a href="${h.url_for( controller='dataset', action='stdout')}">stdout</a></td></tr> + <tr><td>Tool Standard Error:</td><td><a href="${h.url_for( controller='dataset', action='stderr')}">stderr</a></td></tr></table><br /><table class="tabletip"> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: natefoo: Fix the dataset controller's stderr method and add one for stdout. Link to
by Bitbucket 06 Dec '11

06 Dec '11

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/3c9f7bfc1bcf/ changeset: 3c9f7bfc1bcf user: natefoo date: 2011-12-06 17:29:02 summary: Fix the dataset controller's stderr method and add one for stdout. Link to both from the dataset info page. The functional tests will now provide output of these on tool test failure. The functional tests can now use tool_dependency_dir as well. affected #: 4 files diff -r 0a901d00d8807a50017840e72fa37f2bf7cf2bcf -r 3c9f7bfc1bcf2be295c61857b836eb44b7dc8982 lib/galaxy/web/controllers/dataset.py --- a/lib/galaxy/web/controllers/dataset.py +++ b/lib/galaxy/web/controllers/dataset.py @@ -154,10 +154,24 @@ hda = trans.sa_session.query( model.HistoryDatasetAssociation ).get( id ) return trans.fill_template( "dataset/errors.mako", hda=hda ) @web.expose - def stderr( self, trans, id ): - dataset = trans.sa_session.query( model.HistoryDatasetAssociation ).get( id ) - job = dataset.creating_job_associations[0].job + def stdout( self, trans, dataset_id=None, **kwargs ): trans.response.set_content_type( 'text/plain' ) + try: + hda = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( trans.security.decode_id( dataset_id ) ) + assert hda and trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), hda.dataset ) + job = hda.creating_job_associations[0].job + except: + return "Invalid dataset ID or you are not allowed to access this dataset" + return job.stdout + @web.expose + def stderr( self, trans, dataset_id=None, **kwargs ): + trans.response.set_content_type( 'text/plain' ) + try: + hda = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( trans.security.decode_id( dataset_id ) ) + assert hda and trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), hda.dataset ) + job = hda.creating_job_associations[0].job + except: + return "Invalid dataset ID or you are not allowed to access this dataset" return job.stderr @web.expose def report_error( self, trans, id, email='', message="" ): diff -r 0a901d00d8807a50017840e72fa37f2bf7cf2bcf -r 3c9f7bfc1bcf2be295c61857b836eb44b7dc8982 scripts/functional_tests.py --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -58,6 +58,7 @@ tool_path = os.environ.get( 'GALAXY_TEST_TOOL_PATH', 'tools' ) tool_config_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', 'tool_conf.xml.sample' ) tool_data_table_config_path = 'tool_data_table_conf.xml' + tool_dependency_dir = os.environ.get( 'GALAXY_TOOL_DEPENDENCY_DIR', None ) if os.path.exists( 'tool_data_table_conf.test.xml' ): tool_data_table_config_path = 'tool_data_table_conf.test.xml' if start_server: @@ -149,6 +150,9 @@ if not database_connection.startswith( 'sqlite://' ): kwargs['database_engine_option_max_overflow'] = '20' + if tool_dependency_dir is not None: + kwargs['tool_dependency_dir'] = tool_dependency_dir + # Build the Universe Application app = UniverseApplication( job_queue_workers = 5, id_secret = 'changethisinproductiontoo', diff -r 0a901d00d8807a50017840e72fa37f2bf7cf2bcf -r 3c9f7bfc1bcf2be295c61857b836eb44b7dc8982 test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -624,6 +624,14 @@ fd,temp_prefix = tempfile.mkstemp(prefix='tmp',suffix=suffix) return temp_prefix + def get_history_dataset_stdout( self, hda_id ): + self.visit_page( "/datasets/%s/stdout" % hda_id ) + return self.last_page() + + def get_history_dataset_stderr( self, hda_id ): + self.visit_page( "/datasets/%s/stderr" % hda_id ) + return self.last_page() + def verify_dataset_correctness( self, filename, hid=None, wait=True, maxseconds=120, attributes=None ): """Verifies that the attributes and contents of a history item meet expectations""" if wait: diff -r 0a901d00d8807a50017840e72fa37f2bf7cf2bcf -r 3c9f7bfc1bcf2be295c61857b836eb44b7dc8982 test/functional/test_toolbox.py --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -93,7 +93,19 @@ self.assertTrue( elem is not None ) elem_hid = elem.get( 'hid' ) elem_index += 1 - self.verify_dataset_correctness( outfile, hid=elem_hid, maxseconds=testdef.maxseconds, attributes=attributes ) + try: + self.verify_dataset_correctness( outfile, hid=elem_hid, maxseconds=testdef.maxseconds, attributes=attributes ) + except Exception, e: + print '-----------' + print 'Job STDOUT:' + print '-----------' + print self.get_history_dataset_stdout( elem.get( 'id' ) ) + print '-----------' + print 'Job STDERR:' + print '-----------' + print self.get_history_dataset_stderr( elem.get( 'id' ) ) + print '-----------' + raise self.delete_history( id=self.security.encode_id( latest_history.id ) ) def __expand_grouping( self, tool_inputs, declared_inputs, prefix='' ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Update GATK tools handling annotations to allow manually specifying additional annotation names to apply.
by Bitbucket 06 Dec '11

06 Dec '11

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/0a901d00d880/ changeset: 0a901d00d880 user: dan date: 2011-12-06 15:42:56 summary: Update GATK tools handling annotations to allow manually specifying additional annotation names to apply. affected #: 3 files diff -r bbda064ee4b6ffff2d867a389b81d884c772aa12 -r 0a901d00d8807a50017840e72fa37f2bf7cf2bcf tools/gatk/unified_genotyper.xml --- a/tools/gatk/unified_genotyper.xml +++ b/tools/gatk/unified_genotyper.xml @@ -117,6 +117,9 @@ --annotation "${annotation}" #end for #end if + #for $additional_annotation in $analysis_param_type.additional_annotations: + --annotation "${additional_annotation.additional_annotation_name}" + #end for #if str( $analysis_param_type.group ) != "None": #for $group in str( $analysis_param_type.group ).split( ','): --group "${group}" @@ -372,6 +375,9 @@ <filter type="static_value" value="UnifiedGenotyper" column="tools_valid_for"/></options></param> + <repeat name="additional_annotations" title="Additional annotation"> + <param name="additional_annotation_name" type="text" value="" label="Annotation name" /> + </repeat><!-- <conditional name="snpEff_rod_bind_type"><param name="snpEff_rod_bind_type_selector" type="select" label="Provide a snpEff reference-ordered data file"> diff -r bbda064ee4b6ffff2d867a389b81d884c772aa12 -r 0a901d00d8807a50017840e72fa37f2bf7cf2bcf tools/gatk/variant_annotator.xml --- a/tools/gatk/variant_annotator.xml +++ b/tools/gatk/variant_annotator.xml @@ -36,14 +36,9 @@ --excludeAnnotation "${annotation}" #end for #end if -## #for $additional_annotation in $additional_annotations: -## --annotation "${additional_annotation.additional_annotation_type.additional_annotation_type_selector}" -## #for $name, $param in $additional_annotation.additional_annotation_type.iteritems(): -## #if $name not in [ "__current_case__", "additional_annotation_type_selector" ]: -## --${name} "${param}" -## #end if -## #end for -## #end for + #for $additional_annotation in $additional_annotations: + --annotation "${additional_annotation.additional_annotation_name}" + #end for ${reference_source.input_variant_bti} ' @@ -188,6 +183,10 @@ </when></conditional> + <repeat name="additional_annotations" title="Additional annotation"> + <param name="additional_annotation_name" type="text" value="" label="Annotation name" /> + </repeat> + <repeat name="comp_rod_bind" title="Binding for reference-ordered comparison data"><param name="comp_input_rod" type="data" format="vcf" label="ROD file" /><param name="comp_rod_name" type="text" value="Unnamed" label="ROD Name"/> diff -r bbda064ee4b6ffff2d867a389b81d884c772aa12 -r 0a901d00d8807a50017840e72fa37f2bf7cf2bcf tools/gatk/variant_recalibrator.xml --- a/tools/gatk/variant_recalibrator.xml +++ b/tools/gatk/variant_recalibrator.xml @@ -104,6 +104,9 @@ --use_annotation "${annotation}" #end for #end if + #for $additional_annotation in $additional_annotations: + --use_annotation "${additional_annotation.additional_annotation_name}" + #end for --mode "${mode}" ' @@ -344,11 +347,15 @@ </options></param> - <param name="mode" type="select" label="Recalibration mode"> - <option value="SNP" selected="True">SNP</option> - <option value="INDEL">INDEL</option> - <option value="BOTH">BOTH</option> - </param> + <repeat name="additional_annotations" title="Additional annotation"> + <param name="additional_annotation_name" type="text" value="" label="Annotation name" /> + </repeat> + + <param name="mode" type="select" label="Recalibration mode"> + <option value="SNP" selected="True">SNP</option> + <option value="INDEL">INDEL</option> + <option value="BOTH">BOTH</option> + </param><conditional name="gatk_param_type"><param name="gatk_param_type_selector" type="select" label="Basic or Advanced GATK options"> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Add automatic conversions for SAM to BAM and SAM to summary tree. Enable SAM datasets to be visualized in Trackster.
by Bitbucket 06 Dec '11

06 Dec '11

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/bbda064ee4b6/ changeset: bbda064ee4b6 user: jgoecks date: 2011-12-06 15:34:19 summary: Add automatic conversions for SAM to BAM and SAM to summary tree. Enable SAM datasets to be visualized in Trackster. affected #: 8 files diff -r c28e9ef4036aca6a897db2aa93517577b0afddda -r bbda064ee4b6ffff2d867a389b81d884c772aa12 datatypes_conf.xml.sample --- a/datatypes_conf.xml.sample +++ b/datatypes_conf.xml.sample @@ -139,7 +139,10 @@ <datatype extension="qualsolid" type="galaxy.datatypes.qualityscore:QualityScoreSOLiD" display_in_upload="true"/><datatype extension="qual454" type="galaxy.datatypes.qualityscore:QualityScore454" display_in_upload="true"/><datatype extension="Roadmaps" type="galaxy.datatypes.assembly:Roadmaps" display_in_upload="false"/> - <datatype extension="sam" type="galaxy.datatypes.tabular:Sam" display_in_upload="true"/> + <datatype extension="sam" type="galaxy.datatypes.tabular:Sam" display_in_upload="true"> + <converter file="sam_to_bam.xml" target_datatype="bam"/> + <converter file="sam_to_summary_tree_converter.xml" target_datatype="summary_tree"/> + </datatype><datatype extension="scf" type="galaxy.datatypes.binary:Scf" mimetype="application/octet-stream" display_in_upload="true"/><datatype extension="Sequences" type="galaxy.datatypes.assembly:Sequences" display_in_upload="false"/><datatype extension="sff" type="galaxy.datatypes.binary:Sff" mimetype="application/octet-stream" display_in_upload="true"/> diff -r c28e9ef4036aca6a897db2aa93517577b0afddda -r bbda064ee4b6ffff2d867a389b81d884c772aa12 lib/galaxy/datatypes/converters/bam_to_summary_tree_converter.py --- a/lib/galaxy/datatypes/converters/bam_to_summary_tree_converter.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python - -from __future__ import division - -import sys, os -sys.stderr = open(os.devnull, 'w') # suppress stderr as cython produces warning on some systems: - # csamtools.so:6: RuntimeWarning: __builtin__.file size changed - -from galaxy import eggs -import pkg_resources - -if sys.version_info[:2] == (2, 4): - pkg_resources.require( "ctypes" ) -pkg_resources.require( "pysam" ) - -from pysam import csamtools -from galaxy.visualization.tracks.summary import * - -def main(): - - input_fname = sys.argv[1] - index_fname = sys.argv[2] - out_fname = sys.argv[3] - - bamfile = csamtools.Samfile( filename=input_fname, mode='rb', index_filename=index_fname ) - - st = SummaryTree(block_size=25, levels=6, draw_cutoff=150, detail_cutoff=30) - for read in bamfile.fetch(): - st.insert_range(bamfile.getrname(read.rname), read.pos, read.pos + read.rlen) - - st.write(out_fname) - -if __name__ == "__main__": - main() diff -r c28e9ef4036aca6a897db2aa93517577b0afddda -r bbda064ee4b6ffff2d867a389b81d884c772aa12 lib/galaxy/datatypes/converters/bam_to_summary_tree_converter.xml --- a/lib/galaxy/datatypes/converters/bam_to_summary_tree_converter.xml +++ b/lib/galaxy/datatypes/converters/bam_to_summary_tree_converter.xml @@ -1,6 +1,6 @@ <tool id="CONVERTER_bam_to_summary_tree_0" name="Convert BAM to Summary Tree" version="1.0.0" hidden="true"> - <command interpreter="python">bam_to_summary_tree_converter.py $input1 $bai $output1</command> + <command interpreter="python">sam_or_bam_to_summary_tree_converter.py --bam $input1 $bai $output1</command><inputs><page><param format="bam" name="input1" type="data" label="Choose BAM file"/> diff -r c28e9ef4036aca6a897db2aa93517577b0afddda -r bbda064ee4b6ffff2d867a389b81d884c772aa12 lib/galaxy/datatypes/converters/sam_or_bam_to_summary_tree_converter.py --- /dev/null +++ b/lib/galaxy/datatypes/converters/sam_or_bam_to_summary_tree_converter.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +from __future__ import division + +import sys, os, optparse +sys.stderr = open(os.devnull, 'w') # suppress stderr as cython produces warning on some systems: + # csamtools.so:6: RuntimeWarning: __builtin__.file size changed + +from galaxy import eggs +import pkg_resources + +if sys.version_info[:2] == (2, 4): + pkg_resources.require( "ctypes" ) +pkg_resources.require( "pysam" ) + +from pysam import csamtools +from galaxy.visualization.tracks.summary import * + +def main(): + parser = optparse.OptionParser() + parser.add_option( '-S', '--sam', action="store_true", dest="is_sam" ) + parser.add_option( '-B', '--bam', action="store_true", dest="is_bam" ) + options, args = parser.parse_args() + + if options.is_bam: + input_fname = args[0] + index_fname = args[1] + out_fname = args[2] + samfile = csamtools.Samfile( filename=input_fname, mode='rb', index_filename=index_fname ) + elif options.is_sam: + input_fname = args[0] + out_fname = args[1] + samfile = csamtools.Samfile( filename=input_fname, mode='r' ) + + st = SummaryTree(block_size=25, levels=6, draw_cutoff=150, detail_cutoff=30) + for read in samfile.fetch(): + st.insert_range( samfile.getrname( read.rname ), read.pos, read.pos + read.rlen ) + + st.write(out_fname) + +if __name__ == "__main__": + main() diff -r c28e9ef4036aca6a897db2aa93517577b0afddda -r bbda064ee4b6ffff2d867a389b81d884c772aa12 lib/galaxy/datatypes/converters/sam_to_bam.xml --- /dev/null +++ b/lib/galaxy/datatypes/converters/sam_to_bam.xml @@ -0,0 +1,17 @@ +<tool id="CONVERTER_sam_to_bam" name="Convert SAM to BAM" version="1.0.0"> +  +  +  + <command>samtools view -bS $input1 > $output 2> /dev/null </command> + <inputs> + <param name="input1" type="data" format="sam" label="SAM file"/> + </inputs> + <outputs> + <data name="output" format="bam"/> + </outputs> + <help> + </help> +</tool> diff -r c28e9ef4036aca6a897db2aa93517577b0afddda -r bbda064ee4b6ffff2d867a389b81d884c772aa12 lib/galaxy/datatypes/converters/sam_to_summary_tree_converter.xml --- /dev/null +++ b/lib/galaxy/datatypes/converters/sam_to_summary_tree_converter.xml @@ -0,0 +1,14 @@ +<tool id="CONVERTER_sam_to_summary_tree_0" name="Convert SAM to Summary Tree" version="1.0.0" hidden="true"> + + <command interpreter="python">sam_or_bam_to_summary_tree_converter.py --sam $input1 $output1</command> + <inputs> + <page> + <param format="sam" name="input1" type="data" label="Choose sam file"/> + </page> + </inputs> + <outputs> + <data format="summary_tree" name="output1"/> + </outputs> + <help> + </help> +</tool> diff -r c28e9ef4036aca6a897db2aa93517577b0afddda -r bbda064ee4b6ffff2d867a389b81d884c772aa12 lib/galaxy/datatypes/tabular.py --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -368,7 +368,10 @@ if result != 0: raise Exception('Result %s from %s' % (result, cmd)) merge = staticmethod(merge) - + + def get_track_type( self ): + return "ReadTrack", {"data": "bam", "index": "summary_tree"} + class Pileup( Tabular ): """Tab delimited data in pileup (6- or 10-column) format""" file_ext = "pileup" diff -r c28e9ef4036aca6a897db2aa93517577b0afddda -r bbda064ee4b6ffff2d867a389b81d884c772aa12 lib/galaxy/visualization/tracks/data_providers.py --- a/lib/galaxy/visualization/tracks/data_providers.py +++ b/lib/galaxy/visualization/tracks/data_providers.py @@ -725,6 +725,18 @@ max_low, max_high = get_bounds( results, 1, 2 ) return { 'data': results, 'message': message, 'max_low': max_low, 'max_high': max_high } + +class SamDataProvider( BamDataProvider ): + + def __init__( self, converted_dataset=None, original_dataset=None, dependencies=None ): + """ Create SamDataProvider. """ + + # HACK: to use BamDataProvider, original dataset must be BAM and + # converted dataset must be BAI. Use BAI from BAM metadata. + if converted_dataset: + self.converted_dataset = converted_dataset.metadata.bam_index + self.original_dataset = converted_dataset + self.dependencies = dependencies class BBIDataProvider( TracksDataProvider ): """ @@ -957,6 +969,7 @@ "tabix": { Vcf: VcfTabixDataProvider, Bed: BedTabixDataProvider, "default" : TabixDataProvider }, "interval_index": IntervalIndexDataProvider, "bai": BamDataProvider, + "bam": SamDataProvider, "summary_tree": SummaryTreeDataProvider, "bigwig": BigWigDataProvider, "bigbed": BigBedDataProvider Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by Bitbucket 06 Dec '11

06 Dec '11

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/87268f2b439d/ changeset: 87268f2b439d user: dannon date: 2011-12-06 15:15:43 summary: Fix bad merge. affected #: 1 file diff -r 8993f7342095d3fac1feb3458ecee909ba678e11 -r 87268f2b439d844df43ce0065ec01e130c8b1a09 lib/galaxy/datatypes/tabular.py --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -353,7 +353,7 @@ dataset.metadata.columns = 12 dataset.metadata.column_types = ['str', 'int', 'str', 'int', 'int', 'str', 'str', 'int', 'int', 'str', 'str', 'str'] - def merge( split_files, output_file): + def merge( split_files, output_file): """ Multiple SAM files may each have headers. Since the headers should all be the same, remove the headers from files 1-n, keeping them in the first file only https://bitbucket.org/galaxy/galaxy-central/changeset/c28e9ef4036a/ changeset: c28e9ef4036a user: dannon date: 2011-12-06 15:15:57 summary: Merge. affected #: 1 file diff -r 87268f2b439d844df43ce0065ec01e130c8b1a09 -r c28e9ef4036aca6a897db2aa93517577b0afddda test/functional/test_library_templates.py --- a/test/functional/test_library_templates.py +++ b/test/functional/test_library_templates.py @@ -403,7 +403,7 @@ assert ldda is not None, 'Problem retrieving LibraryDatasetDatasetAssociation ldda from the database' self.browse_library( cntrller='library_admin', library_id=self.security.encode_id( library3.id ), - strings_displayed=[ ldda.name, admin_user.email ] ) + strings_displayed=[ ldda.name, 'bed' ] ) # Make sure the library template contents were correctly saved self.ldda_edit_info( 'library_admin', self.security.encode_id( library3.id ), Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix for data library functional test.
by Bitbucket 06 Dec '11

06 Dec '11

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/9eb66dabc6ee/ changeset: 9eb66dabc6ee user: greg date: 2011-12-06 14:56:21 summary: Fix for data library functional test. affected #: 1 file diff -r 8993f7342095d3fac1feb3458ecee909ba678e11 -r 9eb66dabc6ee75a924632dc8140daab470a86f9d test/functional/test_library_templates.py --- a/test/functional/test_library_templates.py +++ b/test/functional/test_library_templates.py @@ -403,7 +403,7 @@ assert ldda is not None, 'Problem retrieving LibraryDatasetDatasetAssociation ldda from the database' self.browse_library( cntrller='library_admin', library_id=self.security.encode_id( library3.id ), - strings_displayed=[ ldda.name, admin_user.email ] ) + strings_displayed=[ ldda.name, 'bed' ] ) # Make sure the library template contents were correctly saved self.ldda_edit_info( 'library_admin', self.security.encode_id( library3.id ), Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by Bitbucket 06 Dec '11

06 Dec '11

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/af58b5faffec/ changeset: af58b5faffec user: John Duddy date: 2011-11-29 00:37:37 summary: Add merge method to SAM datatype so BWA can do split/merge affected #: 1 file diff -r 5db0da0007fcc59bb23f0e0ae2edb1e961702f14 -r af58b5faffec3b6715850e6d50b7bef69a5bbef5 lib/galaxy/datatypes/tabular.py --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -352,6 +352,22 @@ dataset.metadata.comment_lines = comment_lines dataset.metadata.columns = 12 dataset.metadata.column_types = ['str', 'int', 'str', 'int', 'int', 'str', 'str', 'int', 'int', 'str', 'str', 'str'] +355,369d354 + def merge( split_files, output_file): + """ + Multiple SAM files may each have headers. Since the headers should all be the same, remove + the headers from files 1-n, keeping them in the first file only + """ + cmd = 'mv %s %s' % ( split_files[0], output_file ) + result = os.system(cmd) + if result != 0: + raise Exception('Result %s from %s' % (result, cmd)) + if len(split_files) > 1: + cmd = 'egrep -v "^@" %s >> %s' % ( ' '.join(split_files[1:]), output_file ) + result = os.system(cmd) + if result != 0: + raise Exception('Result %s from %s' % (result, cmd)) + merge = staticmethod(merge) class Pileup( Tabular ): """Tab delimited data in pileup (6- or 10-column) format""" https://bitbucket.org/galaxy/galaxy-central/changeset/8993f7342095/ changeset: 8993f7342095 user: dannon date: 2011-12-06 14:43:32 summary: Pull in jdlogicman's SAM merge. affected #: 1 file diff -r 43326a266e8eb5818c695c235df82a025a3b6893 -r 8993f7342095d3fac1feb3458ecee909ba678e11 lib/galaxy/datatypes/tabular.py --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -353,6 +353,22 @@ dataset.metadata.columns = 12 dataset.metadata.column_types = ['str', 'int', 'str', 'int', 'int', 'str', 'str', 'int', 'int', 'str', 'str', 'str'] + def merge( split_files, output_file): + """ + Multiple SAM files may each have headers. Since the headers should all be the same, remove + the headers from files 1-n, keeping them in the first file only + """ + cmd = 'mv %s %s' % ( split_files[0], output_file ) + result = os.system(cmd) + if result != 0: + raise Exception('Result %s from %s' % (result, cmd)) + if len(split_files) > 1: + cmd = 'egrep -v "^@" %s >> %s' % ( ' '.join(split_files[1:]), output_file ) + result = os.system(cmd) + if result != 0: + raise Exception('Result %s from %s' % (result, cmd)) + merge = staticmethod(merge) + class Pileup( Tabular ): """Tab delimited data in pileup (6- or 10-column) format""" file_ext = "pileup" Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by Bitbucket 06 Dec '11

06 Dec '11

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/6ce661b68417/ changeset: 6ce661b68417 user: dannon date: 2011-12-06 14:01:06 summary: Small patch to sanitizer to remove javascript content from sanitized anchors. affected #: 1 file diff -r 35fee32991ce5463d79ce278cb3a9c1155e8982f -r 6ce661b68417f81727c0f23aa8fb5d2769a8c502 lib/galaxy/util/sanitize_html.py --- a/lib/galaxy/util/sanitize_html.py +++ b/lib/galaxy/util/sanitize_html.py @@ -373,7 +373,9 @@ clean_attrs = [] for key, value in self.normalize_attrs(attrs): - if key in acceptable_attributes: + if key=="href" and value.startswith("javascript"): + pass + elif key in acceptable_attributes: key=keymap.get(key,key) clean_attrs.append((key,value)) elif key=='style': @@ -436,4 +438,4 @@ p.feed(htmlSource) data = p.output() data = data.strip().replace('\r\n', '\n') - return data \ No newline at end of file + return data https://bitbucket.org/galaxy/galaxy-central/changeset/43326a266e8e/ changeset: 43326a266e8e user: dannon date: 2011-12-06 14:08:16 summary: Small tweak on patch, strip first to prevent whitespace loophole. affected #: 1 file diff -r 6ce661b68417f81727c0f23aa8fb5d2769a8c502 -r 43326a266e8eb5818c695c235df82a025a3b6893 lib/galaxy/util/sanitize_html.py --- a/lib/galaxy/util/sanitize_html.py +++ b/lib/galaxy/util/sanitize_html.py @@ -373,7 +373,7 @@ clean_attrs = [] for key, value in self.normalize_attrs(attrs): - if key=="href" and value.startswith("javascript"): + if key=="href" and value.strip().startswith("javascript"): pass elif key in acceptable_attributes: key=keymap.get(key,key) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0