- galaxy-dev - lists.galaxyproject.org

[hg] galaxy 2444: Fixes for test_get_data functional test script.
by Greg Von Kuster 11 Jun '09

11 Jun '09

details: http://www.bx.psu.edu/hg/galaxy/rev/004cd81cff72 changeset: 2444:004cd81cff72 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Wed Jun 10 15:46:26 2009 -0400 description: Fixes for test_get_data functional test script. 1 file(s) affected in this change: test/functional/test_get_data.py diffs (152 lines): diff -r 08194c60c290 -r 004cd81cff72 test/functional/test_get_data.py --- a/test/functional/test_get_data.py Wed Jun 10 12:22:49 2009 -0400 +++ b/test/functional/test_get_data.py Wed Jun 10 15:46:26 2009 -0400 @@ -1,87 +1,74 @@ +import galaxy.model +from galaxy.model.orm import * from base.twilltestcase import TwillTestCase -""" Tests are executed in order, sorted by name""" - class UploadData( TwillTestCase ): - def test_00_multi_upload( self ): - """test_get_data.test_multi_upload: Testing multiple uploads""" - self.login() - self.upload_file('1.bed') - self.verify_dataset_correctness('1.bed') - self.upload_file('2.bed', dbkey='hg17') - self.verify_dataset_correctness('2.bed') - self.upload_file('3.bed', dbkey='hg17', ftype='bed') - self.verify_dataset_correctness('3.bed') - self.upload_file('4.bed.gz', dbkey='hg17', ftype='bed') - self.verify_dataset_correctness('4.bed') - self.upload_file('1.scf', ftype='scf') - self.verify_dataset_correctness('1.scf') - self.upload_file('1.scf.zip', ftype='binseq.zip') - self.verify_dataset_correctness('1.scf.zip') + def test_000_upload_files_from_disk( self ): + """Test uploading data files from disk""" + self.logout() + self.login( email='tst(a)bx.psu.edu' ) + history1 = galaxy.model.History.query().order_by( desc( galaxy.model.History.table.c.create_time ) ).first() + self.upload_file( '1.bed' ) + hda1 = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert hda1 is not None, "Problem retrieving hda1 from database" + self.verify_dataset_correctness( '1.bed', hid=str( hda1.hid ) ) + self.upload_file( '2.bed', dbkey='hg17' ) + hda2 = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert hda2 is not None, "Problem retrieving hda2 from database" + self.verify_dataset_correctness( '2.bed', hid=str( hda2.hid ) ) + self.upload_file( '3.bed', dbkey='hg17', ftype='bed' ) + hda3 = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert hda3 is not None, "Problem retrieving hda3 from database" + self.verify_dataset_correctness( '3.bed', hid=str( hda3.hid ) ) + self.upload_file( '4.bed.gz', dbkey='hg17', ftype='bed' ) + hda4 = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert hda4 is not None, "Problem retrieving hda4 from database" + self.verify_dataset_correctness( '4.bed', hid=str( hda4.hid ) ) + self.upload_file( '1.scf', ftype='scf' ) + hda5 = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert hda5 is not None, "Problem retrieving hda5 from database" + self.verify_dataset_correctness( '1.scf', hid=str( hda5.hid ) ) + self.upload_file( '1.scf.zip', ftype='binseq.zip' ) + hda6 = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert hda6 is not None, "Problem retrieving hda6 from database" + self.verify_dataset_correctness( '1.scf.zip', hid=str( hda6.hid ) ) + self.delete_history( id=str( history1.id ) ) + def test_005_url_paste( self ): + """Test url paste behavior""" + # Deleting the current history should have created a new history + self.check_history_for_string( 'Your history is empty' ) + history2 = galaxy.model.History.query().order_by( desc( galaxy.model.History.table.c.create_time ) ).first() self.upload_url_paste( 'hello world' ) self.check_history_for_string( 'Pasted Entry' ) self.check_history_for_string( 'hello world' ) - self.delete_history_item( 1 ) - self.delete_history_item( 2 ) - self.delete_history_item( 3 ) - self.delete_history_item( 4 ) - self.delete_history_item( 5 ) - self.delete_history_item( 6 ) - self.delete_history_item( 7 ) self.upload_url_paste( u'hello world' ) self.check_history_for_string( 'Pasted Entry' ) self.check_history_for_string( 'hello world' ) - self.delete_history_item( 8 ) - def test_9999_clean_up( self ): - self.delete_history() - self.logout() - -class GetEncodeData( TwillTestCase ): - - def test_00_get_encode_data( self ): - """test_get_data.test_get_encode_data""" - self.login() - self.run_tool('encode_import_chromatin_and_chromosomes1', hg17=['cc.EarlyRepSeg.20051216.bed'] ) - #hg17=[ "cc.EarlyRepSeg.20051216.bed", "cc.EarlyRepSeg.20051216.gencode_partitioned.bed", "cc.LateRepSeg.20051216.bed", "cc.LateRepSeg.20051216.gencode_partitioned.bed", "cc.MidRepSeg.20051216.bed", "cc.MidRepSeg.20051216.gencode_partitioned.bed" ] ) + self.delete_history( id=str( history2.id ) ) + def test_010_upload_encode_data( self ): + """Test uploading encode data""" + # Deleting the current history should have created a new history + self.check_history_for_string( 'Your history is empty' ) + history3 = galaxy.model.History.query().order_by( desc( galaxy.model.History.table.c.create_time ) ).first() + self.run_tool( 'encode_import_chromatin_and_chromosomes1', hg17=['cc.EarlyRepSeg.20051216.bed'] ) self.wait() - self.verify_dataset_correctness('cc.EarlyRepSeg.20051216.bed', hid=1) - #self.verify_dataset_correctness('cc.EarlyRepSeg.20051216.gencode_partitioned.bed', hid=2) - #self.verify_dataset_correctness('cc.LateRepSeg.20051216.bed', hid=3) - #self.verify_dataset_correctness('cc.LateRepSeg.20051216.gencode_partitioned.bed', hid=4) - #self.verify_dataset_correctness('cc.MidRepSeg.20051216.bed', hid=5) - #self.verify_dataset_correctness('cc.MidRepSeg.20051216.gencode_partitioned.bed', hid=6) + hda7 = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert hda7 is not None, "Problem retrieving hda7 from database" + self.verify_dataset_correctness( 'cc.EarlyRepSeg.20051216.bed', hid=str( hda7.hid ) ) self.run_tool('encode_import_gencode1', hg17=['gencode.CDS.20051206.bed']) self.wait() - self.verify_dataset_correctness('sc_3D_cds.bed', hid=2) - self.delete_history_item( 1 ) - self.delete_history_item( 2 ) - def test_9999_clean_up( self ): - self.delete_history() + hda8 = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert hda8 is not None, "Problem retrieving hda8 from database" + self.verify_dataset_correctness( 'sc_3D_cds.bed', hid=str( hda8.hid ) ) + self.delete_history( id=str( history3.id ) ) + def test_015_reset_data_for_later_test_runs( self ): + """Reseting data to enable later test runs to pass""" self.logout() - -class DataSources( TwillTestCase ): - - #def test_hbvar(self): - # """Getting hybrid gene mutations from HbVar""" - # #self.load_cookies("hbvar_cookie.txt") - # self.clear_history() - # self.run_tool('hbvar') - # params = dict( - # htyp="any hybrid gene", - # ) - # self.submit_form(form=1, button="Submit Query", **params) - # params = dict( - # display_format="galaxy", - # ) - # self.submit_form(form=1, button="Go", **params) - # params = dict( - # build="hg17", - # ) - # self.submit_form(form=1, button="ok", **params); - # """ - # TODO: Currently fails when using sqlite, although successful when - # using Postgres. Upgrading our version of sqlite may fix this, but - # confirmation is required. - # """ - # self.verify_dataset_correctness('hbvar_hybrid_genes.dat') - pass

1 0

Round-Robin Scheduling
by Assaf Gordon 10 Jun '09

10 Jun '09

Hello, I have another question regarding the local job scheduler: Is it possible to limit the number of jobs *per user* ? That is - any given user can have up to X number of jobs running concurrently, regardless of the value of local_job_queue_workers ? Imagine the following situation: local_job_queue_worker = 5 job_scheduler_policy = galaxy.jobs.schedulingpolicy.roundrobin:UserRoundRobin Which means that at any given moment, galaxy can run only five jobs. Now, Galaxy is completely Idle, no jobs are running. One users starts 7 very long running jobs (each jobs will take about two hours). If I understand correctly - since no jobs are running, 5 of the user's job will be started immediately, even with the round-robin policy, right ? And this means that for the next two hours, every other user which starts a job - his/her job will be either new or limbo-running, but none will actually be started, right ? I think I'm experiencing this situation on my galaxy server. Users are complaining their jobs have been running 'forever' or not even starting for a long long time. Close examination shows that there are running 5 jobs (all from the same user) which have been running for three hours, and they are kind of hogging all the worker threads. To make a long story short - I would like to make sure a single user can't hog Galaxy. Is it possible with the local job runner, and if not - is it possible with the SGE job runner ? Thanks for reading so far, Gordon.

2 1

Wrappers for Sequence Alignment Programs
by SHAUN WEBB 10 Jun '09

10 Jun '09

Hi, I am planning on integrating sequence alignment programs to our local instance of Galaxy. I was wondering if anyone else has developed wrappers for the following programs: Maq, Bowtie, Tophat. Also, is there a central repository for any new tools or wrappers that have been created by other developers? It would certainly be a useful resource and if I need to wrap these tools myself I would be happy share the code. Shaun Webb -- The University of Edinburgh is a charitable body, registered in Scotland, with registration number SC005336.

2 1

[hg] galaxy 2443: Remote user bug found and fixed by Morita Hide...
by Nate Coraor 10 Jun '09

10 Jun '09

details: http://www.bx.psu.edu/hg/galaxy/rev/08194c60c290 changeset: 2443:08194c60c290 user: Nate Coraor <nate(a)bx.psu.edu> date: Wed Jun 10 12:22:49 2009 -0400 description: Remote user bug found and fixed by Morita Hideyuki 1 file(s) affected in this change: lib/galaxy/web/framework/__init__.py diffs (12 lines): diff -r 46bbe610288a -r 08194c60c290 lib/galaxy/web/framework/__init__.py --- a/lib/galaxy/web/framework/__init__.py Wed Jun 10 11:58:21 2009 -0400 +++ b/lib/galaxy/web/framework/__init__.py Wed Jun 10 12:22:49 2009 -0400 @@ -268,7 +268,7 @@ invalidate_existing_session = True user_for_new_session = self.__get_or_create_remote_user( remote_user_email ) log.warning( "User logged in as '%s' externally, but has a cookie as '%s' invalidating session", - remote_user_email, prev_galaxy_session.user.email ) + remote_user_email, galaxy_session.user.email ) else: # No session exists, get/create user for new session user_for_new_session = self.__get_or_create_remote_user( remote_user_email )

1 0

[hg] galaxy 2442: Fix missing split/join char in Join tool when ...
by Nate Coraor 10 Jun '09

10 Jun '09

details: http://www.bx.psu.edu/hg/galaxy/rev/46bbe610288a changeset: 2442:46bbe610288a user: Dan Blankenberg <dan(a)bx.psu.edu> date: Wed Jun 10 11:58:21 2009 -0400 description: Fix missing split/join char in Join tool when filling nonjoined columns. 1 file(s) affected in this change: tools/filters/join.py diffs (20 lines): diff -r c0aa8af62124 -r 46bbe610288a tools/filters/join.py --- a/tools/filters/join.py Tue Jun 09 15:03:04 2009 -0400 +++ b/tools/filters/join.py Wed Jun 10 11:58:21 2009 -0400 @@ -288,12 +288,14 @@ if not written and keep_unmatched: out.write( fill_empty_columns( line1.rstrip( '\r\n' ), split, fill_options.file1_columns ) ) if fill_options: - out.write( fill_empty_columns( "", split, fill_options.file2_columns ) ) + if fill_options.file2_columns: + out.write( "%s%s" % ( split, fill_empty_columns( "", split, fill_options.file2_columns ) ) ) out.write( "\n" ) elif keep_partial: out.write( fill_empty_columns( line1.rstrip( '\r\n' ), split, fill_options.file1_columns ) ) if fill_options: - out.write( fill_empty_columns( "", split, fill_options.file2_columns ) ) + if fill_options.file2_columns: + out.write( "%s%s" % ( split, fill_empty_columns( "", split, fill_options.file2_columns ) ) ) out.write( "\n" ) out.close()

1 0

[hg] galaxy 2440: bug fix for non ajax upload
by Greg Von Kuster 09 Jun '09

09 Jun '09

details: http://www.bx.psu.edu/hg/galaxy/rev/6a992f466b80 changeset: 2440:6a992f466b80 user: Dan Blankenberg <dan(a)bx.psu.edu> date: Tue Jun 09 12:08:35 2009 -0400 description: bug fix for non ajax upload 1 file(s) affected in this change: lib/galaxy/tools/actions/upload.py diffs (12 lines): diff -r 6d00e4ff7129 -r 6a992f466b80 lib/galaxy/tools/actions/upload.py --- a/lib/galaxy/tools/actions/upload.py Tue Jun 09 12:05:46 2009 -0400 +++ b/lib/galaxy/tools/actions/upload.py Tue Jun 09 12:08:35 2009 -0400 @@ -137,7 +137,7 @@ if precreated_dataset is not None: data = precreated_dataset else: - data = trans.app.model.HistoryDatasetAssociation( history = trans.history, extension = ext, create_dataset = True ) + data = trans.app.model.HistoryDatasetAssociation( history = trans.history, create_dataset = True ) trans.app.security_agent.set_all_dataset_permissions( data.dataset, trans.app.security_agent.history_get_default_permissions( trans.history ) ) # See if we have an empty file

1 0

[hg] galaxy 2441: Only refresh history if necessary ( better fix...
by Greg Von Kuster 09 Jun '09

09 Jun '09

details: http://www.bx.psu.edu/hg/galaxy/rev/c0aa8af62124 changeset: 2441:c0aa8af62124 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Tue Jun 09 15:03:04 2009 -0400 description: Only refresh history if necessary ( better fix than my previous commit ), add functional tests to cover behavior when deleing current history, and add better functional tests for sniffing data formats. 3 file(s) affected in this change: lib/galaxy/web/controllers/history.py test/functional/test_history_functions.py test/functional/test_sniffing_and_metadata_settings.py diffs (288 lines): diff -r 6a992f466b80 -r c0aa8af62124 lib/galaxy/web/controllers/history.py --- a/lib/galaxy/web/controllers/history.py Tue Jun 09 12:08:35 2009 -0400 +++ b/lib/galaxy/web/controllers/history.py Tue Jun 09 15:03:04 2009 -0400 @@ -79,6 +79,7 @@ @web.require_login( "work with multiple histories" ) def list( self, trans, **kwargs ): """List all available histories""" + current_history = trans.history status = message = None if 'operation' in kwargs: operation = kwargs['operation'].lower() @@ -104,12 +105,14 @@ if histories: if operation == "switch": status, message = self._list_switch( trans, histories ) + # Current history changed, refresh history frame + trans.template_context['refresh_frames'] = ['history'] elif operation == "delete": status, message = self._list_delete( trans, histories ) + if current_history in histories: + trans.template_context['refresh_frames'] = ['history'] elif operation == "undelete": status, message = self._list_undelete( trans, histories ) - # Current history may have changed, refresh history frame - trans.template_context['refresh_frames'] = ['history'] trans.sa_session.flush() # Render the list view return self.list_grid( trans, status=status, message=message, **kwargs ) diff -r 6a992f466b80 -r c0aa8af62124 test/functional/test_history_functions.py --- a/test/functional/test_history_functions.py Tue Jun 09 12:08:35 2009 -0400 +++ b/test/functional/test_history_functions.py Tue Jun 09 15:03:04 2009 -0400 @@ -47,6 +47,8 @@ latest_history.refresh() if not latest_history.deleted: raise AssertionError, "Problem deleting history id %d" % latest_history.id + # Since we deleted the current history, make sure the history frame was refreshed + self.check_history_for_string( 'Your history is empty.' ) # We'll now test deleting a list of histories # After deleting the current history, a new one should have been created global history1 @@ -60,6 +62,8 @@ self.upload_file( '2.bed', dbkey='hg18' ) ids = '%s,%s' % ( str( history1.id ), str( history2.id ) ) self.delete_history( ids ) + # Since we deleted the current history, make sure the history frame was refreshed + self.check_history_for_string( 'Your history is empty.' ) try: self.view_stored_active_histories( check_str=history1.name ) raise AssertionError, "History %s is displayed in the active history list after it was deleted" % history1.name diff -r 6a992f466b80 -r c0aa8af62124 test/functional/test_sniffing_and_metadata_settings.py --- a/test/functional/test_sniffing_and_metadata_settings.py Tue Jun 09 12:08:35 2009 -0400 +++ b/test/functional/test_sniffing_and_metadata_settings.py Tue Jun 09 15:03:04 2009 -0400 @@ -3,7 +3,7 @@ from base.twilltestcase import TwillTestCase class SniffingAndMetaDataSettings( TwillTestCase ): - def test_00_axt_datatype( self ): + def test_000_axt_datatype( self ): """Testing correctly sniffing axt data type upon upload""" self.logout() self.login( email='test(a)bx.psu.edu' ) @@ -15,7 +15,12 @@ self.verify_dataset_correctness( '1.axt' ) self.check_history_for_string( '1.axt format: axt, database: \? Info: uploaded file' ) self.check_metadata_for_string( 'value="1.axt" value="\?" Change data type selected value="axt" selected="yes"' ) - def test_05_bed_datatype( self ): + latest_hda = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert latest_hda is not None, "Problem retrieving axt hda from the database" + if not latest_hda.name == '1.axt' and not latest_hda.extension == 'axt': + raise AssertionError, "axt data type was not correctly sniffed." + def test_005_bed_datatype( self ): """Testing correctly sniffing bed data type upon upload""" self.upload_file( '1.bed' ) self.verify_dataset_correctness( '1.bed' ) @@ -25,24 +30,64 @@ self.check_metadata_for_string( 'End column: <option value="3" selected> Strand column <option value="6" selected>' ) self.check_metadata_for_string( 'Convert to new format <option value="bed">Genomic Intervals To BED <option value="gff">BED to GFF' ) self.check_metadata_for_string( 'Change data type selected value="bed" selected="yes"' ) - def test_10_customtrack_datatype( self ): + latest_hda = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert latest_hda is not None, "Problem retrieving bed hda from the database" + if not latest_hda.name == '1.bed' and not latest_hda.extension == 'bed': + raise AssertionError, "bed data type was not correctly sniffed." + def test_010_blastxml_datatype( self ): + """Testing correctly sniffing blastxml data type upon upload""" + self.upload_file( 'megablast_xml_parser_test1.gz' ) + self.check_history_for_string( 'NCBI Blast XML data format: blastxml' ) + latest_hda = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert latest_hda is not None, "Problem retrieving blastxml hda from the database" + if not latest_hda.name == 'megablast_xml_parser_test1' and not latest_hda.extension == 'blastxml': + raise AssertionError, "blastxml data type was not correctly sniffed." + def test_015_csfasta_datatype( self ): + """Testing correctly sniffing csfasta data type upon upload""" + self.upload_file( 'shrimp_cs_test1.csfasta' ) + self.verify_dataset_correctness( 'shrimp_cs_test1.csfasta' ) + self.check_history_for_string( '162.6 Kb, format: csfasta, <td>>2_14_26_F3,-1282216.0</td>' ) + self.check_metadata_for_string( 'value="shrimp_cs_test1.csfasta" value="\?" Change data type value="csfasta" selected="yes"' ) + latest_hda = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert latest_hda is not None, "Problem retrieving csfasta hda from the database" + if not latest_hda.name == 'shrimp_cs_test1.csfasta' and not latest_hda.extension == 'csfasta': + raise AssertionError, "csfasta data type was not correctly sniffed." + def test_020_customtrack_datatype( self ): """Testing correctly sniffing customtrack data type upon upload""" self.upload_file( '1.customtrack' ) self.verify_dataset_correctness( '1.customtrack' ) self.check_history_for_string( '1.customtrack format: customtrack, database: \? Info: uploaded file' ) self.check_metadata_for_string( 'value="1.customtrack" value="\?" Change data type selected value="customtrack" selected="yes"' ) - def test_15_fasta_datatype( self ): + latest_hda = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert latest_hda is not None, "Problem retrieving customtrack hda from the database" + if not latest_hda.name == '1.customtrack' and not latest_hda.extension == 'customtrack': + raise AssertionError, "customtrack data type was not correctly sniffed." + def test_025_fasta_datatype( self ): """Testing correctly sniffing fasta data type upon upload""" self.upload_file( '1.fasta' ) self.verify_dataset_correctness( '1.fasta' ) self.check_history_for_string( '1.fasta format: fasta, database: \? Info: uploaded file' ) self.check_metadata_for_string( 'value="1.fasta" value="\?" Change data type selected value="fasta" selected="yes"' ) - def test_18_fastqsolexa_datatype( self ): + latest_hda = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert latest_hda is not None, "Problem retrieving fasta hda from the database" + if not latest_hda.name == '1.fasta' and not latest_hda.extension == 'fasta': + raise AssertionError, "fasta data type was not correctly sniffed." + def test_030_fastqsolexa_datatype( self ): """Testing correctly sniffing fastqsolexa ( the Solexa variant ) data type upon upload""" self.upload_file( '1.fastqsolexa' ) self.verify_dataset_correctness( '1.fastqsolexa' ) self.check_history_for_string( '1.fastqsolexa format: fastqsolexa, database: \? Info: uploaded fastqsolexa file' ) - def test_20_gff_datatype( self ): + latest_hda = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert latest_hda is not None, "Problem retrieving fastqsolexa hda from the database" + if not latest_hda.name == '1.fastqsolexa' and not latest_hda.extension == 'fastqsolexa': + raise AssertionError, "fastqsolexa data type was not correctly sniffed." + def test_035_gff_datatype( self ): """Testing correctly sniffing gff data type upon upload""" self.upload_file( '5.gff' ) self.verify_dataset_correctness( '5.gff' ) @@ -50,7 +95,12 @@ self.check_metadata_for_string( 'value="5.gff" value="\?"' ) self.check_metadata_for_string( 'Convert to new format <option value="bed">GFF to BED' ) self.check_metadata_for_string( 'Change data type selected value="gff" selected="yes"' ) - def test_25_gff3_datatype( self ): + latest_hda = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert latest_hda is not None, "Problem retrieving gff hda from the database" + if not latest_hda.name == '5.gff' and not latest_hda.extension == 'gff': + raise AssertionError, "gff data type was not correctly sniffed." + def test_040_gff3_datatype( self ): """Testing correctly sniffing gff3 data type upon upload""" self.upload_file( '5.gff3' ) self.verify_dataset_correctness( '5.gff3' ) @@ -58,7 +108,21 @@ self.check_metadata_for_string( 'value="5.gff3" value="\?"' ) self.check_metadata_for_string( 'Convert to new format <option value="bed">GFF to BED' ) self.check_metadata_for_string( 'Change data type selected value="gff3" selected="yes"' ) - def test_30_interval_datatype( self ): + latest_hda = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert latest_hda is not None, "Problem retrieving gff3 hda from the database" + if not latest_hda.name == '5.gff3' and not latest_hda.extension == 'gff3': + raise AssertionError, "gff3 data type was not correctly sniffed." + def test_045_html_datatype( self ): + """Testing correctly sniffing html data type upon upload""" + self.upload_file( 'html_file.txt' ) + self.check_history_for_string( 'An error occurred running this job: No data: you attempted to upload an inappropriate file.' ) + latest_hda = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert latest_hda is not None, "Problem retrieving html hda from the database" + if not latest_hda.name == 'html_file.txt' and not latest_hda.extension == 'data': + raise AssertionError, "html data type was not correctly sniffed." + def test_050_interval_datatype( self ): """Testing correctly sniffing interval data type upon upload""" self.upload_file( '1.interval' ) self.verify_dataset_correctness( '1.interval' ) @@ -68,14 +132,24 @@ self.check_metadata_for_string( 'End column: <option value="3" selected> Strand column <option value="6" selected>' ) self.check_metadata_for_string( 'Convert to new format <option value="bed">Genomic Intervals To BED' ) self.check_metadata_for_string( 'Change data type selected value="interval" selected="yes"' ) - def test_35_lav_datatype( self ): + latest_hda = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert latest_hda is not None, "Problem retrieving interval hda from the database" + if not latest_hda.name == '1.interval' and not latest_hda.extension == 'interval': + raise AssertionError, "interval data type was not correctly sniffed." + def test_055_lav_datatype( self ): """Testing correctly sniffing lav data type upon upload""" self.upload_file( '1.lav' ) self.verify_dataset_correctness( '1.lav' ) self.check_history_for_string( '1.lav format: lav, database: \? Info: uploaded file' ) self.check_metadata_for_string( 'value="1.lav" value="\?"' ) self.check_metadata_for_string( 'Change data type selected value="lav" selected="yes"' ) - def test_40_maf_datatype( self ): + latest_hda = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert latest_hda is not None, "Problem retrieving lav hda from the database" + if not latest_hda.name == '1.lav' and not latest_hda.extension == 'lav': + raise AssertionError, "lav data type was not correctly sniffed." + def test_060_maf_datatype( self ): """Testing correctly sniffing maf data type upon upload""" self.upload_file( '3.maf' ) self.verify_dataset_correctness( '3.maf' ) @@ -83,40 +157,57 @@ self.check_metadata_for_string( 'value="3.maf" value="\?"' ) self.check_metadata_for_string( 'Convert to new format <option value="interval">MAF to Genomic Intervals <option value="fasta">MAF to Fasta' ) self.check_metadata_for_string( 'Change data type selected value="maf" selected="yes"' ) - def test_45_tabular_datatype( self ): + latest_hda = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert latest_hda is not None, "Problem retrieving maf hda from the database" + if not latest_hda.name == '3.maf' and not latest_hda.extension == 'maf': + raise AssertionError, "maf data type was not correctly sniffed." + def test_065_qual454_datatype( self ): + """Testing correctly sniffing qual454 data type upon upload""" + self.upload_file( 'qualscores.qual454' ) + self.verify_dataset_correctness( 'qualscores.qual454' ) + self.check_history_for_string( '5.6 Kb, format: qual454, database: \?' ) + self.check_metadata_for_string( 'Change data type value="qual454" selected="yes">qual454' ) + latest_hda = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert latest_hda is not None, "Problem retrieving qual454 hda from the database" + if not latest_hda.name == 'qualscores.qual454' and not latest_hda.extension == 'qual454': + raise AssertionError, "qual454 data type was not correctly sniffed." + def test_070_qualsolid_datatype( self ): + """Testing correctly sniffing qualsolid data type upon upload""" + self.upload_file( 'qualscores.qualsolid' ) + self.verify_dataset_correctness('qualscores.qualsolid' ) + self.check_history_for_string('2.5 Kb, format: qualsolid, database: \? Info: uploaded file' ) + self.check_metadata_for_string( 'Change data type value="qualsolid" selected="yes">qualsolid' ) + latest_hda = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert latest_hda is not None, "Problem retrieving qualsolid hda from the database" + if not latest_hda.name == 'qualscores.qualsolid' and not latest_hda.extension == 'qualsolid': + raise AssertionError, "qualsolid data type was not correctly sniffed." + def test_075_tabular_datatype( self ): """Testing correctly sniffing tabular data type upon upload""" self.upload_file( '1.tabular' ) self.verify_dataset_correctness( '1.tabular' ) self.check_history_for_string( '1.tabular format: tabular, database: \? Info: uploaded file' ) self.check_metadata_for_string( 'value="1.tabular" value="\?"' ) self.check_metadata_for_string( 'Change data type selected value="tabular" selected="yes"' ) - def test_50_wig_datatype( self ): + latest_hda = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert latest_hda is not None, "Problem retrieving tabular hda from the database" + if not latest_hda.name == '1.tabular' and not latest_hda.extension == 'tabular': + raise AssertionError, "tabular data type was not correctly sniffed." + def test_080_wig_datatype( self ): """Testing correctly sniffing wig data type upon upload""" self.upload_file( '1.wig' ) self.verify_dataset_correctness( '1.wig' ) self.check_history_for_string( '1.wig format: wig, database: \? Info: uploaded file' ) self.check_metadata_for_string( 'value="1.wig" value="\?"' ) self.check_metadata_for_string( 'Change data type selected value="wig" selected="yes"' ) - def test_55_blastxml_datatype( self ): - """Testing correctly sniffing blastxml data type upon upload""" - self.upload_file( 'megablast_xml_parser_test1.gz' ) - self.check_history_for_string( 'NCBI Blast XML data' ) - self.check_history_for_string( 'format: blastxml' ) - """ - TODO: It is currently not even possible to set the following format on upload. They - should be included in the File Format select list on the upload form if they are to - be tested here... - def test_60_qualsolid_datatype( self ): - Testing correctly sniffing qualsolid data type upon upload - self.upload_file( 'qualscores.qualsolid' ) - self.verify_dataset_correctness('qualscores.qualsolid' ) - self.check_history_for_string('2.5 Kb, format: fasta, database: \? Info: uploaded file' ) - def test_65_qual454_datatype( self ): - Testing correctly sniffing qual454 data type upon upload - self.upload_file( 'qualscores.qual454' ) - self.verify_dataset_correctness( 'qualscores.qual454' ) - self.check_history_for_string( '5.6 Kb, format: qual454, database: \? Info: uploaded qual454 file' ) - """ + latest_hda = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert latest_hda is not None, "Problem retrieving wig hda from the database" + if not latest_hda.name == '1.wig' and not latest_hda.extension == 'wig': + raise AssertionError, "wig data type was not correctly sniffed." def test_9999_clean_up( self ): self.delete_history( id=str( history1.id ) ) self.logout()

1 0

[hg] galaxy 2439: Fix for handling the of the error given by upl...
by Greg Von Kuster 09 Jun '09

09 Jun '09

details: http://www.bx.psu.edu/hg/galaxy/rev/6d00e4ff7129 changeset: 2439:6d00e4ff7129 user: Dan Blankenberg <dan(a)bx.psu.edu> date: Tue Jun 09 12:05:46 2009 -0400 description: Fix for handling the of the error given by upload of html, empty, etc, files 2 file(s) affected in this change: lib/galaxy/tools/actions/upload.py test/base/twilltestcase.py diffs (125 lines): diff -r 17fd27c7f286 -r 6d00e4ff7129 lib/galaxy/tools/actions/upload.py --- a/lib/galaxy/tools/actions/upload.py Tue Jun 09 11:01:19 2009 -0400 +++ b/lib/galaxy/tools/actions/upload.py Tue Jun 09 12:05:46 2009 -0400 @@ -126,10 +126,24 @@ return dict( output=data ) def add_file( self, trans, temp_name, file_name, file_type, is_multi_byte, dbkey, info=None, space_to_tab=False, precreated_dataset=None ): + def dataset_no_data_error( data, message = 'there was an error uploading your file' ): + data.info = "No data: %s." % message + data.state = data.states.ERROR + if data.extension is None: + data.extension = 'data' + return data data_type = None + + if precreated_dataset is not None: + data = precreated_dataset + else: + data = trans.app.model.HistoryDatasetAssociation( history = trans.history, extension = ext, create_dataset = True ) + trans.app.security_agent.set_all_dataset_permissions( data.dataset, trans.app.security_agent.history_get_default_permissions( trans.history ) ) + # See if we have an empty file if not os.path.getsize( temp_name ) > 0: - raise BadFileException( "you attempted to upload an empty file." ) + return dataset_no_data_error( data, message = 'you attempted to upload an empty file' ) + #raise BadFileException( "you attempted to upload an empty file." ) if is_multi_byte: ext = sniff.guess_ext( temp_name, is_multi_byte=True ) else: @@ -138,7 +152,8 @@ # we'll decompress on the fly. is_gzipped, is_valid = self.check_gzip( temp_name ) if is_gzipped and not is_valid: - raise BadFileException( "you attempted to upload an inappropriate file." ) + return dataset_no_data_error( data, message = 'you attempted to upload an inappropriate file' ) + #raise BadFileException( "you attempted to upload an inappropriate file." ) elif is_gzipped and is_valid: # We need to uncompress the temp_name file CHUNK_SIZE = 2**20 # 1Mb @@ -150,7 +165,8 @@ except IOError: os.close( fd ) os.remove( uncompressed ) - raise BadFileException( 'problem decompressing gzipped data.' ) + return dataset_no_data_error( data, message = 'problem decompressing gzipped data' ) + #raise BadFileException( 'problem decompressing gzipped data.' ) if not chunk: break os.write( fd, chunk ) @@ -165,16 +181,20 @@ # See if we have a zip archive is_zipped, is_valid, test_ext = self.check_zip( temp_name ) if is_zipped and not is_valid: - raise BadFileException( "you attempted to upload an inappropriate file." ) + return dataset_no_data_error( data, message = 'you attempted to upload an inappropriate file' ) + #raise BadFileException( "you attempted to upload an inappropriate file." ) elif is_zipped and is_valid: # Currently, we force specific tools to handle this case. We also require the user # to manually set the incoming file_type if ( test_ext == 'ab1' or test_ext == 'scf' ) and file_type != 'binseq.zip': - raise BadFileException( "Invalid 'File Format' for archive consisting of binary files - use 'Binseq.zip'." ) + return dataset_no_data_error( data, message = "Invalid 'File Format' for archive consisting of binary files - use 'Binseq.zip'" ) + #raise BadFileException( "Invalid 'File Format' for archive consisting of binary files - use 'Binseq.zip'." ) elif test_ext == 'txt' and file_type != 'txtseq.zip': - raise BadFileException( "Invalid 'File Format' for archive consisting of text files - use 'Txtseq.zip'." ) + return dataset_no_data_error( data, message = "Invalid 'File Format' for archive consisting of text files - use 'Txtseq.zip'" ) + #raise BadFileException( "Invalid 'File Format' for archive consisting of text files - use 'Txtseq.zip'." ) if not ( file_type == 'binseq.zip' or file_type == 'txtseq.zip' ): - raise BadFileException( "you must manually set the 'File Format' to either 'Binseq.zip' or 'Txtseq.zip' when uploading zip files." ) + return dataset_no_data_error( data, message = "you must manually set the 'File Format' to either 'Binseq.zip' or 'Txtseq.zip' when uploading zip files" ) + #raise BadFileException( "you must manually set the 'File Format' to either 'Binseq.zip' or 'Txtseq.zip' when uploading zip files." ) data_type = 'zip' ext = file_type if not data_type: @@ -183,16 +203,20 @@ if len( parts ) > 1: ext = parts[1].strip().lower() if not( ext == 'ab1' or ext == 'scf' ): - raise BadFileException( "you attempted to upload an inappropriate file." ) + return dataset_no_data_error( data, message = "you attempted to upload an inappropriate file" ) + #raise BadFileException( "you attempted to upload an inappropriate file." ) if ext == 'ab1' and file_type != 'ab1': - raise BadFileException( "you must manually set the 'File Format' to 'Ab1' when uploading ab1 files." ) + return dataset_no_data_error( data, message = "you must manually set the 'File Format' to 'Ab1' when uploading ab1 files" ) + #raise BadFileException( "you must manually set the 'File Format' to 'Ab1' when uploading ab1 files." ) elif ext == 'scf' and file_type != 'scf': - raise BadFileException( "you must manually set the 'File Format' to 'Scf' when uploading scf files." ) + return dataset_no_data_error( data, message = "you must manually set the 'File Format' to 'Scf' when uploading scf files" ) + #raise BadFileException( "you must manually set the 'File Format' to 'Scf' when uploading scf files." ) data_type = 'binary' if not data_type: # We must have a text file if trans.app.datatypes_registry.get_datatype_by_extension( file_type ).composite_type != 'auto_primary_file' and self.check_html( temp_name ): - raise BadFileException( "you attempted to upload an inappropriate file." ) + return dataset_no_data_error( data, message = "you attempted to upload an inappropriate file" ) + #raise BadFileException( "you attempted to upload an inappropriate file." ) if data_type != 'binary' and data_type != 'zip': if space_to_tab: self.line_count = sniff.convert_newlines_sep2tabs( temp_name ) @@ -205,12 +229,7 @@ data_type = ext if info is None: info = 'uploaded %s file' %data_type - if precreated_dataset is not None: - data = precreated_dataset - data.extension = ext - else: - data = trans.app.model.HistoryDatasetAssociation( history = trans.history, extension = ext, create_dataset = True ) - trans.app.security_agent.set_all_dataset_permissions( data.dataset, trans.app.security_agent.history_get_default_permissions( trans.history ) ) + data.extension = ext data.name = file_name data.dbkey = dbkey data.info = info diff -r 17fd27c7f286 -r 6d00e4ff7129 test/base/twilltestcase.py --- a/test/base/twilltestcase.py Tue Jun 09 11:01:19 2009 -0400 +++ b/test/base/twilltestcase.py Tue Jun 09 12:05:46 2009 -0400 @@ -552,7 +552,7 @@ for value in kwd[ control.name ]: if value not in control.value and True not in [ value in item_label for item_label in item_labels ]: changed = True - break + break if changed: # Clear Control and set to proper value control.clear()

1 0

[hg] galaxy 2437: Merge genetics.py
by Greg Von Kuster 09 Jun '09

09 Jun '09

details: http://www.bx.psu.edu/hg/galaxy/rev/e27e392806df changeset: 2437:e27e392806df user: Greg Von Kuster <greg(a)bx.psu.edu> date: Tue Jun 09 10:28:27 2009 -0400 description: Merge genetics.py 1 file(s) affected in this change: lib/galaxy/datatypes/genetics.py diffs (159 lines): diff -r 2c6720a9130c -r e27e392806df lib/galaxy/datatypes/genetics.py --- a/lib/galaxy/datatypes/genetics.py Mon Jun 08 21:02:41 2009 -0400 +++ b/lib/galaxy/datatypes/genetics.py Tue Jun 09 10:28:27 2009 -0400 @@ -17,6 +17,7 @@ from galaxy import util from cgi import escape import urllib +from galaxy.web import url_for from galaxy.datatypes import metadata from galaxy.datatypes.metadata import MetadataElement #from galaxy.datatypes.data import Text @@ -80,7 +81,7 @@ for site_name, site_url in util.get_ucsc_by_build(dataset.dbkey): if site_name in app.config.ucsc_display_sites: site_url = site_url.replace('/hgTracks?','/hgGenome?') # for genome graphs - display_url = urllib.quote_plus( "%s/display_as?id=%i&display_app=%s" % (base_url, dataset.id, type) ) + display_url = urllib.quote_plus( "%s%s/display_as?id=%i&display_app=%s" % (base_url, url_for( controller='root' ), dataset.id, type)) sl = ["%sdb=%s" % (site_url,dataset.dbkey ),] sl.append("&hgGenome_dataSetName=%s&hgGenome_dataSetDescription=%s" % (dataset.name, 'GalaxyGG_data')) sl.append("&hgGenome_formatType=best%20guess&hgGenome_markerType=best%20guess") @@ -116,8 +117,8 @@ class Rgenetics(Html): """class to use for rgenetics""" """Add metadata elements""" - MetadataElement( name="base_name", desc="base name for all transformed versions of this genetic dataset", default="galaxy", readonly=True) + file_ext="html" composite_type = 'auto_primary_file' @@ -234,7 +235,128 @@ """fake class to distinguish different species of Rgenetics data collections """ file_ext="snptest" + +class RexpBase( Html ): + """base class for BioC data structures in Galaxy + must be constructed with the pheno data in place since that + goes into the metadata for each instance""" + """Add metadata elements""" + MetadataElement( name="columns", default=0, desc="Number of columns", readonly=True, visible=False ) + MetadataElement( name="column_names", default=[], desc="Column names", readonly=True,visible=True ) + MetadataElement( name="base_name", + desc="base name for all transformed versions of this genetic dataset", readonly=True) + MetadataElement( name="pheno_path", + desc="Path to phenotype data for this experiment", readonly=True) + MetadataElement( name="pheno", + desc="Phenotype data for this experiment", readonly=True) + + file_ext = None + + def set_peek( self, dataset ): + """expects a .pheno file in the extra_files_dir - ugh + note that R is wierd and does not include the row.name in + the header. why?""" + p = file(dataset.metadata.pheno_path,'r').readlines() + head = p[0].strip().split('\t') + head.insert(0,'ChipFileName') # fix R write.table b0rken-ness + p[0] = '\t'.join(head) + p = '\n'.join(p) + dataset.peek = p + dataset.metadata.pheno = p + dataset.blurb = 'R loadable BioC expression object for the Rexpression Galaxy toolkit' + + # stolen from Tabular + # class Tabular( data.Text ): + """Tab delimited data""" + + """Add metadata elements""" + def init_meta( self, dataset, copy_from=None ): + if copy_from: + dataset.metadata = copy_from.metadata + + + #def set_readonly_meta( self, dataset, skip=0, **kwd ): + # """Resets the values of readonly metadata elements.""" + # RexpBase.set_meta( self, dataset, skip=skip ) + + def set_readonly_meta( self, dataset, **kwd ): + """Resets the values of readonly metadata elements.""" + RexpBase.set_meta( self, dataset ) + + #def set_meta( self, dataset, skip=0, **kwd ): + def set_meta( self, dataset, **kwd ): + + """ + NOTE we apply the tabular machinary to the phenodata extracted + from a BioC eSet or affybatch. + + """ + if not dataset.peek: + dataset.set_peek() + pk = dataset.peek # use the peek which is the pheno data insead of dataset (!) + if pk: + p = pk.split('\n') + h = p[0].strip().split('\t') # hope is header + h = [escape(x) for x in h] + dataset.metadata.column_names = h + dataset.metadata.columns = len(h) + else: + dataset.metadata.column_names = [] + dataset.metadata.columns = 0 + + def make_html_table( self, dataset): + """Create HTML table, used for displaying peek""" + out = ['<table cellspacing="0" cellpadding="3">',] + try: + # Generate column header + pk = dataset.peek + p = pk.split('\n') + for i,row in enumerate(p): + lrow = row.strip().split('\t') + if i == 0: + orow = ['<th>%s</th>' % escape(x) for x in lrow] + orow.insert(0,'<tr>') + orow.append('</tr>') + else: + orow = ['<td>%s</td>' % escape(x) for x in lrow] + orow.insert(0,'<tr>') + orow.append('</tr>') + out.append(''.join(orow)) + out.append( '</table>' ) + out = "\n".join( out ) + except Exception, exc: + out = "Can't create peek %s" % str( exc ) + return out + + def display_peek( self, dataset ): + """Returns formatted html of peek""" + if not dataset.peek: + dataset.set_peek() + return self.make_html_table( dataset ) + + def get_mime(self): + """Returns the mime type of the datatype""" + return 'application/gzip' + + def sniff(self): + """ can we be bothered looking for the signature or loading via rpy? + """ + return true + +class AffyBatch( RexpBase ): + """derived class for BioC data structures in Galaxy """ + file_ext = "affybatch" + + +class ESet( RexpBase ): + """derived class for BioC data structures in Galaxy """ + file_ext = "eset" + + +class MAList( RexpBase ): + """derived class for BioC data structures in Galaxy """ + file_ext = "malist" if __name__ == '__main__':

1 0

[hg] galaxy 2438: Refresh the history frame when the current his...
by Greg Von Kuster 09 Jun '09

09 Jun '09

details: http://www.bx.psu.edu/hg/galaxy/rev/17fd27c7f286 changeset: 2438:17fd27c7f286 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Tue Jun 09 11:01:19 2009 -0400 description: Refresh the history frame when the current history is deleted ( fixes issue # 40 ). 1 file(s) affected in this change: lib/galaxy/web/controllers/history.py diffs (27 lines): diff -r e27e392806df -r 17fd27c7f286 lib/galaxy/web/controllers/history.py --- a/lib/galaxy/web/controllers/history.py Tue Jun 09 10:28:27 2009 -0400 +++ b/lib/galaxy/web/controllers/history.py Tue Jun 09 11:01:19 2009 -0400 @@ -104,12 +104,12 @@ if histories: if operation == "switch": status, message = self._list_switch( trans, histories ) - # Current history changed, refresh history frame - trans.template_context['refresh_frames'] = ['history'] elif operation == "delete": status, message = self._list_delete( trans, histories ) elif operation == "undelete": status, message = self._list_undelete( trans, histories ) + # Current history may have changed, refresh history frame + trans.template_context['refresh_frames'] = ['history'] trans.sa_session.flush() # Render the list view return self.list_grid( trans, status=status, message=message, **kwargs ) @@ -189,7 +189,7 @@ # Regardless of whether it was previously deleted, we make a new # history active trans.new_history() - return trans.show_ok_message( "History deleted, a new history is active" ) + return trans.show_ok_message( "History deleted, a new history is active", refresh_frames=['history'] ) @web.expose def rename_async( self, trans, id=None, new_name=None ): history = model.History.get( id )

1 0