December 2008 - galaxy-dev - lists.galaxyproject.org

[hg] galaxy 1671: Server errors thrown when switching between Ga...
by Greg Von Kuster 16 Dec '08

16 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/87f793e3dce5 changeset: 1671:87f793e3dce5 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Tue Dec 16 10:04:30 2008 -0500 description: Server errors thrown when switching between Galaxy and reports in the same browser was due to the id_secret being different in the Galaxy config and the reports config, so add comments to ensure this is not repeated. 4 file(s) affected in this change: lib/galaxy/config.py lib/galaxy/webapps/reports/config.py reports_wsgi.ini.sample universe_wsgi.ini.sample diffs (79 lines): diff -r cdb2935dc336 -r 87f793e3dce5 lib/galaxy/config.py --- a/lib/galaxy/config.py Mon Dec 15 17:04:06 2008 -0500 +++ b/lib/galaxy/config.py Tue Dec 16 10:04:30 2008 -0500 @@ -35,6 +35,9 @@ self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root ) self.tool_config = resolve_path( kwargs.get( 'tool_config_file', 'tool_conf.xml' ), self.root ) self.tool_secret = kwargs.get( "tool_secret", "" ) + # CRITICAL: the value of id_secret MUST be the same in universe_wsgi.ini and reports_wsgi.ini + # or the framework's __ensure_valid_session method will throw exceptions when switching between + # Galaxy and the reports in the same browser. self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" ) self.use_remote_user = string_as_bool( kwargs.get( "use_remote_user", "False" ) ) self.remote_user_maildomain = kwargs.get( "remote_user_maildomain", None ) diff -r cdb2935dc336 -r 87f793e3dce5 lib/galaxy/webapps/reports/config.py --- a/lib/galaxy/webapps/reports/config.py Mon Dec 15 17:04:06 2008 -0500 +++ b/lib/galaxy/webapps/reports/config.py Tue Dec 16 10:04:30 2008 -0500 @@ -25,14 +25,14 @@ # Where dataset files are stored self.file_path = resolve_path( kwargs.get( "file_path", "database/files" ), self.root ) self.new_file_path = resolve_path( kwargs.get( "new_file_path", "database/tmp" ), self.root ) + # CRITICAL: the value of id_secret MUST be the same in universe_wsgi.ini and reports_wsgi.ini + # or the framework's __ensure_valid_session method will throw exceptions when switching between + # Galaxy and the reports in the same browser. self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" ) self.use_remote_user = string_as_bool( kwargs.get( "use_remote_user", "False" ) ) self.template_path = resolve_path( kwargs.get( "template_path", "templates" ), self.root ) self.template_cache = resolve_path( kwargs.get( "template_cache_path", "database/reports/compiled_templates" ), self.root ) - self.admin_pass = kwargs.get('admin_pass',"galaxy") self.sendmail_path = kwargs.get('sendmail_path',"/usr/sbin/sendmail") - self.mailing_join_addr = kwargs.get('mailing_join_addr',"galaxy-user-join(a)bx.psu.edu") - self.error_email_to = kwargs.get( 'error_email_to', None ) self.brand = kwargs.get( 'brand', None ) self.wiki_url = kwargs.get( 'wiki_url', "http://g2.trac.bx.psu.edu/" ) self.bugs_email = kwargs.get( 'bugs_email', "mailto:galaxy-bugs@bx.psu.edu" ) diff -r cdb2935dc336 -r 87f793e3dce5 reports_wsgi.ini.sample --- a/reports_wsgi.ini.sample Mon Dec 15 17:04:06 2008 -0500 +++ b/reports_wsgi.ini.sample Tue Dec 16 10:04:30 2008 -0500 @@ -36,15 +36,16 @@ session_key = galaxysessions session_secret = changethisinproduction +# Galaxy session security +# The value of id_secret MUST be the same as the value of id_secret in universe_wsgi.ini +id_secret = changethisinproductiontoo + # Configuration for debugging middleware debug = true use_lint = false # NEVER enable this on a public site (even test or QA) # use_interactive = true - -# Admin Password -admin_pass = "galaxy" # path to sendmail sendmail_path = /usr/sbin/sendmail @@ -58,10 +59,6 @@ # Profiling middleware (cProfile based) ## use_profile = True -# Mail -smtp_server = coltrane.bx.psu.edu -error_email_to = galaxy_bugs(a)bx.psu.edu - # Use the new iframe / javascript based layout use_new_layout = true diff -r cdb2935dc336 -r 87f793e3dce5 universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample Mon Dec 15 17:04:06 2008 -0500 +++ b/universe_wsgi.ini.sample Tue Dec 16 10:04:30 2008 -0500 @@ -73,6 +73,7 @@ session_secret = changethisinproduction # Galaxy session security +# The value of id_secret MUST be the same as the value of id_secret in reports_wsgi.ini id_secret = changethisinproductiontoo # Use user provided in an upstream server's $REMOTE_USER variable

1 0

[hg] galaxy 1666: Attempting to merge conflicts.
by Greg Von Kuster 15 Dec '08

15 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/ced77aa5f967 changeset: 1666:ced77aa5f967 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Mon Dec 15 08:58:33 2008 -0500 description: Attempting to merge conflicts. 1 file(s) affected in this change: tools/data_source/gbrowse_datasource.py diffs (55 lines): diff -r 503fcdb79851 -r ced77aa5f967 tools/data_source/gbrowse_datasource.py --- a/tools/data_source/gbrowse_datasource.py Mon Dec 15 08:55:16 2008 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,51 +0,0 @@ -#!/usr/bin/env python -#Retreives data from GMOD and stores in a file. GBrowse parameters are provided in the input/output file. -import urllib, sys, os, gzip, tempfile, shutil - -assert sys.version_info[:2] >= ( 2, 4 ) - -def stop_err( msg ): - sys.stderr.write( msg ) - sys.exit() - -def __main__(): - filename = sys.argv[1] - params = {} - - for line in open( filename, 'r' ): - try: - line = line.strip() - fields = line.split( '\t' ) - params[ fields[0] ] = fields[1] - except: - continue - - URL = params.get( 'URL', None ) - if not URL: - open( filename, 'w' ).write( "" ) - stop_err( 'Datasource has not sent back a URL parameter.' ) - - for i, param in enumerate( params.keys() ): - if i == 0: - sep = '?' - else: - sep = '&' - if param != '__collected_datasets__': - URL += "%s%s=%s" % ( sep, param, params.get( param ) ) - - CHUNK_SIZE = 2**20 # 1Mb - try: - page = urllib.urlopen( URL ) - except Exception, exc: - raise Exception( 'Problems connecting to %s (%s)' % ( URL, exc ) ) - sys.exit( 1 ) - - fp = open( filename, 'wb' ) - while 1: - chunk = page.read( CHUNK_SIZE ) - if not chunk: - break - fp.write( chunk ) - fp.close() - -if __name__ == "__main__": __main__()

1 0

[hg] galaxy 1667: Reset functional test port which somehow got c...
by Greg Von Kuster 15 Dec '08

15 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/3a0b07161c1a changeset: 1667:3a0b07161c1a user: Greg Von Kuster <greg(a)bx.psu.edu> date: Mon Dec 15 09:06:11 2008 -0500 description: Reset functional test port which somehow got committed when attempting to merge conflicts. 1 file(s) affected in this change: test/functional/__init__.py diffs (12 lines): diff -r ced77aa5f967 -r 3a0b07161c1a test/functional/__init__.py --- a/test/functional/__init__.py Mon Dec 15 08:58:33 2008 -0500 +++ b/test/functional/__init__.py Mon Dec 15 09:06:11 2008 -0500 @@ -24,7 +24,7 @@ # server (for running the tests against a running instance) default_galaxy_test_host = "localhost" -default_galaxy_test_port = "9879" +default_galaxy_test_port = "9999" galaxy_test_file_dir = "test-data" server = None app = None

1 0

[hg] galaxy 1668: Rename gbrowse_elegans to wormbase_test, add a...
by Greg Von Kuster 15 Dec '08

15 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/87a9cf297998 changeset: 1668:87a9cf297998 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Mon Dec 15 10:37:39 2008 -0500 description: Rename gbrowse_elegans to wormbase_test, add a config for the wormbase production server. 4 file(s) affected in this change: tool_conf.xml.sample tools/data_source/gbrowse_elegans.xml tools/data_source/wormbase.xml tools/data_source/wormbase_test.xml diffs (104 lines): diff -r 3a0b07161c1a -r 87a9cf297998 tool_conf.xml.sample --- a/tool_conf.xml.sample Mon Dec 15 09:06:11 2008 -0500 +++ b/tool_conf.xml.sample Mon Dec 15 10:37:39 2008 -0500 @@ -9,7 +9,8 @@ <tool file="data_source/microbial_import.xml" /> <tool file="data_source/biomart.xml" /> <tool file="data_source/biomart_test.xml" /> - <tool file="data_source/gbrowse_elegans.xml" /> + <tool file="data_source/wormbase.xml" /> + <tool file="data_source/wormbase_test.xml" /> <tool file="data_source/flymine.xml" /> <tool file="data_source/encode_db.xml" /> <tool file="data_source/epigraph_import.xml" /> diff -r 3a0b07161c1a -r 87a9cf297998 tools/data_source/gbrowse_elegans.xml --- a/tools/data_source/gbrowse_elegans.xml Mon Dec 15 09:06:11 2008 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,27 +0,0 @@ -<?xml version="1.0"?> -<tool name="C. Elegans" id="gbrowse_elegans" tool_type="data_source" URL_method="get"> - <description>server</description> - <command interpreter="python">data_source.py $output</command> - <inputs action="http://dev.wormbase.org/db/seq/gbrowse/c_elegans/" check_values="false" method="get" target="_top"> - <display>go to C. Elegans server $GALAXY_URL</display> - <param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=gbrowse_elegans" /> - </inputs> - <request_param_translation> - <request_param galaxy_name="URL" remote_name="URL" missing=""> - <add_to_url> - <param_from_source name="d" missing="" /> - <param_from_source name="dbkey" missing="" /> - <param_from_source name="q" missing="" /> - <param_from_source name="s" missing="" /> - <param_from_source name="t" missing="" /> - </add_to_url> - </request_param> - <request_param galaxy_name="data_type" remote_name="data_type" missing="gff3" /> - </request_param_translation> - <uihints minwidth="800"/> - <outputs> - <data name="output" format="txt" /> - </outputs> - <options sanitize="False" refresh="True"/> -</tool> - diff -r 3a0b07161c1a -r 87a9cf297998 tools/data_source/wormbase.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/data_source/wormbase.xml Mon Dec 15 10:37:39 2008 -0500 @@ -0,0 +1,26 @@ +<?xml version="1.0"?> +<tool name="Wormbase" id="wormbase" tool_type="data_source" URL_method="get"> + <description>server</description> + <command interpreter="python">data_source.py $output</command> + <inputs action="http://www.wormbase.org/db/seq/gbrowse/c_elegans/" check_values="false" method="get" target="_top"> + <display>go to Wormbase server $GALAXY_URL</display> + <param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=wormbase" /> + </inputs> + <request_param_translation> + <request_param galaxy_name="URL" remote_name="URL" missing=""> + <add_to_url> + <param_from_source name="d" missing="" /> + <param_from_source name="dbkey" missing="" /> + <param_from_source name="q" missing="" /> + <param_from_source name="s" missing="" /> + <param_from_source name="t" missing="" /> + </add_to_url> + </request_param> + <request_param galaxy_name="data_type" remote_name="data_type" missing="txt" /> + </request_param_translation> + <uihints minwidth="800"/> + <outputs> + <data name="output" format="txt" /> + </outputs> + <options sanitize="False" refresh="True"/> +</tool> diff -r 3a0b07161c1a -r 87a9cf297998 tools/data_source/wormbase_test.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/data_source/wormbase_test.xml Mon Dec 15 10:37:39 2008 -0500 @@ -0,0 +1,26 @@ +<?xml version="1.0"?> +<tool name="Wormbase" id="wormbase_test" tool_type="data_source" URL_method="get"> + <description>test server</description> + <command interpreter="python">data_source.py $output</command> + <inputs action="http://dev.wormbase.org/db/seq/gbrowse/c_elegans/" check_values="false" method="get" target="_top"> + <display>go to Wormbase test server $GALAXY_URL</display> + <param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=wormbase_test" /> + </inputs> + <request_param_translation> + <request_param galaxy_name="URL" remote_name="URL" missing=""> + <add_to_url> + <param_from_source name="d" missing="" /> + <param_from_source name="dbkey" missing="" /> + <param_from_source name="q" missing="" /> + <param_from_source name="s" missing="" /> + <param_from_source name="t" missing="" /> + </add_to_url> + </request_param> + <request_param galaxy_name="data_type" remote_name="data_type" missing="txt" /> + </request_param_translation> + <uihints minwidth="800"/> + <outputs> + <data name="output" format="txt" /> + </outputs> + <options sanitize="False" refresh="True"/> +</tool>

1 0

[hg] galaxy 1669: Fix for the users reports.
by Greg Von Kuster 15 Dec '08

15 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/493a4a3ec2d8 changeset: 1669:493a4a3ec2d8 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Mon Dec 15 11:25:47 2008 -0500 description: Fix for the users reports. 1 file(s) affected in this change: lib/galaxy/webapps/reports/controllers/users.py diffs (18 lines): diff -r 87a9cf297998 -r 493a4a3ec2d8 lib/galaxy/webapps/reports/controllers/users.py --- a/lib/galaxy/webapps/reports/controllers/users.py Mon Dec 15 10:37:39 2008 -0500 +++ b/lib/galaxy/webapps/reports/controllers/users.py Mon Dec 15 11:25:47 2008 -0500 @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import * from time import strftime import calendar, operator from galaxy.webapps.reports.base.controller import * @@ -98,7 +98,7 @@ def last_access_date( self, trans, **kwd ): params = util.Params( kwd ) msg = params.get( 'msg', '' ) - not_logged_in_for_days = params.get( 'not_logged_in_for_days', 0 ) + not_logged_in_for_days = params.get( 'not_logged_in_for_days', 90 ) if not not_logged_in_for_days: not_logged_in_for_days = 0 cutoff_time = datetime.utcnow() - timedelta( days=int( not_logged_in_for_days ) )

1 0

[hg] galaxy 1665: merging heads
by Greg Von Kuster 15 Dec '08

15 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/503fcdb79851 changeset: 1665:503fcdb79851 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Mon Dec 15 08:55:16 2008 -0500 description: merging heads 3 file(s) affected in this change: lib/galaxy/web/framework/__init__.py test/functional/__init__.py tools/data_source/gbrowse_datasource.py diffs (116 lines): diff -r 5f4511a5b4d6 -r 503fcdb79851 lib/galaxy/web/framework/__init__.py --- a/lib/galaxy/web/framework/__init__.py Sat Dec 13 23:35:29 2008 -0500 +++ b/lib/galaxy/web/framework/__init__.py Mon Dec 15 08:55:16 2008 -0500 @@ -105,6 +105,8 @@ self.app = app self.webapp = webapp self.security = webapp.security + # FIXME: the following 3 attributes are not currently used + # Remove them if they are not going to be... self.__user = NOT_SET self.__history = NOT_SET self.__galaxy_session = NOT_SET @@ -178,9 +180,17 @@ Support for universe_session and universe_user cookies has been removed as of 31 Oct 2008. """ + #log.debug("####In __ensure_valid_session...") + #try: + # log.debug("####In __ensure_valid_session, self.galaxy_session: %s" % str( self.galaxy_session ) ) + # log.debug("####In __ensure_valid_session, self.galaxy_session.is_valid: %s" % str(self.galaxy_session.is_valid)) + #except: + # log.debug("####In __ensure_valid_session, self.galaxy_session does not yet exist...") sa_session = self.sa_session # Try to load an existing session + #log.debug("###In __ensure_valid_session, before secure_id = self.get_cookie( name='galaxysession' )...") secure_id = self.get_cookie( name='galaxysession' ) + #log.debug("###In __ensure_valid_session, secure_id: %s" % str( secure_id) ) galaxy_session = None prev_galaxy_session = None user_for_new_session = None @@ -191,8 +201,10 @@ if secure_id: # Decode the cookie value to get the session_key session_key = self.security.decode_session_key( secure_id ) + #log.debug("###In __ensure_valid_session, session_key: %s" % str( session_key) ) # Retrive the galaxy_session id via the unique session_key galaxy_session = sa_session.query( self.app.model.GalaxySession ).filter_by( session_key=session_key, is_valid=True ).first() + #log.debug("###In __ensure_valid_session, galaxy_session: %s" % str( galaxy_session) ) # If remote user is in use it can invalidate the session, so we need to # to check some things now. if self.app.config.use_remote_user: @@ -248,6 +260,7 @@ Caller is responsible for flushing the returned session. """ + #log.debug("###In __create_new_session...") session_key = self.security.get_new_session_key() galaxy_session = self.app.model.GalaxySession( session_key=session_key, diff -r 5f4511a5b4d6 -r 503fcdb79851 test/functional/__init__.py --- a/test/functional/__init__.py Sat Dec 13 23:35:29 2008 -0500 +++ b/test/functional/__init__.py Mon Dec 15 08:55:16 2008 -0500 @@ -24,7 +24,7 @@ # server (for running the tests against a running instance) default_galaxy_test_host = "localhost" -default_galaxy_test_port = "9999" +default_galaxy_test_port = "9879" galaxy_test_file_dir = "test-data" server = None app = None diff -r 5f4511a5b4d6 -r 503fcdb79851 tools/data_source/gbrowse_datasource.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/data_source/gbrowse_datasource.py Mon Dec 15 08:55:16 2008 -0500 @@ -0,0 +1,51 @@ +#!/usr/bin/env python +#Retreives data from GMOD and stores in a file. GBrowse parameters are provided in the input/output file. +import urllib, sys, os, gzip, tempfile, shutil + +assert sys.version_info[:2] >= ( 2, 4 ) + +def stop_err( msg ): + sys.stderr.write( msg ) + sys.exit() + +def __main__(): + filename = sys.argv[1] + params = {} + + for line in open( filename, 'r' ): + try: + line = line.strip() + fields = line.split( '\t' ) + params[ fields[0] ] = fields[1] + except: + continue + + URL = params.get( 'URL', None ) + if not URL: + open( filename, 'w' ).write( "" ) + stop_err( 'Datasource has not sent back a URL parameter.' ) + + for i, param in enumerate( params.keys() ): + if i == 0: + sep = '?' + else: + sep = '&' + if param != '__collected_datasets__': + URL += "%s%s=%s" % ( sep, param, params.get( param ) ) + + CHUNK_SIZE = 2**20 # 1Mb + try: + page = urllib.urlopen( URL ) + except Exception, exc: + raise Exception( 'Problems connecting to %s (%s)' % ( URL, exc ) ) + sys.exit( 1 ) + + fp = open( filename, 'wb' ) + while 1: + chunk = page.read( CHUNK_SIZE ) + if not chunk: + break + fp.write( chunk ) + fp.close() + +if __name__ == "__main__": __main__()

1 0

[hg] galaxy 1670: Some code cleanup in framework/__init__.py
by Greg Von Kuster 15 Dec '08

15 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/cdb2935dc336 changeset: 1670:cdb2935dc336 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Mon Dec 15 17:04:06 2008 -0500 description: Some code cleanup in framework/__init__.py 1 file(s) affected in this change: lib/galaxy/web/framework/__init__.py diffs (62 lines): diff -r 493a4a3ec2d8 -r cdb2935dc336 lib/galaxy/web/framework/__init__.py --- a/lib/galaxy/web/framework/__init__.py Mon Dec 15 11:25:47 2008 -0500 +++ b/lib/galaxy/web/framework/__init__.py Mon Dec 15 17:04:06 2008 -0500 @@ -25,7 +25,7 @@ import mako.lookup pkg_resources.require( "SQLAlchemy >= 0.4" ) -from sqlalchemy import desc +from sqlalchemy import and_ import logging log = logging.getLogger( __name__ ) @@ -180,17 +180,9 @@ Support for universe_session and universe_user cookies has been removed as of 31 Oct 2008. """ - #log.debug("####In __ensure_valid_session...") - #try: - # log.debug("####In __ensure_valid_session, self.galaxy_session: %s" % str( self.galaxy_session ) ) - # log.debug("####In __ensure_valid_session, self.galaxy_session.is_valid: %s" % str(self.galaxy_session.is_valid)) - #except: - # log.debug("####In __ensure_valid_session, self.galaxy_session does not yet exist...") sa_session = self.sa_session # Try to load an existing session - #log.debug("###In __ensure_valid_session, before secure_id = self.get_cookie( name='galaxysession' )...") secure_id = self.get_cookie( name='galaxysession' ) - #log.debug("###In __ensure_valid_session, secure_id: %s" % str( secure_id) ) galaxy_session = None prev_galaxy_session = None user_for_new_session = None @@ -201,10 +193,9 @@ if secure_id: # Decode the cookie value to get the session_key session_key = self.security.decode_session_key( secure_id ) - #log.debug("###In __ensure_valid_session, session_key: %s" % str( session_key) ) - # Retrive the galaxy_session id via the unique session_key - galaxy_session = sa_session.query( self.app.model.GalaxySession ).filter_by( session_key=session_key, is_valid=True ).first() - #log.debug("###In __ensure_valid_session, galaxy_session: %s" % str( galaxy_session) ) + # Retrieve the galaxy_session id via the unique session_key + galaxy_session = self.app.model.GalaxySession.filter( and_( self.app.model.GalaxySession.table.c.session_key==session_key, + self.app.model.GalaxySession.table.c.is_valid==True ) ).first() # If remote user is in use it can invalidate the session, so we need to # to check some things now. if self.app.config.use_remote_user: @@ -260,7 +251,6 @@ Caller is responsible for flushing the returned session. """ - #log.debug("###In __create_new_session...") session_key = self.security.get_new_session_key() galaxy_session = self.app.model.GalaxySession( session_key=session_key, @@ -293,8 +283,7 @@ """ Update the 'galaxysession' cookie to match the current session. """ - self.set_cookie( name='galaxysession', - value=self.security.encode_session_key( self.galaxy_session.session_key ) ) + self.set_cookie( self.security.encode_session_key( self.galaxy_session.session_key ), name='galaxysession' ) def handle_user_login( self, user ): """

1 0

[hg] galaxy 1664: Add ability for data_source tools to append pa...
by Greg Von Kuster 14 Dec '08

14 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/5f4511a5b4d6 changeset: 1664:5f4511a5b4d6 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Sat Dec 13 23:35:29 2008 -0500 description: Add ability for data_source tools to append parameters passed in the initial response to the value of URL prior to Galaxy's post to the URL. This is a cleaner method for Biomart and also gets GBrowse to wrok. 8 file(s) affected in this change: lib/galaxy/tools/__init__.py lib/galaxy/util/__init__.py tools/data_source/biomart.xml tools/data_source/biomart_test.xml tools/data_source/data_source.py tools/data_source/gbrowse_datasource.py tools/data_source/gbrowse_elegans.xml tools/data_source/gbrowse_filter_code.py diffs (293 lines): diff -r 0af3897ce585 -r 5f4511a5b4d6 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py Fri Dec 12 14:33:51 2008 -0500 +++ b/lib/galaxy/tools/__init__.py Sat Dec 13 23:35:29 2008 -0500 @@ -231,8 +231,6 @@ # data_source tool if self.tool_type == "data_source": self.URL_method = root.get( "URL_method", "get" ) # get is the default - # TODO: Biomart hack - eliminate when they encode URL - they'll let us know when... - self.add_to_URL = root.get( "add_to_URL", None ) self.param_trans_dict = {} req_param_trans = root.find( "request_param_translation" ) if req_param_trans is not None: @@ -255,6 +253,22 @@ galaxy_format = format.get( "galaxy_format" ) format_trans_dict[ remote_format ] = galaxy_format trans_list.append( format_trans_dict ) + elif req_param.get( "galaxy_name" ) == "URL": + # Some remote data sources ( e.g., Gbrowse ) send parameters back to + # Galaxy in the initial response that must be added to URL prior to + # Galaxy sending the secondary request to the URL. The tag set looks + # asomething like: + # <add_to_url> + # <param_from_source name="d" missing="" /> + # </add_to_url> + add_to_url = req_param.find( "add_to_url" ) + if add_to_url is not None: + add_to_url_dict = {} + for param_from_source in add_to_url.findall( "param_from_source" ): + name = param_from_source.get( "name" ) + value = param_from_source.get( "missing" ) # only used if the source doesn't send the param name + add_to_url_dict[ name ] = value + trans_list.append( add_to_url_dict ) self.param_trans_dict[ remote_name ] = trans_list # Command line (template). Optional for tools that do not invoke a local program command = root.find("command") @@ -1162,11 +1176,16 @@ description = param_dict.get( 'position', '' ) if not description: description = 'unknown position' + gb_landmark_region = param_dict.get( 'q' ) data_type = param_dict.get( 'data_type' ) items = out_data.items() for name, data in items: if organism and table and description: + # This is UCSC data.name = '%s on %s: %s (%s)' % ( data.name, organism, table, description ) + elif gb_landmark_region: + # This is GBrowse + data.name = '%s on %s' % ( data.name, gb_landmark_region ) data.info = info data.dbkey = dbkey try: diff -r 0af3897ce585 -r 5f4511a5b4d6 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py Fri Dec 12 14:33:51 2008 -0500 +++ b/lib/galaxy/util/__init__.py Sat Dec 13 23:35:29 2008 -0500 @@ -159,10 +159,31 @@ try: # The Galaxy "data_type entry is special in that it can include the ability # to translate the format to a Galaxy supported format. In the dict, this entry - # looks something like: {'hgta_outputType': ['data_type', 'bed', {'selectedFields': 'tabular'}] } + # looks something like: + # {'hgta_outputType': ['data_type', 'bed', {'selectedFields': 'tabular'}] } format_trans_dict = tool.param_trans_dict[ key ][2] if value in format_trans_dict: new_value = format_trans_dict[ value ] + except: + pass + elif new_key == 'URL': + # As above, the URL can include a set of params from the remote data source + # that must be appended to the URL prior to the post. In this case, the + # dict entry would look something like: + # ['URL', '', {'q': '', 's': '', 'd': '', 'dbkey': '', 't': ''}] + try: + add_to_url_dict = tool.param_trans_dict[ key ][2] + if new_value.count( '?' ) == 0: + sep = '?' + else: + sep = '&' + for param_name, missing_value in add_to_url_dict.items(): + param_value = params.get( param_name, None ) + if not param_value and missing_value: + param_value = missing_value + if param_value: + new_value += '%s%s=%s' % ( sep, param_name, param_value ) + sep = '&' except: pass if not value and not new_value: @@ -174,9 +195,6 @@ if tool and tool.tool_type == 'data_source': # Add the tool's URL_method to params self.__dict__[ 'URL_method' ] = tool.URL_method - # TODO: Biomart hack - eliminate when they encode URL - they'll let us know when... - if tool.add_to_URL is not None: - self.__dict__[ 'add_to_URL' ] = tool.add_to_URL for key, value in tool.param_trans_dict.items(): # Make sure that all translated values used in Galaxy are added to the params galaxy_name = tool.param_trans_dict[ key ][0] diff -r 0af3897ce585 -r 5f4511a5b4d6 tools/data_source/biomart.xml --- a/tools/data_source/biomart.xml Fri Dec 12 14:33:51 2008 -0500 +++ b/tools/data_source/biomart.xml Sat Dec 13 23:35:29 2008 -0500 @@ -7,7 +7,7 @@ TODO: Hack to get biomart to work - the 'add_to_URL' param can be eliminated when the Biomart team encodes URL prior to sending, meanwhile everything including and beyond the first '&' is truncated from URL. They said they'll let us know when this is fixed at their end. --> -<tool name="BioMart" id="biomart" tool_type="data_source" URL_method="get" add_to_URL="biomart_hack"> +<tool name="BioMart" id="biomart" tool_type="data_source" URL_method="get"> <description>Central server</description> <command interpreter="python">data_source.py $output</command> <inputs action="http://www.biomart.org/biomart/martview" check_values="false" method="get" target="_top"> @@ -15,7 +15,12 @@ <param name="GALAXY_URL" type="baseurl" value="/tool_runner/biomart" /> </inputs> <request_param_translation> - <request_param galaxy_name="URL" remote_name="URL" missing="" /> + <request_param galaxy_name="URL" remote_name="URL" missing=""> + <add_to_url> + <param_from_source name="_export" missing="1" /> + <param_from_source name="GALAXY_URL" missing="0" /> + </add_to_url> + </request_param> <request_param galaxy_name="dbkey" remote_name="dbkey" missing="?" /> <request_param galaxy_name="organism" remote_name="organism" missing="" /> <request_param galaxy_name="table" remote_name="table" missing="" /> diff -r 0af3897ce585 -r 5f4511a5b4d6 tools/data_source/biomart_test.xml --- a/tools/data_source/biomart_test.xml Fri Dec 12 14:33:51 2008 -0500 +++ b/tools/data_source/biomart_test.xml Sat Dec 13 23:35:29 2008 -0500 @@ -7,7 +7,7 @@ TODO: Hack to get biomart to work - the 'add_to_URL' param can be eliminated when the Biomart team encodes URL prior to sending, meanwhile everything including and beyond the first '&' is truncated from URL. They said they'll let us know when this is fixed at their end. --> -<tool name="BioMart" id="biomart_test" tool_type="data_source" URL_method="get" add_to_URL="biomart_hack"> +<tool name="BioMart" id="biomart_test" tool_type="data_source" URL_method="get"> <description>Test server</description> <command interpreter="python">data_source.py $output</command> <inputs action="http://test.biomart.org/biomart/martview" check_values="false" method="get" target="_top"> @@ -15,7 +15,12 @@ <param name="GALAXY_URL" type="baseurl" value="/tool_runner/biomart" /> </inputs> <request_param_translation> - <request_param galaxy_name="URL" remote_name="URL" missing="" /> + <request_param galaxy_name="URL" remote_name="URL" missing=""> + <add_to_url> + <param_from_source name="_export" missing="1" /> + <param_from_source name="GALAXY_URL" missing="0" /> + </add_to_url> + </request_param> <request_param galaxy_name="dbkey" remote_name="dbkey" missing="?" /> <request_param galaxy_name="organism" remote_name="organism" missing="" /> <request_param galaxy_name="table" remote_name="table" missing="" /> diff -r 0af3897ce585 -r 5f4511a5b4d6 tools/data_source/data_source.py --- a/tools/data_source/data_source.py Fri Dec 12 14:33:51 2008 -0500 +++ b/tools/data_source/data_source.py Sat Dec 13 23:35:29 2008 -0500 @@ -33,12 +33,6 @@ if not URL: open( filename, 'w' ).write( "" ) stop_err( 'The remote data source application has not sent back a URL parameter in the request.' ) - # TODO: Hack to get biomart to work - this can be eliminated when the Biomart team encodes URL prior to sending, meanwhile - # everything including and beyond the first '&' is truncated from URL. They said they'll let us know when this is fixed - # at their end. - add_to_URL = params.get( 'add_to_URL', None ) - if add_to_URL: - URL += '&_export=1&GALAXY_URL=0' URL_method = params.get( 'URL_method', None ) out = open( filename, 'w' ) CHUNK_SIZE = 2**20 # 1Mb diff -r 0af3897ce585 -r 5f4511a5b4d6 tools/data_source/gbrowse_datasource.py --- a/tools/data_source/gbrowse_datasource.py Fri Dec 12 14:33:51 2008 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,53 +0,0 @@ -#!/usr/bin/env python -#Retreives data from GMOD and stores in a file. GBrowse parameters are provided in the input/output file. -import urllib, sys, os, gzip, tempfile, shutil -from galaxy import eggs -from galaxy.datatypes import data - -assert sys.version_info[:2] >= ( 2, 4 ) - -def stop_err( msg ): - sys.stderr.write( msg ) - sys.exit() - -def __main__(): - filename = sys.argv[1] - params = {} - - for line in open( filename, 'r' ): - try: - line = line.strip() - fields = line.split( '\t' ) - params[ fields[0] ] = fields[1] - except: - continue - - URL = params.get( 'URL', None ) - if not URL: - open( filename, 'w' ).write( "" ) - stop_err( 'Datasource has not sent back a URL parameter.' ) - - for i, param in enumerate( params.keys() ): - if i == 0: - sep = '?' - else: - sep = '&' - if param != '__collected_datasets__': - URL += "%s%s=%s" % ( sep, param, params.get( param ) ) - - CHUNK_SIZE = 2**20 # 1Mb - try: - page = urllib.urlopen( URL ) - except Exception, exc: - raise Exception( 'Problems connecting to %s (%s)' % ( URL, exc ) ) - sys.exit( 1 ) - - fp = open( filename, 'wb' ) - while 1: - chunk = page.read( CHUNK_SIZE ) - if not chunk: - break - fp.write( chunk ) - fp.close() - -if __name__ == "__main__": __main__() diff -r 0af3897ce585 -r 5f4511a5b4d6 tools/data_source/gbrowse_elegans.xml --- a/tools/data_source/gbrowse_elegans.xml Fri Dec 12 14:33:51 2008 -0500 +++ b/tools/data_source/gbrowse_elegans.xml Sat Dec 13 23:35:29 2008 -0500 @@ -1,13 +1,24 @@ <?xml version="1.0"?> -<tool name="C. Elegans" id="gbrowse_elegans"> +<tool name="C. Elegans" id="gbrowse_elegans" tool_type="data_source" URL_method="get"> <description>server</description> - <command interpreter="python">gbrowse_datasource.py $output</command> - <inputs action="http://www.wormbase.org/db/seq/gbrowse/c_elegans/" check_values="false" method="get" target="_top"> + <command interpreter="python">data_source.py $output</command> + <inputs action="http://dev.wormbase.org/db/seq/gbrowse/c_elegans/" check_values="false" method="get" target="_top"> <display>go to C. Elegans server $GALAXY_URL</display> <param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=gbrowse_elegans" /> </inputs> + <request_param_translation> + <request_param galaxy_name="URL" remote_name="URL" missing=""> + <add_to_url> + <param_from_source name="d" missing="" /> + <param_from_source name="dbkey" missing="" /> + <param_from_source name="q" missing="" /> + <param_from_source name="s" missing="" /> + <param_from_source name="t" missing="" /> + </add_to_url> + </request_param> + <request_param galaxy_name="data_type" remote_name="data_type" missing="gff3" /> + </request_param_translation> <uihints minwidth="800"/> - <code file="gbrowse_filter_code.py"/> <outputs> <data name="output" format="txt" /> </outputs> diff -r 0af3897ce585 -r 5f4511a5b4d6 tools/data_source/gbrowse_filter_code.py --- a/tools/data_source/gbrowse_filter_code.py Fri Dec 12 14:33:51 2008 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,34 +0,0 @@ -# Code for direct connection to GMOD -from galaxy.datatypes import sniff -import urllib - -import logging -log = logging.getLogger( __name__ ) - -def exec_before_job( app, inp_data, out_data, param_dict, tool=None ): - """Sets the attributes of the data""" - gb_settings = urllib.unquote( param_dict.get( 't', None ) ) # t=CG+TS+ESTB+SAGE+EXPR+EXPR_PATTERN+SNPs+PolyA+BLASTX+LINK+ETILE - gb_landmark_region = urllib.unquote( param_dict.get( 'q' ) ) # q=IV:6070000..6100000& - gb_land_mark, gb_region = gb_landmark_region.split( ':' ) - items = out_data.items() - for name, data in items: - data.name = "%s on %s" % ( data.name, gb_landmark_region ) - data.dbkey = param_dict.get( 'dbkey', '?' ) - # Store GMOD / GBrowse parameters temporarily in output file - out = open( data.file_name, 'w' ) - for key, value in param_dict.items(): - out.write( "%s\t%s\n" % ( key, value ) ) - out.close() - out_data[ name ] = data - -def exec_after_process( app, inp_data, out_data, param_dict, tool=None, stdout=None, stderr=None ): - """Verifies the data after the run""" - name, data = out_data.items()[0] - data.set_size() - if data.state == data.states.OK: - data.info = data.name - if data.extension == 'txt': - data_type = sniff.guess_ext( data.file_name, sniff_order=app.datatypes_registry.sniff_order ) - data = app.datatypes_registry.change_datatype( data, data_type ) - data.set_peek() - data.flush()

1 0

what is the status of GALAXY integration with GBrowse?
by David M. Goodstein 12 Dec '08

12 Dec '08

Specifically the ability to pull and push data to/from Galaxy to a Gbrowser? thx, --David David M. Goodstein Joint Genome Institute / Lawrence Berkeley National Lab Center for Integrative Genomics / UCBerkeley

3 3

[hg] galaxy 1663: Fix for showing deleted datasets in a history, ...
by Greg Von Kuster 12 Dec '08

12 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/0af3897ce585 changeset: 1663:0af3897ce585 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Fri Dec 12 14:33:51 2008 -0500 description: Fix for showing deleted datasets in a history, also yet another fix for the new users report, users are now sorted descending by last login, as the report states. 4 file(s) affected in this change: lib/galaxy/webapps/reports/controllers/users.py lib/galaxy/webapps/reports/templates/users_last_access_date.mako templates/history/options.mako templates/root/history.mako diffs (73 lines): diff -r d53da5b089a8 -r 0af3897ce585 lib/galaxy/webapps/reports/controllers/users.py --- a/lib/galaxy/webapps/reports/controllers/users.py Fri Dec 12 12:12:59 2008 -0500 +++ b/lib/galaxy/webapps/reports/controllers/users.py Fri Dec 12 14:33:51 2008 -0500 @@ -1,6 +1,6 @@ from datetime import datetime, timedelta from time import strftime -import calendar +import calendar, operator from galaxy.webapps.reports.base.controller import * import galaxy.model from galaxy.model.orm import * @@ -112,6 +112,7 @@ else: # The user has never logged in users.append( ( user.email, "never logged in" ) ) + users = sorted( users, key=operator.itemgetter( 1 ), reverse=True ) return trans.fill_template( 'users_last_access_date.mako', users=users, not_logged_in_for_days=not_logged_in_for_days, diff -r d53da5b089a8 -r 0af3897ce585 lib/galaxy/webapps/reports/templates/users_last_access_date.mako --- a/lib/galaxy/webapps/reports/templates/users_last_access_date.mako Fri Dec 12 12:12:59 2008 -0500 +++ b/lib/galaxy/webapps/reports/templates/users_last_access_date.mako Fri Dec 12 14:33:51 2008 -0500 @@ -13,7 +13,12 @@ <td> <form method="post" controller="users" action="last_access_date"> <p> - Users that have not logged in to Galaxy for + %if users: + ${len( users ) } + %else: + 0 + %endif +  users have not logged in to Galaxy for <input type="textfield" value="${not_logged_in_for_days}" size="3" name="not_logged_in_for_days"> days.  <button name="action" value="not_logged_in_for_days">Go</button> </p> diff -r d53da5b089a8 -r 0af3897ce585 templates/history/options.mako --- a/templates/history/options.mako Fri Dec 12 12:12:59 2008 -0500 +++ b/templates/history/options.mako Fri Dec 12 14:33:51 2008 -0500 @@ -11,15 +11,15 @@ <ul> %if user: - <li><a href="${h.url_for( action='history_rename', id=history.id )}" target="galaxy_main">Rename</a> current history (stored as "${history.name}")</li> - <li><a href="${h.url_for( action='history_available')}" target="galaxy_main">List</a> previously stored histories</li> + <li><a href="${h.url_for( controller='root', action='history_rename', id=history.id )}" target="galaxy_main">Rename</a> current history (stored as "${history.name}")</li> + <li><a href="${h.url_for( controller='root', action='history_available')}" target="galaxy_main">List</a> previously stored histories</li> %if len( history.active_datasets ) > 0: - <li><a href="${h.url_for('/history_new')}">Create</a> a new empty history</li> + <li><a href="${h.url_for( controller='root', action='history_new' )}">Create</a> a new empty history</li> %endif <li><a href="${h.url_for( controller='workflow', action='build_from_current_history' )}">Construct workflow</a> from the current history</li> - <li><a href="${h.url_for( action='history_share' )}" target="galaxy_main">Share</a> current history</div> + <li><a href="${h.url_for( controller='root', action='history_share' )}" target="galaxy_main">Share</a> current history</div> %endif - <li><a href="${h.url_for( action='history', show_deleted=True)}" target="galaxy_history">Show deleted</a> datasets in history</li> - <li><a href="${h.url_for( action='history_delete', id=history.id )}" confirm="Are you sure you want to delete the current history?">Delete</a> current history</div> + <li><a href="${h.url_for( controller='root', action='history', show_deleted=True)}" target="galaxy_history">Show deleted</a> datasets in history</li> + <li><a href="${h.url_for( controller='root', action='history_delete', id=history.id )}" confirm="Are you sure you want to delete the current history?">Delete</a> current history</div> </ul> diff -r d53da5b089a8 -r 0af3897ce585 templates/root/history.mako --- a/templates/root/history.mako Fri Dec 12 12:12:59 2008 -0500 +++ b/templates/root/history.mako Fri Dec 12 14:33:51 2008 -0500 @@ -248,7 +248,7 @@ <%namespace file="history_common.mako" import="render_dataset" /> -%if ( show_deleted and len( history.datasets ) < 1 ) or len( history.active_datasets ) < 1: +%if ( show_deleted and not history.activatable_datasets ) or ( not show_deleted and not history.active_datasets ): <div class="infomessagesmall" id="emptyHistoryMessage"> %else: <%

1 0