details: http://www.bx.psu.edu/hg/galaxy/rev/affd3085eee3 changeset: 1741:affd3085eee3 user: Greg Von Kuster greg@bx.psu.edu date: Fri Mar 20 02:49:30 2009 -0400 description: Fixes for i18n support - all charsets set to utf-8, now able to paste multi-byte chars in upload form, templates use utf-8 as output_charset, other miscellaneous fixes.

21 file(s) affected in this change:

lib/galaxy/datatypes/sniff.py lib/galaxy/tools/actions/upload.py lib/galaxy/util/__init__.py lib/galaxy/web/controllers/ucsc_proxy.py lib/galaxy/web/framework/__init__.py lib/galaxy/web/framework/base.py lib/galaxy/webapps/reports/templates/base_panels.mako lib/galaxy/webapps/reports/templates/masthead.mako scripts/paster.py templates/base.mako templates/base_panels.mako templates/dataset/errors.tmpl templates/dataset/validation.tmpl templates/root/history.mako templates/root/masthead.mako templates/root/tool_menu.mako templates/tool_executed.mako templates/tool_form.mako templates/workflow/build_from_current_history.mako tools/data_source/wormbase.xml tools/data_source/wormbase_test.xml

diffs (440 lines):

diff -r a5a995ca3abc -r affd3085eee3 lib/galaxy/datatypes/sniff.py --- a/lib/galaxy/datatypes/sniff.py Thu Mar 19 21:41:06 2009 -0400 +++ b/lib/galaxy/datatypes/sniff.py Fri Mar 20 02:49:30 2009 -0400 @@ -21,7 +21,8 @@ chunk = stream.read(1048576) if not chunk: break - os.write(fd, chunk) + # TODO: does this work on binary files? + os.write( fd, chunk.encode( "utf-8" ) ) os.close(fd) return temp_name

diff -r a5a995ca3abc -r affd3085eee3 lib/galaxy/tools/actions/upload.py --- a/lib/galaxy/tools/actions/upload.py Thu Mar 19 21:41:06 2009 -0400 +++ b/lib/galaxy/tools/actions/upload.py Fri Mar 20 02:49:30 2009 -0400 @@ -105,7 +105,7 @@ try: data_list.append( self.add_file( trans, temp_name, 'Pasted Entry', file_type, dbkey, info="pasted entry", space_to_tab=space_to_tab ) ) except Exception, e: - log.excception( 'exception in add_file using StringIO.StringIO( url_paste ) temp_name %s: %s' % ( str( temp_name ), str( e ) ) ) + log.exception( 'exception in add_file using StringIO.StringIO( url_paste ) temp_name %s: %s' % ( str( temp_name ), str( e ) ) ) self.remove_tempfile( temp_name ) return self.upload_empty( trans, job, "Error:", str( e ) ) else: @@ -146,12 +146,11 @@

def add_file( self, trans, temp_name, file_name, file_type, dbkey, info=None, space_to_tab=False ): data_type = None - # See if we have an empty file if not os.path.getsize( temp_name ) > 0: raise BadFileException( "you attempted to upload an empty file." ) - - # See if we have a gzipped file, which, if it passes our restrictions, we'll decompress on the fly. + # See if we have a gzipped file, which, if it passes our restrictions, + # we'll decompress on the fly. is_gzipped, is_valid = self.check_gzip( temp_name ) if is_gzipped and not is_valid: raise BadFileException( "you attempted to upload an inappropriate file." ) @@ -176,7 +175,7 @@ shutil.move( uncompressed, temp_name ) file_name = file_name.rstrip( '.gz' ) data_type = 'gzip' - + ext = '' if not data_type: # See if we have a zip archive is_zipped, is_valid, test_ext = self.check_zip( temp_name ) @@ -193,30 +192,28 @@ raise BadFileException( "you must manually set the 'File Format' to either 'Binseq.zip' or 'Txtseq.zip' when uploading zip files." ) data_type = 'zip' ext = file_type - if not data_type: if self.check_binary( temp_name ): - ext = file_name.split( "." )[1].strip().lower() - if not( ext == 'ab1' or ext == 'scf' ): - raise BadFileException( "you attempted to upload an inappropriate file." ) - if ext == 'ab1' and file_type != 'ab1': - raise BadFileException( "you must manually set the 'File Format' to 'Ab1' when uploading ab1 files." ) - elif ext == 'scf' and file_type != 'scf': - raise BadFileException( "you must manually set the 'File Format' to 'Scf' when uploading scf files." ) + parts = file_name.split( "." ) + if len( parts ) > 1: + ext = parts[1].strip().lower() + if not( ext == 'ab1' or ext == 'scf' ): + raise BadFileException( "you attempted to upload an inappropriate file." ) + if ext == 'ab1' and file_type != 'ab1': + raise BadFileException( "you must manually set the 'File Format' to 'Ab1' when uploading ab1 files." ) + elif ext == 'scf' and file_type != 'scf': + raise BadFileException( "you must manually set the 'File Format' to 'Scf' when uploading scf files." ) data_type = 'binary' - if not data_type: # We must have a text file if self.check_html( temp_name ): raise BadFileException( "you attempted to upload an inappropriate file." ) - if data_type != 'binary' and data_type != 'zip': if space_to_tab: self.line_count = sniff.convert_newlines_sep2tabs( temp_name ) else: self.line_count = sniff.convert_newlines( temp_name ) if file_type == 'auto': - log.debug("In upload, in if file_type == 'auto':") ext = sniff.guess_ext( temp_name, sniff_order=trans.app.datatypes_registry.sniff_order ) else: ext = file_type @@ -312,7 +309,6 @@ if chunk is None: temp.close() return False - def check_binary( self, temp_name, chunk=None ): if chunk is None: temp = open( temp_name, "U" ) diff -r a5a995ca3abc -r affd3085eee3 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py Thu Mar 19 21:41:06 2009 -0400 +++ b/lib/galaxy/util/__init__.py Fri Mar 20 02:49:30 2009 -0400 @@ -107,9 +107,9 @@

def sanitize_param(value): """Clean incoming parameters (strings or lists)""" - if type(value) == type('x'): + if isinstance( value, basestring ): return sanitize_text(value) - elif type(value) == type([]): + elif isinstance( value, list ): return map(sanitize_text, value) else: raise Exception, 'Unknown parameter type' diff -r a5a995ca3abc -r affd3085eee3 lib/galaxy/web/controllers/ucsc_proxy.py --- a/lib/galaxy/web/controllers/ucsc_proxy.py Thu Mar 19 21:41:06 2009 -0400 +++ b/lib/galaxy/web/controllers/ucsc_proxy.py Fri Mar 20 02:49:30 2009 -0400 @@ -121,7 +121,7 @@

<head> <title>Galaxy</title> -<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" /> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <link href="/static/style/base.css" rel="stylesheet" type="text/css" /> <script language="javascript" type="text/javascript"> function changeTarget(target) diff -r a5a995ca3abc -r affd3085eee3 lib/galaxy/web/framework/__init__.py --- a/lib/galaxy/web/framework/__init__.py Thu Mar 19 21:41:06 2009 -0400 +++ b/lib/galaxy/web/framework/__init__.py Fri Mar 20 02:49:30 2009 -0400 @@ -87,7 +87,8 @@ self.mako_template_lookup = mako.lookup.TemplateLookup( directories = [ galaxy_app.config.template_path ] , module_directory = galaxy_app.config.template_cache, - collection_size = 500 ) + collection_size = 500, + output_encoding = 'utf-8' ) # Security helper self.security = galaxy_app.security def handle_controller_exception( self, e, trans, **kwargs ): @@ -131,7 +132,7 @@ # Default to English locales = 'en' t = Translations.load( dirname='locale', locales=locales, domain='ginga' ) - self.template_context.update ( dict( _=t.gettext, n_=t.gettext, N_=t.ngettext ) ) + self.template_context.update ( dict( _=t.ugettext, n_=t.ugettext, N_=t.ungettext ) ) @property def sa_session( self ): """ @@ -465,10 +466,12 @@ return self.fill_template_mako( filename, **kwargs ) else: template = Template( file=os.path.join(self.app.config.template_path, filename), - searchList=[kwargs, self.template_context, dict(caller=self, t=self, h=webhelpers, util=util, request=self.request, response=self.response, app=self.app)] ) + searchList=[kwargs, self.template_context, dict(caller=self, t=self, h=webhelpers, util=util, request=self.request, response=self.response, app=self.app)], + output_encoding='utf-8' ) return str( template ) def fill_template_mako( self, filename, **kwargs ): template = self.webapp.mako_template_lookup.get_template( filename ) + template.output_encoding = 'utf-8' data = dict( caller=self, t=self, trans=self, h=webhelpers, util=util, request=self.request, response=self.response, app=self.app ) data.update( self.template_context ) data.update( kwargs ) @@ -477,7 +480,9 @@ """ Fill in a template, putting any keyword arguments on the context. """ - template = Template( source=template_string, searchList=[context or kwargs, dict(caller=self)] ) + template = Template( source=template_string, + searchList=[context or kwargs, dict(caller=self)], + output_encoding='utf-8' ) return str(template)

class FormBuilder( object ): diff -r a5a995ca3abc -r affd3085eee3 lib/galaxy/web/framework/base.py --- a/lib/galaxy/web/framework/base.py Thu Mar 19 21:41:06 2009 -0400 +++ b/lib/galaxy/web/framework/base.py Fri Mar 20 02:49:30 2009 -0400 @@ -14,8 +14,10 @@ pkg_resources.require( "Paste" ) pkg_resources.require( "Routes" ) pkg_resources.require( "flup" ) +pkg_resources.require( "WebOb" )

import routes +import webob

# We will use some very basic HTTP/wsgi utilities from the paste library from paste.request import parse_headers, get_cookies, parse_formvars @@ -212,7 +214,7 @@ else: return None

-class Request( object ): +class Request( webob.Request ): """ Encapsulates an HTTP request. """ @@ -220,26 +222,24 @@ """ Create a new request wrapping the WSGI environment `environ` """ - self.environ = environ + ## self.environ = environ + webob.Request.__init__( self, environ, charset='utf-8', decode_param_names=False ) # Properties that are computed and cached on first use @lazy_property def remote_host( self ): try: - return socket.gethostbyname( self.environ['REMOTE_ADDR'] ) + return socket.gethostbyname( self.remote_addr ) except socket.error: - return self.environ['REMOTE_ADDR'] - @lazy_property - def headers( self ): - return dict( parse_headers( self.environ ) ) + return self.remote_addr @lazy_property def cookies( self ): return get_cookies( self.environ ) @lazy_property def base( self ): - return ( self.scheme + "://" + self.environ['HTTP_HOST'] ) - @lazy_property - def params( self ): - return parse_formvars( self.environ ) + return ( self.scheme + "://" + self.host ) + ## @lazy_property + ## def params( self ): + ## return parse_formvars( self.environ ) @lazy_property def path( self ): return self.environ['SCRIPT_NAME'] + self.environ['PATH_INFO'] @@ -247,15 +247,15 @@ def browser_url( self ): return self.base + self.path # Descriptors that map properties to the associated environment - scheme = WSGIEnvironmentProperty( 'wsgi.url_scheme' ) - remote_addr = WSGIEnvironmentProperty( 'REMOTE_ADDR' ) + ## scheme = WSGIEnvironmentProperty( 'wsgi.url_scheme' ) + ## remote_addr = WSGIEnvironmentProperty( 'REMOTE_ADDR' ) remote_port = WSGIEnvironmentProperty( 'REMOTE_PORT' ) - method = WSGIEnvironmentProperty( 'REQUEST_METHOD' ) - script_name = WSGIEnvironmentProperty( 'SCRIPT_NAME' ) + ## method = WSGIEnvironmentProperty( 'REQUEST_METHOD' ) + ## script_name = WSGIEnvironmentProperty( 'SCRIPT_NAME' ) protocol = WSGIEnvironmentProperty( 'SERVER_PROTOCOL' ) - query_string = WSGIEnvironmentProperty( 'QUERY_STRING' ) - path_info = WSGIEnvironmentProperty( 'PATH_INFO' ) - + ## query_string = WSGIEnvironmentProperty( 'QUERY_STRING' ) + ## path_info = WSGIEnvironmentProperty( 'PATH_INFO' ) + class Response( object ): """ Describes an HTTP response. Currently very simple since the actual body @@ -298,7 +298,7 @@ return "%d %s" % ( exception.code, exception.title ) else: return self.status - + # ---- Utilities ------------------------------------------------------------

CHUNK_SIZE = 2**16 diff -r a5a995ca3abc -r affd3085eee3 lib/galaxy/webapps/reports/templates/base_panels.mako --- a/lib/galaxy/webapps/reports/templates/base_panels.mako Thu Mar 19 21:41:06 2009 -0400 +++ b/lib/galaxy/webapps/reports/templates/base_panels.mako Fri Mar 20 02:49:30 2009 -0400 @@ -31,7 +31,7 @@ <html lang="en"> <head> <title>${self.title()}</title> - <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" /> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> ${self.javascripts()} ${self.stylesheets()} </head> diff -r a5a995ca3abc -r affd3085eee3 lib/galaxy/webapps/reports/templates/masthead.mako --- a/lib/galaxy/webapps/reports/templates/masthead.mako Thu Mar 19 21:41:06 2009 -0400 +++ b/lib/galaxy/webapps/reports/templates/masthead.mako Fri Mar 20 02:49:30 2009 -0400 @@ -2,7 +2,7 @@ <html> <head> <title>Galaxy</title> - <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" /> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <link href="${h.url_for('/static/style/base.css')}" rel="stylesheet" type="text/css" /> <link href="${h.url_for('/static/style/masthead.css')}" rel="stylesheet" type="text/css" /> </head> diff -r a5a995ca3abc -r affd3085eee3 scripts/paster.py --- a/scripts/paster.py Thu Mar 19 21:41:06 2009 -0400 +++ b/scripts/paster.py Fri Mar 20 02:49:30 2009 -0400 @@ -7,7 +7,6 @@

import os, sys

-sys.setdefaultencoding( 'utf_8' ) assert sys.version_info[:2] >= ( 2, 4 )

new_path = [ os.path.join( os.getcwd(), "lib" ) ] diff -r a5a995ca3abc -r affd3085eee3 templates/base.mako --- a/templates/base.mako Thu Mar 19 21:41:06 2009 -0400 +++ b/templates/base.mako Fri Mar 20 02:49:30 2009 -0400 @@ -4,7 +4,7 @@

<head> <title>a ${self.title()}</title> -<meta http-equiv="Content-Type" content="text/html; charset=${_('iso-8859-1')}" /> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> ${self.stylesheets()} ${self.javascripts()} </head> diff -r a5a995ca3abc -r affd3085eee3 templates/base_panels.mako --- a/templates/base_panels.mako Thu Mar 19 21:41:06 2009 -0400 +++ b/templates/base_panels.mako Fri Mar 20 02:49:30 2009 -0400 @@ -42,7 +42,7 @@ <html> <head> <title>${self.title()}</title> - <meta http-equiv="Content-Type" content="text/html; charset=${_('iso-8859-1')}" /> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> ${self.javascripts()} ${self.stylesheets()} </head> diff -r a5a995ca3abc -r affd3085eee3 templates/dataset/errors.tmpl --- a/templates/dataset/errors.tmpl Thu Mar 19 21:41:06 2009 -0400 +++ b/templates/dataset/errors.tmpl Fri Mar 20 02:49:30 2009 -0400 @@ -2,7 +2,7 @@ <html> <head> <title>Dataset generation errors</title> - <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" /> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <link href="/static/style/base.css" rel="stylesheet" type="text/css" /> <style> pre diff -r a5a995ca3abc -r affd3085eee3 templates/dataset/validation.tmpl --- a/templates/dataset/validation.tmpl Thu Mar 19 21:41:06 2009 -0400 +++ b/templates/dataset/validation.tmpl Fri Mar 20 02:49:30 2009 -0400 @@ -3,7 +3,7 @@

<head> <title>Galaxy</title> -<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" /> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <link href="/static/style/base.css" rel="stylesheet" type="text/css" /> <script type="text/javascript" src="/static/universe.js">var dummy1=0;</script> </head> diff -r a5a995ca3abc -r affd3085eee3 templates/root/history.mako --- a/templates/root/history.mako Thu Mar 19 21:41:06 2009 -0400 +++ b/templates/root/history.mako Fri Mar 20 02:49:30 2009 -0400 @@ -11,7 +11,7 @@ <!-- running: do not change this comment, used by TwillTestCase.wait --> %endif

-<meta http-equiv="Content-Type" content="text/html; charset=${_('iso-8859-1')}" /> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <meta http-equiv="Pragma" content="no-cache"> <link href="${h.url_for('/static/style/base.css')}" rel="stylesheet" type="text/css" /> <link href="${h.url_for('/static/style/history.css')}" rel="stylesheet" type="text/css" /> diff -r a5a995ca3abc -r affd3085eee3 templates/root/masthead.mako --- a/templates/root/masthead.mako Thu Mar 19 21:41:06 2009 -0400 +++ b/templates/root/masthead.mako Fri Mar 20 02:49:30 2009 -0400 @@ -5,7 +5,7 @@

<head> <title>Galaxy</title> -<meta http-equiv="Content-Type" content="text/html; charset=${_('iso-8859-1')}" /> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <link href="${h.url_for('/static/style/base.css')}" rel="stylesheet" type="text/css" /> <link href="${h.url_for('/static/style/masthead.css')}" rel="stylesheet" type="text/css" /> </head> diff -r a5a995ca3abc -r affd3085eee3 templates/root/tool_menu.mako --- a/templates/root/tool_menu.mako Thu Mar 19 21:41:06 2009 -0400 +++ b/templates/root/tool_menu.mako Fri Mar 20 02:49:30 2009 -0400 @@ -49,7 +49,7 @@ <html> <head> <title>${_('Galaxy Tools')}</title> - <meta http-equiv="Content-Type" content="text/html; charset=${_('iso-8859-1')}" /> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <link href="${h.url_for('/static/style/base.css')}" rel="stylesheet" type="text/css" /> <link href="${h.url_for('/static/style/tool_menu.css')}" rel="stylesheet" type="text/css" />

diff -r a5a995ca3abc -r affd3085eee3 templates/tool_executed.mako --- a/templates/tool_executed.mako Thu Mar 19 21:41:06 2009 -0400 +++ b/templates/tool_executed.mako Fri Mar 20 02:49:30 2009 -0400 @@ -3,7 +3,7 @@

<head> <title>Galaxy</title> -<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" /> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <link href="${h.url_for('/static/style/base.css')}" rel="stylesheet" type="text/css" /> <script type="text/javascript"> var inside_galaxy_frameset = false; diff -r a5a995ca3abc -r affd3085eee3 templates/tool_form.mako --- a/templates/tool_form.mako Thu Mar 19 21:41:06 2009 -0400 +++ b/templates/tool_form.mako Fri Mar 20 02:49:30 2009 -0400 @@ -9,7 +9,7 @@

<head> <title>Galaxy</title> -<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" /> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <link href="${h.url_for('/static/style/base.css')}" rel="stylesheet" type="text/css" /> <script type='text/javascript' src="${h.url_for('/static/scripts/jquery.js')}"> </script> <script type="text/javascript"> diff -r a5a995ca3abc -r affd3085eee3 templates/workflow/build_from_current_history.mako --- a/templates/workflow/build_from_current_history.mako Thu Mar 19 21:41:06 2009 -0400 +++ b/templates/workflow/build_from_current_history.mako Fri Mar 20 02:49:30 2009 -0400 @@ -6,7 +6,7 @@

<head> <title>Galaxy</title> -<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" /> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <link href="${h.url_for('/static/style/history.css')}" rel="stylesheet" type="text/css" /> <link href="${h.url_for('/static/style/base.css')}" rel="stylesheet" type="text/css" />

diff -r a5a995ca3abc -r affd3085eee3 tools/data_source/wormbase.xml --- a/tools/data_source/wormbase.xml Thu Mar 19 21:41:06 2009 -0400 +++ b/tools/data_source/wormbase.xml Fri Mar 20 02:49:30 2009 -0400 @@ -2,7 +2,7 @@ <tool name="Wormbase" id="wormbase" tool_type="data_source" URL_method="post"> <description>server</description> <command interpreter="python">data_source.py $output</command> - <inputs action="http://www.wormbase.org/db/seq/gbrowse/c_elegans/" check_values="false" method="get" target="_top"> + <inputs action="http://www.wormbase.org/db/seq/gbrowse/c_elegans/" check_values="false" target="_top"> <display>go to Wormbase server $GALAXY_URL</display> <param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=wormbase" /> </inputs> diff -r a5a995ca3abc -r affd3085eee3 tools/data_source/wormbase_test.xml --- a/tools/data_source/wormbase_test.xml Thu Mar 19 21:41:06 2009 -0400 +++ b/tools/data_source/wormbase_test.xml Fri Mar 20 02:49:30 2009 -0400 @@ -2,7 +2,7 @@ <tool name="Wormbase" id="wormbase_test" tool_type="data_source" URL_method="post"> <description>test server</description> <command interpreter="python">data_source.py $output</command> - <inputs action="http://dev.wormbase.org/db/seq/gbrowse/c_elegans/" check_values="false" method="get" target="_top"> + <inputs action="http://dev.wormbase.org/db/seq/gbrowse/c_elegans/" check_values="false" target="_top"> <display>go to Wormbase test server $GALAXY_URL</display> <param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=wormbase_test" /> </inputs>