details: http://www.bx.psu.edu/hg/galaxy/rev/b819249af24d changeset: 2796:b819249af24d user: Nate Coraor <nate@bx.psu.edu> date: Tue Sep 29 17:14:20 2009 -0400 description: Reintroduce nginx upload module support. http://www.grid.net.ru/nginx/upload.en.html The following config variables are added to universe_wsgi.ini: nginx_upload_store = Path to nginx upload store ex: = database/upload_store nginx_upload_path = URL (from root of the Galaxy server) to direct upload POSTs to ex: = /_upload The following nginx config supports such a configuration: location /_upload { upload_store /path/to/galaxy/database/upload_store; upload_pass_form_field "tool_id"; upload_pass_form_field "tool_state"; upload_pass_form_field "async_datasets"; upload_pass_form_field "^files_[0-9]+\|.*"; upload_pass_form_field "file_type"; upload_pass_form_field "dbkey"; upload_pass_form_field "runtool_btn"; upload_pass_form_field "ajax_upload"; upload_pass_form_field "upload_option"; upload_pass_form_field "library_id"; upload_pass_form_field "folder_id"; upload_pass_form_field "message"; upload_pass_form_field "roles"; upload_set_form_field "__${upload_field_name}__is_composite" "true"; upload_set_form_field "__${upload_field_name}__keys" "name path"; upload_set_form_field "${upload_field_name}_name" "$upload_file_name"; upload_set_form_field "${upload_field_name}_path" "$upload_tmp_path"; upload_pass_args on; upload_pass /_upload_done; } location /_upload_done { set $dst /tool_runner/index; if ($args ~ nginx_redir=([^&]+)) { set $dst $1; } rewrite "" $dst; } 10 file(s) affected in this change: lib/galaxy/config.py lib/galaxy/tools/__init__.py lib/galaxy/tools/actions/upload_common.py lib/galaxy/tools/parameters/basic.py lib/galaxy/web/controllers/library.py lib/galaxy/web/controllers/library_admin.py templates/admin/library/upload.mako templates/library/library_dataset_common.mako templates/library/upload.mako tools/data_source/upload.xml diffs (166 lines): diff -r 023ea1a1d38a -r b819249af24d lib/galaxy/config.py --- a/lib/galaxy/config.py Tue Sep 29 13:20:48 2009 -0400 +++ b/lib/galaxy/config.py Tue Sep 29 17:14:20 2009 -0400 @@ -89,9 +89,10 @@ raise ConfigurationError( "user_library_import_dir specified in config (%s) does not exist" % self.user_library_import_dir ) # Configuration options for taking advantage of nginx features self.nginx_x_accel_redirect_base = kwargs.get( 'nginx_x_accel_redirect_base', False ) - self.nginx_upload_location = kwargs.get( 'nginx_upload_store', False ) - if self.nginx_upload_location: - self.nginx_upload_location = os.path.abspath( self.nginx_upload_location ) + self.nginx_upload_store = kwargs.get( 'nginx_upload_store', False ) + self.nginx_upload_path = kwargs.get( 'nginx_upload_path', False ) + if self.nginx_upload_store: + self.nginx_upload_store = os.path.abspath( self.nginx_upload_store ) # Parse global_conf and save the parser global_conf = kwargs.get( 'global_conf', None ) global_conf_parser = ConfigParser.ConfigParser() diff -r 023ea1a1d38a -r b819249af24d lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py Tue Sep 29 13:20:48 2009 -0400 +++ b/lib/galaxy/tools/__init__.py Tue Sep 29 17:14:20 2009 -0400 @@ -464,7 +464,11 @@ if input_elem: # Handle properties of the input form self.check_values = util.string_as_bool( input_elem.get("check_values", "true") ) - self.action = input_elem.get( "action", "/tool_runner/index") + self.nginx_upload = util.string_as_bool( input_elem.get( "nginx_upload", "false" ) ) + if self.nginx_upload and self.app.config.nginx_upload_path: + self.action = input_elem.get( "action", self.app.config.nginx_upload_path + "?nginx_redir=/tool_runner/index" ) + else: + self.action = input_elem.get( "action", "/tool_runner/index") self.target = input_elem.get( "target", "galaxy_main" ) self.method = input_elem.get( "method", "post" ) # Parse the actual parameters diff -r 023ea1a1d38a -r b819249af24d lib/galaxy/tools/actions/upload_common.py --- a/lib/galaxy/tools/actions/upload_common.py Tue Sep 29 13:20:48 2009 -0400 +++ b/lib/galaxy/tools/actions/upload_common.py Tue Sep 29 17:14:20 2009 -0400 @@ -23,6 +23,8 @@ f.file.close() upload_dataset['file_data'] = dict( filename = f.filename, local_filename = local_filename ) + elif type( f ) == dict and 'filename' and 'local_filename' not in f: + raise Exception( 'Uploaded file was encoded in a way not understood by Galaxy.' ) if upload_dataset['url_paste'].strip() != '': upload_dataset['url_paste'] = datatypes.sniff.stream_to_file( StringIO.StringIO( upload_dataset['url_paste'] ), prefix="strio_url_paste_" )[0] else: diff -r 023ea1a1d38a -r b819249af24d lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py Tue Sep 29 13:20:48 2009 -0400 +++ b/lib/galaxy/tools/parameters/basic.py Tue Sep 29 17:14:20 2009 -0400 @@ -304,22 +304,21 @@ def get_html_field( self, trans=None, value=None, other_values={} ): return form_builder.FileField( self.name, ajax = self.ajax, value = value ) def from_html( self, value, trans=None, other_values={} ): - # TODO: Fix nginx upload module support # Middleware or proxies may encode files in special ways (TODO: this # should be pluggable) - #if type( value ) == dict: - # upload_location = self.tool.app.config.nginx_upload_location - # assert upload_location, \ - # "Request appears to have been processed by nginx_upload_module \ - # but Galaxy is not configured to recgonize it" - # # Check that the file is in the right location - # local_filename = os.path.abspath( value['path'] ) - # assert local_filename.startswith( upload_location ), \ - # "Filename provided by nginx is not in correct directory" - # value = Bunch( - # filename = value["name"], - # local_filename = local_filename - # ) + if type( value ) == dict: + upload_store = self.tool.app.config.nginx_upload_store + assert upload_store, \ + "Request appears to have been processed by nginx_upload_module \ + but Galaxy is not configured to recognize it" + # Check that the file is in the right location + local_filename = os.path.abspath( value['path'] ) + assert local_filename.startswith( upload_store ), \ + "Filename provided by nginx is not in correct directory" + value = dict( + filename = value["name"], + local_filename = local_filename + ) return value def get_required_enctype( self ): """ diff -r 023ea1a1d38a -r b819249af24d lib/galaxy/web/controllers/library.py --- a/lib/galaxy/web/controllers/library.py Tue Sep 29 13:20:48 2009 -0400 +++ b/lib/galaxy/web/controllers/library.py Tue Sep 29 17:14:20 2009 -0400 @@ -792,8 +792,13 @@ # Send the current history to the form to enable importing datasets from history to library history = trans.get_history() history.refresh() + # If we're using nginx upload, override the form action + action = web.url_for( controller='library', action='library_dataset_dataset_association' ) + if upload_option == 'upload_file' and trans.app.config.nginx_upload_path: + action = web.url_for( trans.app.config.nginx_upload_path ) + '?nginx_redir=' + action return trans.fill_template( '/library/upload.mako', upload_option=upload_option, + action=action, library_id=library_id, folder_id=folder_id, replace_dataset=replace_dataset, diff -r 023ea1a1d38a -r b819249af24d lib/galaxy/web/controllers/library_admin.py --- a/lib/galaxy/web/controllers/library_admin.py Tue Sep 29 13:20:48 2009 -0400 +++ b/lib/galaxy/web/controllers/library_admin.py Tue Sep 29 17:14:20 2009 -0400 @@ -487,8 +487,13 @@ # Send the current history to the form to enable importing datasets from history to library history = trans.get_history() history.refresh() + # If we're using nginx upload, override the form action + action = web.url_for( controller='library_admin', action='library_dataset_dataset_association' ) + if upload_option == 'upload_file' and trans.app.config.nginx_upload_path: + action = web.url_for( trans.app.config.nginx_upload_path ) + '?nginx_redir=' + action return trans.fill_template( '/admin/library/upload.mako', upload_option=upload_option, + action=action, library_id=library_id, folder_id=folder_id, replace_dataset=replace_dataset, diff -r 023ea1a1d38a -r b819249af24d templates/admin/library/upload.mako --- a/templates/admin/library/upload.mako Tue Sep 29 13:20:48 2009 -0400 +++ b/templates/admin/library/upload.mako Tue Sep 29 17:14:20 2009 -0400 @@ -32,4 +32,4 @@ ${render_msg( msg, messagetype )} %endif -${render_upload_form( 'library_admin', upload_option, library_id, folder_id, replace_dataset, file_formats, dbkeys, roles, history, )} +${render_upload_form( 'library_admin', upload_option, action, library_id, folder_id, replace_dataset, file_formats, dbkeys, roles, history, )} diff -r 023ea1a1d38a -r b819249af24d templates/library/library_dataset_common.mako --- a/templates/library/library_dataset_common.mako Tue Sep 29 13:20:48 2009 -0400 +++ b/templates/library/library_dataset_common.mako Tue Sep 29 17:14:20 2009 -0400 @@ -1,4 +1,4 @@ -<%def name="render_upload_form( controller, upload_option, library_id, folder_id, replace_dataset, file_formats, dbkeys, roles, history )"> +<%def name="render_upload_form( controller, upload_option, action, library_id, folder_id, replace_dataset, file_formats, dbkeys, roles, history )"> <% import os, os.path %> %if upload_option in [ 'upload_file', 'upload_directory' ]: <div class="toolForm" id="upload_library_dataset"> @@ -8,7 +8,7 @@ <div class="toolFormTitle">Upload a directory of files</div> %endif <div class="toolFormBody"> - <form name="upload_library_dataset" action="${h.url_for( controller=controller, action='library_dataset_dataset_association' )}" enctype="multipart/form-data" method="post"> + <form name="upload_library_dataset" action="${action}" enctype="multipart/form-data" method="post"> <input type="hidden" name="tool_id" value="upload1"/> <input type="hidden" name="tool_state" value="None"/> <input type="hidden" name="library_id" value="${library_id}"/> diff -r 023ea1a1d38a -r b819249af24d templates/library/upload.mako --- a/templates/library/upload.mako Tue Sep 29 13:20:48 2009 -0400 +++ b/templates/library/upload.mako Tue Sep 29 17:14:20 2009 -0400 @@ -32,4 +32,4 @@ ${render_msg( msg, messagetype )} %endif -${render_upload_form( 'library', upload_option, library_id, folder_id, replace_dataset, file_formats, dbkeys, roles, history, )} +${render_upload_form( 'library', upload_option, action, library_id, folder_id, replace_dataset, file_formats, dbkeys, roles, history, )} diff -r 023ea1a1d38a -r b819249af24d tools/data_source/upload.xml --- a/tools/data_source/upload.xml Tue Sep 29 13:20:48 2009 -0400 +++ b/tools/data_source/upload.xml Tue Sep 29 17:14:20 2009 -0400 @@ -14,7 +14,7 @@ ${output.dataset.dataset.id}:${output} #end while </command> - <inputs> + <inputs nginx_upload="true"> <param name="file_type" type="select" label="File Format" help="Which format? See help below"> <options from_parameter="tool.app.datatypes_registry.upload_file_formats" transform_lines="[ "%s%s%s" % ( line, self.separator, line ) for line in obj ]"> <column name="value" index="1"/>