# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Nate Coraor <nate@bx.psu.edu> # Date 1288969490 14400 # Node ID 7e9ae46051057d822aff82a9b1f3aec0200086f2 # Parent cdd6ce1f38b0e22802ed63e3e0cb4f6a58cb93b2 Add a new "Files uploaded via FTP" grid to the upload form and related parameter types, form fields, etc. --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -1,6 +1,6 @@ <?xml version="1.0"?> -<tool name="Upload File" id="upload1" version="1.1.1"> +<tool name="Upload File" id="upload1" version="1.1.2"><description> from your computer </description> @@ -29,6 +29,7 @@ <validator type="expression" message="You will need to reselect the file you specified (%s)." substitute_value_in_message="True">not ( ( isinstance( value, unicode ) or isinstance( value, str ) ) and value != "" )</validator><!-- use validator to post message to user about needing to reselect the file, since most browsers won't accept the value attribute for file inputs --></param><param name="url_paste" type="text" area="true" size="5x35" label="URL/Text" help="Here you may specify a list of URLs (one per line) or paste the contents of a file."/> + <param name="ftp_files" type="ftpfile" label="Files uploaded via FTP"/><param name="space_to_tab" type="select" display="checkboxes" multiple="True" label="Convert spaces to tabs" help="Use this option if you are entering intervals by hand."><option value="Yes">Yes</option></param> --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -2,9 +2,10 @@ Classes for generating HTML forms """ -import logging,sys +import logging, sys, os, time from cgi import escape -from galaxy.util import restore_text +from galaxy.util import restore_text, relpath, nice_size +from galaxy.web import url_for log = logging.getLogger(__name__) @@ -145,6 +146,68 @@ class FileField(BaseField): ajax_text = ' galaxy-ajax-upload="true"' return '<input type="file" name="%s%s"%s%s>' % ( prefix, self.name, ajax_text, value_text ) +class FTPFileField(BaseField): + """ + An FTP file upload input. + """ + thead = ''' + <table id="grid-table" class="grid"> + <thead id="grid-table-header"> + <tr> + <th id="select-header"></th> + <th id="name-header"> + File + </th> + <th id="size-header"> + Size + </th> + <th id="date-header"> + Date + </th> + </tr> + </thead> + <tbody id="grid-table-body"> + ''' + trow = ''' + <tr> + <td><input type="checkbox" name="%s%s" value="%s"/></td> + <td>%s</td> + <td>%s</td> + <td>%s</td> + </tr> + ''' + tfoot = ''' + </tbody> + </table> + ''' + def __init__( self, name, dir, ftp_site, value = None ): + self.name = name + self.dir = dir + self.ftp_site = ftp_site + self.value = value + def get_html( self, prefix="" ): + rval = FTPFileField.thead + if self.dir is None: + rval += '<tr><td colspan="3"><em>Please <a href="%s">create</a> or <a href="%s">log in to</a> a Galaxy account to view files uploaded via FTP.</em></td></tr>' % ( url_for( controller='user', action='create', referer=url_for( controller='root' ) ), url_for( controller='user', action='login', referer=url_for( controller='root' ) ) ) + elif not os.path.exists( self.dir ): + rval += '<tr><td colspan="3"><em>Your FTP upload directory contains no files.</em></td></tr>' + else: + uploads = [] + for ( dirpath, dirnames, filenames ) in os.walk( self.dir ): + for filename in filenames: + path = relpath( os.path.join( dirpath, filename ), self.dir ) + statinfo = os.lstat( os.path.join( dirpath, filename ) ) + uploads.append( dict( path=path, + size=nice_size( statinfo.st_size ), + ctime=time.strftime( "%m/%d/%Y %I:%M:%S %p", time.localtime( statinfo.st_ctime ) ) ) ) + if not uploads: + rval += '<tr><td colspan="3"><em>Your FTP upload directory contains no files.</em></td></tr>' + for upload in uploads: + rval += FTPFileField.trow % ( prefix, self.name, upload['path'], upload['path'], upload['size'], upload['ctime'] ) + rval += FTPFileField.tfoot + rval += '<div class="toolParamHelp">This Galaxy server allows you to upload files via FTP. To upload some files, log in to the FTP server at <strong>%s</strong> using your Galaxy credentials (email address and password).</div>' % self.ftp_site + return rval + class HiddenField(BaseField): """ A hidden field. --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -40,6 +40,11 @@ class ToolParameter( object ): for elem in param.findall("validator"): self.validators.append( validation.Validator.from_element( self, elem ) ) + @property + def visible( self ): + """Return true if the parameter should be rendered on the form""" + return True + def get_label( self ): """Return user friendly name for the parameter""" if self.label: return self.label @@ -362,6 +367,41 @@ class FileToolParameter( ToolParameter ) def get_initial_value( self, trans, context ): return None +class FTPFileToolParameter( ToolParameter ): + """ + Parameter that takes a file uploaded via FTP as a value. + """ + def __init__( self, tool, elem ): + """ + Example: C{<param name="bins" type="file" />} + """ + ToolParameter.__init__( self, tool, elem ) + @property + def visible( self ): + if self.tool.app.config.ftp_upload_dir is None or self.tool.app.config.ftp_upload_site is None: + return False + return True + def get_html_field( self, trans=None, value=None, other_values={} ): + if trans is None or trans.user is None: + user_ftp_dir = None + else: + user_ftp_dir = os.path.join( trans.app.config.ftp_upload_dir, trans.user.email ) + return form_builder.FTPFileField( self.name, user_ftp_dir, trans.app.config.ftp_upload_site, value = value ) + def from_html( self, value, trans=None, other_values={} ): + return util.listify( value ) + def to_string( self, value, app ): + if value in [ None, '' ]: + return None + elif isinstance( value, unicode ) or isinstance( value, str ) or isinstance( value, list ): + return value + def to_python( self, value, app ): + if value is None: + return None + elif isinstance( value, unicode ) or isinstance( value, str ) or isinstance( value, list ): + return value + def get_initial_value( self, trans, context ): + return None + class HiddenToolParameter( ToolParameter ): """ Parameter that takes one of two values. @@ -1427,6 +1467,7 @@ parameter_types = dict( text = Te hidden = HiddenToolParameter, baseurl = BaseURLToolParameter, file = FileToolParameter, + ftpfile = FTPFileToolParameter, data = DataToolParameter, drill_down = DrillDownSelectToolParameter ) --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -480,6 +480,32 @@ def umask_fix_perms( path, umask, unmask current_group, e ) ) +def nice_size(size): + """ + Returns a readably formatted string with the size + + >>> nice_size(100) + '100.0 bytes' + >>> nice_size(10000) + '9.8 Kb' + >>> nice_size(1000000) + '976.6 Kb' + >>> nice_size(100000000) + '95.4 Mb' + """ + words = [ 'bytes', 'Kb', 'Mb', 'Gb' ] + try: + size = float( size ) + except: + return '??? bytes' + for ind, word in enumerate(words): + step = 1024 ** (ind + 1) + if step > size: + size = size / float(1024 ** ind) + out = "%.1f %s" % (size, word) + return out + return '??? bytes' + galaxy_root_path = os.path.join(__path__[0], "..","..","..") # The dbnames list is used in edit attributes and the upload tool dbnames = read_dbnames( os.path.join( galaxy_root_path, "tool-data", "shared", "ucsc", "builds.txt" ) ) --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -324,6 +324,18 @@ use_interactive = True # Enable the (experimental! beta!) Web API. Documentation forthcoming. #enable_api = False +# Enable Galaxy's "Upload via FTP" interface. You'll need to install and +# configure an FTP server (we've used ProFTPd since it can use Galaxy's +# database for authentication) and set the following two options. + +# This should point to a directory containing subdirectories matching users' +# email addresses, where Galaxy will look for files. +#ftp_upload_dir = None + +# This should be the hostname of your FTP server, which will be provided to +# users in the help text. +#ftp_upload_site = None + # -- Job Execution # If running multiple Galaxy processes, one can be designated as the job --- a/templates/tool_form.mako +++ b/templates/tool_form.mako @@ -51,7 +51,9 @@ function checkUncheckAll( name, check ) <%def name="do_inputs( inputs, tool_state, errors, prefix, other_values=None )"><% other_values = ExpressionContext( tool_state, other_values ) %> %for input_index, input in enumerate( inputs.itervalues() ): - %if input.type == "repeat": + %if not input.visible: + <% pass %> + %elif input.type == "repeat": <div class="repeat-group"><div class="form-title-row"><b>${input.title_plural}</b></div><% repeat_state = tool_state[input.name] %> --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -929,7 +929,11 @@ class Tool: assert isinstance( out_data, odict ) return 'tool_executed.mako', dict( out_data=out_data ) except: - return 'message.mako', dict( status='error', message='odict not returned from tool execution', refresh_frames=[] ) + if isinstance( out_data, str ): + message = out_data + else: + message = 'Failure executing tool (odict not returned from tool execution)' + return 'message.mako', dict( status='error', message=message, refresh_frames=[] ) # Otherwise move on to the next page else: state.page += 1 --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -12,11 +12,14 @@ import StringIO, os, urllib from galaxy.datatypes import sniff from galaxy.util.bunch import Bunch from galaxy.util.odict import odict -from galaxy.util import json +from galaxy.util import json, relpath class Group( object ): def __init__( self ): self.name = None + @property + def visible( self ): + return True def value_to_basic( self, value, app ): """ Convert value to a (possibly nested) representation using only basic @@ -267,6 +270,7 @@ class UploadDataset( Group ): rval = [] data_file = context['file_data'] url_paste = context['url_paste'] + ftp_files = context['ftp_files'] name = context.get( 'NAME', None ) info = context.get( 'INFO', None ) space_to_tab = False @@ -281,6 +285,31 @@ class UploadDataset( Group ): if file_bunch.path: file_bunch.space_to_tab = space_to_tab rval.append( file_bunch ) + # look for files uploaded via FTP + valid_files = [] + if ftp_files: + if trans.user is None: + log.warning( 'Anonymous user passed values in ftp_files: %s' % ftp_files ) + ftp_files = [] + # TODO: warning to the user (could happen if session has become invalid) + else: + user_ftp_dir = os.path.join( trans.app.config.ftp_upload_dir, trans.user.email ) + for ( dirpath, dirnames, filenames ) in os.walk( user_ftp_dir ): + for filename in filenames: + path = relpath( os.path.join( dirpath, filename ), user_ftp_dir ) + if not os.path.islink( os.path.join( dirpath, filename ) ): + valid_files.append( path ) + for ftp_file in ftp_files: + if ftp_file not in valid_files: + log.warning( 'User passed an invalid file path in ftp_files: %s' % ftp_file ) + continue + # TODO: warning to the user (could happen if file is already imported) + ftp_data_file = { 'local_filename' : os.path.abspath( os.path.join( user_ftp_dir, ftp_file ) ), + 'filename' : os.path.basename( ftp_file ) } + file_bunch = get_data_file_filename( ftp_data_file, override_name = name, override_info = info ) + if file_bunch.path: + file_bunch.space_to_tab = space_to_tab + rval.append( file_bunch ) return rval file_type = self.get_file_type( context ) d_type = self.get_datatype( trans, context ) --- a/static/june_2007_style/blue/base.css +++ b/static/june_2007_style/blue/base.css @@ -39,8 +39,7 @@ div.form-title-row{padding:5px 10px;} div.repeat-group-item{border-left:solid #d8b365 5px;margin-left:10px;margin-bottom:10px;} div.form-row-error{background:#FFCCCC;} div.form-row label{font-weight:bold;display:block;margin-bottom:.2em;} -div.form-row-input{float:left;width:300px;} -div.form-row-input > input{max-width:300px;} +div.form-row-input{float:left;} div.form-row-input label{font-weight:normal;display:inline;} div.form-row-error-message{width:300px;float:left;color:red;font-weight:bold;padding:3px 0 0 1em;} select,input,textarea{font:inherit;font-size:115%;} --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -419,36 +419,16 @@ class Newick( Text ): # ------------- Utility methods -------------- +# nice_size used to be here, but to resolve cyclical dependencies it's been +# moved to galaxy.util. It belongs there anyway since it's used outside +# datatypes. +nice_size = util.nice_size + def get_test_fname( fname ): """Returns test data filename""" path, name = os.path.split(__file__) full_path = os.path.join( path, 'test', fname ) return full_path -def nice_size(size): - """ - Returns a readably formatted string with the size - - >>> nice_size(100) - '100.0 bytes' - >>> nice_size(10000) - '9.8 Kb' - >>> nice_size(1000000) - '976.6 Kb' - >>> nice_size(100000000) - '95.4 Mb' - """ - words = [ 'bytes', 'Kb', 'Mb', 'Gb' ] - try: - size = float( size ) - except: - return '??? bytes' - for ind, word in enumerate(words): - step = 1024 ** (ind + 1) - if step > size: - size = size / float(1024 ** ind) - out = "%.1f %s" % (size, word) - return out - return '??? bytes' def get_file_peek( file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5 ): """ Returns the first LINE_COUNT lines wrapped to WIDTH --- a/lib/galaxy/tools/actions/upload.py +++ b/lib/galaxy/tools/actions/upload.py @@ -21,7 +21,7 @@ class UploadToolAction( ToolAction ): upload_common.cleanup_unused_precreated_datasets( precreated_datasets ) if not uploaded_datasets: - return 'No data was entered in the upload form, please go back and choose data to upload.' + return None, 'No data was entered in the upload form, please go back and choose data to upload.' json_file_path = upload_common.create_paramfile( trans, uploaded_datasets ) data_list = [ ud.data for ud in uploaded_datasets ] --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -99,6 +99,10 @@ class Configuration( object ): self.user_library_import_dir = kwargs.get( 'user_library_import_dir', None ) if self.user_library_import_dir is not None and not os.path.exists( self.user_library_import_dir ): raise ConfigurationError( "user_library_import_dir specified in config (%s) does not exist" % self.user_library_import_dir ) + self.ftp_upload_dir = kwargs.get( 'ftp_upload_dir', None ) + if self.ftp_upload_dir is not None and not os.path.exists( self.ftp_upload_dir ): + os.makedirs( self.ftp_upload_dir ) + self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None ) self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False ) self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' ) # Location for dependencies --- a/static/june_2007_style/base.css.tmpl +++ b/static/june_2007_style/base.css.tmpl @@ -212,11 +212,6 @@ div.form-row label { div.form-row-input { float: left; - width: 300px; -} - -div.form-row-input > input { - max-width: 300px; } div.form-row-input label {