[galaxy-commits] galaxy-dist commit 7e9ae4605105: Add a new "Files uploaded via FTP" grid to the upload form and related parameter types, form fields, etc.

20 Nov 2010

# HG changeset patch -- Bitbucket.org
# Project galaxy-dist
# URL http://bitbucket.org/galaxy/galaxy-dist/overview
# User Nate Coraor <nate@bx.psu.edu>
# Date 1288969490 14400
# Node ID 7e9ae46051057d822aff82a9b1f3aec0200086f2
# Parent  cdd6ce1f38b0e22802ed63e3e0cb4f6a58cb93b2
Add a new "Files uploaded via FTP" grid to the upload form and related parameter types, form fields, etc.

--- a/tools/data_source/upload.xml
+++ b/tools/data_source/upload.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0"?>
 
-<tool name="Upload File" id="upload1" version="1.1.1">
+<tool name="Upload File" id="upload1" version="1.1.2"><description>
     from your computer  
   </description>
@@ -29,6 +29,7 @@
         <validator type="expression" message="You will need to reselect the file you specified (%s)." substitute_value_in_message="True">not ( ( isinstance( value, unicode ) or isinstance( value, str ) ) and value != "" )</validator><!-- use validator to post message to user about needing to reselect the file, since most browsers won't accept the value attribute for file inputs --></param><param name="url_paste" type="text" area="true" size="5x35" label="URL/Text" help="Here you may specify a list of URLs (one per line) or paste the contents of a file."/> 
+      <param name="ftp_files" type="ftpfile" label="Files uploaded via FTP"/><param name="space_to_tab" type="select" display="checkboxes" multiple="True" label="Convert spaces to tabs" help="Use this option if you are entering intervals by hand."><option value="Yes">Yes</option></param>

--- a/lib/galaxy/web/form_builder.py
+++ b/lib/galaxy/web/form_builder.py
@@ -2,9 +2,10 @@
 Classes for generating HTML forms
 """
 
-import logging,sys
+import logging, sys, os, time
 from cgi import escape
-from galaxy.util import restore_text
+from galaxy.util import restore_text, relpath, nice_size
+from galaxy.web import url_for
 
 log = logging.getLogger(__name__)
 
@@ -145,6 +146,68 @@ class FileField(BaseField):
             ajax_text = ' galaxy-ajax-upload="true"'
         return '<input type="file" name="%s%s"%s%s>' % ( prefix, self.name, ajax_text, value_text )
 
+class FTPFileField(BaseField):
+    """
+    An FTP file upload input.
+    """
+    thead = '''
+        <table id="grid-table" class="grid"> 
+            <thead id="grid-table-header">
+                <tr>
+                    <th id="select-header"></th>
+                    <th id="name-header">
+                        File
+                    </th>
+                    <th id="size-header">
+                        Size
+                    </th>
+                    <th id="date-header">
+                        Date
+                    </th>
+                </tr>
+            </thead>
+            <tbody id="grid-table-body">
+    '''
+    trow = '''
+                <tr>
+                    <td><input type="checkbox" name="%s%s" value="%s"/></td>
+                    <td>%s</td>
+                    <td>%s</td>
+                    <td>%s</td>
+                </tr>
+    '''
+    tfoot = '''
+            </tbody>
+        </table>
+    '''
+    def __init__( self, name, dir, ftp_site, value = None ):
+        self.name = name
+        self.dir = dir
+        self.ftp_site = ftp_site
+        self.value = value
+    def get_html( self, prefix="" ):
+        rval = FTPFileField.thead
+        if self.dir is None:
+            rval += '<tr><td colspan="3"><em>Please <a href="%s">create</a> or <a href="%s">log in to</a> a Galaxy account to view files uploaded via FTP.</em></td></tr>' % ( url_for( controller='user', action='create', referer=url_for( controller='root' ) ), url_for( controller='user', action='login', referer=url_for( controller='root' ) ) )
+        elif not os.path.exists( self.dir ):
+            rval += '<tr><td colspan="3"><em>Your FTP upload directory contains no files.</em></td></tr>'
+        else:
+            uploads = []
+            for ( dirpath, dirnames, filenames ) in os.walk( self.dir ):
+                for filename in filenames:
+                    path = relpath( os.path.join( dirpath, filename ), self.dir )
+                    statinfo = os.lstat( os.path.join( dirpath, filename ) )
+                    uploads.append( dict( path=path,
+                                          size=nice_size( statinfo.st_size ),
+                                          ctime=time.strftime( "%m/%d/%Y %I:%M:%S %p", time.localtime( statinfo.st_ctime ) ) ) )
+            if not uploads:
+                rval += '<tr><td colspan="3"><em>Your FTP upload directory contains no files.</em></td></tr>'
+            for upload in uploads:
+                rval += FTPFileField.trow % ( prefix, self.name, upload['path'], upload['path'], upload['size'], upload['ctime'] )
+        rval += FTPFileField.tfoot
+        rval += '<div class="toolParamHelp">This Galaxy server allows you to upload files via FTP.  To upload some files, log in to the FTP server at <strong>%s</strong> using your Galaxy credentials (email address and password).</div>' % self.ftp_site
+        return rval
+
 class HiddenField(BaseField):
     """
     A hidden field.

--- a/lib/galaxy/tools/parameters/basic.py
+++ b/lib/galaxy/tools/parameters/basic.py
@@ -40,6 +40,11 @@ class ToolParameter( object ):
         for elem in param.findall("validator"):
             self.validators.append( validation.Validator.from_element( self, elem ) )
 
+    @property
+    def visible( self ):
+        """Return true if the parameter should be rendered on the form"""
+        return True
+
     def get_label( self ):
         """Return user friendly name for the parameter"""
         if self.label: return self.label
@@ -362,6 +367,41 @@ class FileToolParameter( ToolParameter )
     def get_initial_value( self, trans, context ):
         return None
         
+class FTPFileToolParameter( ToolParameter ):
+    """
+    Parameter that takes a file uploaded via FTP as a value.
+    """
+    def __init__( self, tool, elem ):
+        """
+        Example: C{<param name="bins" type="file" />}
+        """
+        ToolParameter.__init__( self, tool, elem )
+    @property
+    def visible( self ):
+        if self.tool.app.config.ftp_upload_dir is None or self.tool.app.config.ftp_upload_site is None:
+            return False
+        return True
+    def get_html_field( self, trans=None, value=None, other_values={}  ):
+        if trans is None or trans.user is None:
+            user_ftp_dir = None
+        else:
+            user_ftp_dir = os.path.join( trans.app.config.ftp_upload_dir, trans.user.email )
+        return form_builder.FTPFileField( self.name, user_ftp_dir, trans.app.config.ftp_upload_site, value = value )
+    def from_html( self, value, trans=None, other_values={} ):
+        return util.listify( value )
+    def to_string( self, value, app ):
+        if value in [ None, '' ]:
+            return None
+        elif isinstance( value, unicode ) or isinstance( value, str ) or isinstance( value, list ):
+            return value
+    def to_python( self, value, app ):
+        if value is None:
+            return None
+        elif isinstance( value, unicode ) or isinstance( value, str ) or isinstance( value, list ):
+            return value
+    def get_initial_value( self, trans, context ):
+        return None
+        
 class HiddenToolParameter( ToolParameter ):
     """
     Parameter that takes one of two values. 
@@ -1427,6 +1467,7 @@ parameter_types = dict( text        = Te
                         hidden      = HiddenToolParameter,
                         baseurl     = BaseURLToolParameter,
                         file        = FileToolParameter,
+                        ftpfile     = FTPFileToolParameter,
                         data        = DataToolParameter,
                         drill_down = DrillDownSelectToolParameter )
 

--- a/lib/galaxy/util/__init__.py
+++ b/lib/galaxy/util/__init__.py
@@ -480,6 +480,32 @@ def umask_fix_perms( path, umask, unmask
                                                                                                           current_group,
                                                                                                           e ) )
 
+def nice_size(size):
+    """
+    Returns a readably formatted string with the size
+
+    >>> nice_size(100)
+    '100.0 bytes'
+    >>> nice_size(10000)
+    '9.8 Kb'
+    >>> nice_size(1000000)
+    '976.6 Kb'
+    >>> nice_size(100000000)
+    '95.4 Mb'
+    """
+    words = [ 'bytes', 'Kb', 'Mb', 'Gb' ]
+    try:
+        size = float( size )
+    except:
+        return '??? bytes'
+    for ind, word in enumerate(words):
+        step  = 1024 ** (ind + 1)
+        if step > size:
+            size = size / float(1024 ** ind)
+            out  = "%.1f %s" % (size, word)
+            return out
+    return '??? bytes'
+
 galaxy_root_path = os.path.join(__path__[0], "..","..","..")
 # The dbnames list is used in edit attributes and the upload tool
 dbnames = read_dbnames( os.path.join( galaxy_root_path, "tool-data", "shared", "ucsc", "builds.txt" ) )

--- a/universe_wsgi.ini.sample
+++ b/universe_wsgi.ini.sample
@@ -324,6 +324,18 @@ use_interactive = True
 # Enable the (experimental! beta!) Web API.  Documentation forthcoming.
 #enable_api = False
 
+# Enable Galaxy's "Upload via FTP" interface.  You'll need to install and
+# configure an FTP server (we've used ProFTPd since it can use Galaxy's
+# database for authentication) and set the following two options.
+
+# This should point to a directory containing subdirectories matching users'
+# email addresses, where Galaxy will look for files.
+#ftp_upload_dir = None
+
+# This should be the hostname of your FTP server, which will be provided to
+# users in the help text.
+#ftp_upload_site = None
+
 # -- Job Execution
 
 # If running multiple Galaxy processes, one can be designated as the job

--- a/templates/tool_form.mako
+++ b/templates/tool_form.mako
@@ -51,7 +51,9 @@ function checkUncheckAll( name, check ) 
     <%def name="do_inputs( inputs, tool_state, errors, prefix, other_values=None )"><% other_values = ExpressionContext( tool_state, other_values ) %>
         %for input_index, input in enumerate( inputs.itervalues() ):
-            %if input.type == "repeat":
+            %if not input.visible:
+                <% pass %>
+            %elif input.type == "repeat":
               <div class="repeat-group"><div class="form-title-row"><b>${input.title_plural}</b></div><% repeat_state = tool_state[input.name] %>

--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -929,7 +929,11 @@ class Tool:
                     assert isinstance( out_data, odict )
                     return 'tool_executed.mako', dict( out_data=out_data )
                 except:
-                    return 'message.mako', dict( status='error', message='odict not returned from tool execution', refresh_frames=[] )
+                    if isinstance( out_data, str ):
+                        message = out_data
+                    else:
+                        message = 'Failure executing tool (odict not returned from tool execution)'
+                    return 'message.mako', dict( status='error', message=message, refresh_frames=[] )
             # Otherwise move on to the next page
             else:
                 state.page += 1

--- a/lib/galaxy/tools/parameters/grouping.py
+++ b/lib/galaxy/tools/parameters/grouping.py
@@ -12,11 +12,14 @@ import StringIO, os, urllib
 from galaxy.datatypes import sniff
 from galaxy.util.bunch import Bunch
 from galaxy.util.odict import odict
-from galaxy.util import json
+from galaxy.util import json, relpath
 
 class Group( object ):
     def __init__( self ):
         self.name = None
+    @property
+    def visible( self ):
+        return True
     def value_to_basic( self, value, app ):
         """
         Convert value to a (possibly nested) representation using only basic
@@ -267,6 +270,7 @@ class UploadDataset( Group ):
             rval = []
             data_file = context['file_data']
             url_paste = context['url_paste']
+            ftp_files = context['ftp_files']
             name = context.get( 'NAME', None )
             info = context.get( 'INFO', None )
             space_to_tab = False
@@ -281,6 +285,31 @@ class UploadDataset( Group ):
                 if file_bunch.path:
                     file_bunch.space_to_tab = space_to_tab
                     rval.append( file_bunch )
+            # look for files uploaded via FTP
+            valid_files = []
+            if ftp_files:
+                if trans.user is None:
+                    log.warning( 'Anonymous user passed values in ftp_files: %s' % ftp_files )
+                    ftp_files = []
+                    # TODO: warning to the user (could happen if session has become invalid)
+                else:
+                    user_ftp_dir = os.path.join( trans.app.config.ftp_upload_dir, trans.user.email )
+                    for ( dirpath, dirnames, filenames ) in os.walk( user_ftp_dir ):
+                        for filename in filenames:
+                            path = relpath( os.path.join( dirpath, filename ), user_ftp_dir )
+                            if not os.path.islink( os.path.join( dirpath, filename ) ):
+                                valid_files.append( path )
+            for ftp_file in ftp_files:
+                if ftp_file not in valid_files:
+                    log.warning( 'User passed an invalid file path in ftp_files: %s' % ftp_file )
+                    continue
+                    # TODO: warning to the user (could happen if file is already imported)
+                ftp_data_file = { 'local_filename' : os.path.abspath( os.path.join( user_ftp_dir, ftp_file ) ),
+                                  'filename' : os.path.basename( ftp_file ) }
+                file_bunch = get_data_file_filename( ftp_data_file, override_name = name, override_info = info )
+                if file_bunch.path:
+                    file_bunch.space_to_tab = space_to_tab
+                    rval.append( file_bunch )
             return rval
         file_type = self.get_file_type( context )
         d_type = self.get_datatype( trans, context )

--- a/static/june_2007_style/blue/base.css
+++ b/static/june_2007_style/blue/base.css
@@ -39,8 +39,7 @@ div.form-title-row{padding:5px 10px;}
 div.repeat-group-item{border-left:solid #d8b365 5px;margin-left:10px;margin-bottom:10px;}
 div.form-row-error{background:#FFCCCC;}
 div.form-row label{font-weight:bold;display:block;margin-bottom:.2em;}
-div.form-row-input{float:left;width:300px;}
-div.form-row-input > input{max-width:300px;}
+div.form-row-input{float:left;}
 div.form-row-input label{font-weight:normal;display:inline;}
 div.form-row-error-message{width:300px;float:left;color:red;font-weight:bold;padding:3px 0 0 1em;}
 select,input,textarea{font:inherit;font-size:115%;}

--- a/lib/galaxy/datatypes/data.py
+++ b/lib/galaxy/datatypes/data.py
@@ -419,36 +419,16 @@ class Newick( Text ):
 
 # ------------- Utility methods --------------
 
+# nice_size used to be here, but to resolve cyclical dependencies it's been
+# moved to galaxy.util.  It belongs there anyway since it's used outside
+# datatypes.
+nice_size = util.nice_size
+
 def get_test_fname( fname ):
     """Returns test data filename"""
     path, name = os.path.split(__file__)
     full_path = os.path.join( path, 'test', fname )
     return full_path
-def nice_size(size):
-    """
-    Returns a readably formatted string with the size
-
-    >>> nice_size(100)
-    '100.0 bytes'
-    >>> nice_size(10000)
-    '9.8 Kb'
-    >>> nice_size(1000000)
-    '976.6 Kb'
-    >>> nice_size(100000000)
-    '95.4 Mb'
-    """
-    words = [ 'bytes', 'Kb', 'Mb', 'Gb' ]
-    try:
-        size = float( size )
-    except:
-        return '??? bytes'
-    for ind, word in enumerate(words):
-        step  = 1024 ** (ind + 1)
-        if step > size:
-            size = size / float(1024 ** ind)
-            out  = "%.1f %s" % (size, word)
-            return out
-    return '??? bytes'
 def get_file_peek( file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5 ):
     """
     Returns the first LINE_COUNT lines wrapped to WIDTH

--- a/lib/galaxy/tools/actions/upload.py
+++ b/lib/galaxy/tools/actions/upload.py
@@ -21,7 +21,7 @@ class UploadToolAction( ToolAction ):
         upload_common.cleanup_unused_precreated_datasets( precreated_datasets )
         
         if not uploaded_datasets:
-            return 'No data was entered in the upload form, please go back and choose data to upload.'
+            return None, 'No data was entered in the upload form, please go back and choose data to upload.'
         
         json_file_path = upload_common.create_paramfile( trans, uploaded_datasets )
         data_list = [ ud.data for ud in uploaded_datasets ]

--- a/lib/galaxy/config.py
+++ b/lib/galaxy/config.py
@@ -99,6 +99,10 @@ class Configuration( object ):
         self.user_library_import_dir = kwargs.get( 'user_library_import_dir', None )
         if self.user_library_import_dir is not None and not os.path.exists( self.user_library_import_dir ):
             raise ConfigurationError( "user_library_import_dir specified in config (%s) does not exist" % self.user_library_import_dir )
+        self.ftp_upload_dir = kwargs.get( 'ftp_upload_dir', None )
+        if self.ftp_upload_dir is not None and not os.path.exists( self.ftp_upload_dir ):
+            os.makedirs( self.ftp_upload_dir )
+        self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None )
         self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False )
         self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' )
         # Location for dependencies

--- a/static/june_2007_style/base.css.tmpl
+++ b/static/june_2007_style/base.css.tmpl
@@ -212,11 +212,6 @@ div.form-row label {
 
 div.form-row-input {
     float: left;
-    width: 300px;
-}
-
-div.form-row-input > input {
-    max-width: 300px;
 }
 
 div.form-row-input label {

    

[galaxy-commits] galaxy-dist commit 7e9ae4605105: Add a new "Files uploaded via FTP" grid to the upload form and related parameter types, form fields, etc.

commits-noreply＠bitbucket.org