[hg] galaxy 2519: Enhance data_source.py to take advantage of th...

4 Aug 2009

details:   http://www.bx.psu.edu/hg/galaxy/rev/a7ba71cade35
changeset: 2519:a7ba71cade35
user:      Dan Blankenberg <dan@bx.psu.edu>
date:      Mon Aug 03 12:02:54 2009 -0400
description:
Enhance data_source.py to take advantage of the content length when known. If content length provided by external application is
greater than config.output_size_limit, no data will be retrieved and an error message will be provided to the user.

Tools using this script have been updated to provide the max file size on the command line.

Resolves ticket #93.

15 file(s) affected in this change:

lib/galaxy/jobs/__init__.py
tools/data_source/biomart.xml
tools/data_source/biomart_test.xml
tools/data_source/data_source.py
tools/data_source/epigraph_import.xml
tools/data_source/epigraph_import_test.xml
tools/data_source/eupathdb.xml
tools/data_source/flymine.xml
tools/data_source/flymine_test.xml
tools/data_source/gramene_mart.xml
tools/data_source/ucsc_tablebrowser.xml
tools/data_source/ucsc_tablebrowser_archaea.xml
tools/data_source/ucsc_tablebrowser_test.xml
tools/data_source/wormbase.xml
tools/data_source/wormbase_test.xml

diffs (193 lines):

diff -r 07dffb2735dd -r a7ba71cade35 lib/galaxy/jobs/__init__.py

--- a/lib/galaxy/jobs/__init__.py	Fri Jul 31 12:01:02 2009 -0400
+++ b/lib/galaxy/jobs/__init__.py	Mon Aug 03 12:02:54 2009 -0400
@@ -595,7 +595,7 @@
                 self.real_path = real_path
                 self.false_path = false_path
             def __str__( self ):
-                if false_path is None:
+                if self.false_path is None:
                     return self.real_path
                 else:
                     return self.false_path
diff -r 07dffb2735dd -r a7ba71cade35 tools/data_source/biomart.xml
--- a/tools/data_source/biomart.xml	Fri Jul 31 12:01:02 2009 -0400
+++ b/tools/data_source/biomart.xml	Mon Aug 03 12:02:54 2009 -0400
@@ -9,7 +9,7 @@
 -->
 <tool name="BioMart" id="biomart" tool_type="data_source" URL_method="get" version="1.0.1">
 	<description>Central server</description>
-	<command interpreter="python">data_source.py $output</command>
+	<command interpreter="python">data_source.py $output $__app__.config.output_size_limit</command>
 	<inputs action="http://www.biomart.org/biomart/martview" check_values="false" method="get" target="_top">
 		<display>go to BioMart Central $GALAXY_URL</display>
 		<param name="GALAXY_URL" type="baseurl" value="/tool_runner/biomart" />
diff -r 07dffb2735dd -r a7ba71cade35 tools/data_source/biomart_test.xml
--- a/tools/data_source/biomart_test.xml	Fri Jul 31 12:01:02 2009 -0400
+++ b/tools/data_source/biomart_test.xml	Mon Aug 03 12:02:54 2009 -0400
@@ -9,7 +9,7 @@
 -->
 <tool name="BioMart" id="biomart_test" tool_type="data_source" URL_method="get" version="1.0.1">
 	<description>Test server</description>
-	<command interpreter="python">data_source.py $output</command>
+	<command interpreter="python">data_source.py $output $__app__.config.output_size_limit</command>
 	<inputs action="http://test.biomart.org/biomart/martview" check_values="false" method="get" target="_top">
 		<display>go to BioMart Central $GALAXY_URL</display>
 		<param name="GALAXY_URL" type="baseurl" value="/tool_runner/biomart" />
diff -r 07dffb2735dd -r a7ba71cade35 tools/data_source/data_source.py
--- a/tools/data_source/data_source.py	Fri Jul 31 12:01:02 2009 -0400
+++ b/tools/data_source/data_source.py	Mon Aug 03 12:02:54 2009 -0400
@@ -21,6 +21,10 @@
 
 def __main__():
     filename = sys.argv[1]
+    try:
+        max_file_size = int( sys.argv[2] )
+    except:
+        max_file_size = 0
     params = {}
     for line in open( filename, 'r' ):
         try:
@@ -50,6 +54,10 @@
             page = urllib.urlopen( URL, urllib.urlencode( params ) )
     except Exception, e:
         stop_err( 'The remote data source application may be off line, please try again later. Error: %s' % str( e ) )
+    if max_file_size:
+        file_size = int( page.info().get( 'Content-Length', 0 ) )
+        if file_size > max_file_size:
+            stop_err( 'The size of the data (%d bytes) you have requested exceeds the maximum allowed (%d bytes) on this server.' % ( file_size, max_file_size ) )
     out = open( filename, 'w' )
     while 1:
         chunk = page.read( CHUNK_SIZE )
diff -r 07dffb2735dd -r a7ba71cade35 tools/data_source/epigraph_import.xml
--- a/tools/data_source/epigraph_import.xml	Fri Jul 31 12:01:02 2009 -0400
+++ b/tools/data_source/epigraph_import.xml	Mon Aug 03 12:02:54 2009 -0400
@@ -6,7 +6,7 @@
 -->
 <tool name="EpiGRAPH" id="epigraph_import" tool_type="data_source" URL_method="get">
     <description> server</description>
-    <command interpreter="python">data_source.py $output</command>
+    <command interpreter="python">data_source.py $output $__app__.config.output_size_limit</command>
     <inputs action="http://epigraph.mpi-inf.mpg.de/WebGRAPH/faces/Login.jsp" check_values="false" method="get">
         <display>go to EpiGRAPH server $GALAXY_URL</display>
         <param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=epigraph_import" />
diff -r 07dffb2735dd -r a7ba71cade35 tools/data_source/epigraph_import_test.xml
--- a/tools/data_source/epigraph_import_test.xml	Fri Jul 31 12:01:02 2009 -0400
+++ b/tools/data_source/epigraph_import_test.xml	Mon Aug 03 12:02:54 2009 -0400
@@ -6,7 +6,7 @@
 -->
 <tool name="EpiGRAPH" id="epigraph_import_test" tool_type="data_source" URL_method="get">
     <description> test server</description>
-    <command interpreter="python">data_source.py $output</command>
+    <command interpreter="python">data_source.py $output $__app__.config.output_size_limit</command>
     <inputs action="http://epigraph.mpi-inf.mpg.de/WebGRAPH_Public_Test/faces/Login.jsp" check_values="false" method="get">
         <display>go to EpiGRAPH server $GALAXY_URL</display>
         <param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=epigraph_import_test" />
diff -r 07dffb2735dd -r a7ba71cade35 tools/data_source/eupathdb.xml
--- a/tools/data_source/eupathdb.xml	Fri Jul 31 12:01:02 2009 -0400
+++ b/tools/data_source/eupathdb.xml	Mon Aug 03 12:02:54 2009 -0400
@@ -5,7 +5,7 @@
 -->
 <tool name="EuPathDB" id="eupathdb" tool_type="data_source" url_method="post">
     <description>server</description>
-    <command interpreter="python">data_source.py $output</command>
+    <command interpreter="python">data_source.py $output $__app__.config.output_size_limit</command>
     <inputs action="http://galaxy.eupathdb.org/eupathdb.galaxy/queries_tools.jsp" check_values="false" method="post"> 
         <display>go to EuPathDB server $GALAXY_URL</display>
         <param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=eupathdb" />
diff -r 07dffb2735dd -r a7ba71cade35 tools/data_source/flymine.xml
--- a/tools/data_source/flymine.xml	Fri Jul 31 12:01:02 2009 -0400
+++ b/tools/data_source/flymine.xml	Mon Aug 03 12:02:54 2009 -0400
@@ -6,7 +6,7 @@
 -->
 <tool name="Flymine" id="flymine" tool_type="data_source" URL_method="post">
     <description>server</description>
-    <command interpreter="python">data_source.py $output</command>
+    <command interpreter="python">data_source.py $output $__app__.config.output_size_limit</command>
     <inputs action="http://www.flymine.org" check_values="false" method="get" target="_top"> 
         <display>go to Flymine server $GALAXY_URL</display>
         <param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=flymine" />
diff -r 07dffb2735dd -r a7ba71cade35 tools/data_source/flymine_test.xml
--- a/tools/data_source/flymine_test.xml	Fri Jul 31 12:01:02 2009 -0400
+++ b/tools/data_source/flymine_test.xml	Mon Aug 03 12:02:54 2009 -0400
@@ -6,7 +6,7 @@
 -->
 <tool name="Flymine test" id="flymine_test" tool_type="data_source" URL_method="post">
     <description>server</description>
-    <command interpreter="python">data_source.py $output</command>
+    <command interpreter="python">data_source.py $output $__app__.config.output_size_limit</command>
     <inputs action="http://preview.flymine.org/preview/begin.do" check_values="false" method="get" target="_top"> 
         <display>go to Flymine server $GALAXY_URL</display>
         <param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=flymine" />
diff -r 07dffb2735dd -r a7ba71cade35 tools/data_source/gramene_mart.xml
--- a/tools/data_source/gramene_mart.xml	Fri Jul 31 12:01:02 2009 -0400
+++ b/tools/data_source/gramene_mart.xml	Mon Aug 03 12:02:54 2009 -0400
@@ -9,7 +9,7 @@
 -->
 <tool name="GrameneMart" id="gramenemart" tool_type="data_source" URL_method="get" version="1.0.1">
     <description> Central server</description>
-    <command interpreter="python">data_source.py $output</command>
+    <command interpreter="python">data_source.py $output $__app__.config.output_size_limit</command>
     <inputs action="http://www.gramene.org/biomart/martview" check_values="false" method="get" target="_top">
         <display>go to GrameneMart Central $GALAXY_URL</display>
         <param name="GALAXY_URL" type="baseurl" value="/tool_runner/biomart" />
diff -r 07dffb2735dd -r a7ba71cade35 tools/data_source/ucsc_tablebrowser.xml
--- a/tools/data_source/ucsc_tablebrowser.xml	Fri Jul 31 12:01:02 2009 -0400
+++ b/tools/data_source/ucsc_tablebrowser.xml	Mon Aug 03 12:02:54 2009 -0400
@@ -6,7 +6,7 @@
 -->
 <tool name="UCSC Main" id="ucsc_table_direct1" tool_type="data_source" URL_method="post">
 	<description>table browser</description>
-	<command interpreter="python">data_source.py $output</command>
+	<command interpreter="python">data_source.py $output $__app__.config.output_size_limit</command>
 	<inputs action="http://genome.ucsc.edu/cgi-bin/hgTables" check_values="false" method="get">
 		<display>go to UCSC Table Browser $GALAXY_URL</display>
 		<param name="GALAXY_URL" type="baseurl" value="/tool_runner" />
diff -r 07dffb2735dd -r a7ba71cade35 tools/data_source/ucsc_tablebrowser_archaea.xml
--- a/tools/data_source/ucsc_tablebrowser_archaea.xml	Fri Jul 31 12:01:02 2009 -0400
+++ b/tools/data_source/ucsc_tablebrowser_archaea.xml	Mon Aug 03 12:02:54 2009 -0400
@@ -6,7 +6,7 @@
 -->
 <tool name="UCSC Archaea" id="ucsc_table_direct_archaea1" tool_type="data_source" URL_method="post">
 	<description>table browser</description>
-	<command interpreter="python">data_source.py $output</command>
+	<command interpreter="python">data_source.py $output $__app__.config.output_size_limit</command>
 	<inputs action="http://archaea.ucsc.edu/cgi-bin/hgTables" check_values="false" method="get">
 		<display>go to UCSC Table Browser $GALAXY_URL</display>
 		<param name="GALAXY_URL" type="baseurl" value="/tool_runner" />
diff -r 07dffb2735dd -r a7ba71cade35 tools/data_source/ucsc_tablebrowser_test.xml
--- a/tools/data_source/ucsc_tablebrowser_test.xml	Fri Jul 31 12:01:02 2009 -0400
+++ b/tools/data_source/ucsc_tablebrowser_test.xml	Mon Aug 03 12:02:54 2009 -0400
@@ -6,7 +6,7 @@
 -->
 <tool name="UCSC Test" id="ucsc_table_direct_test1" tool_type="data_source" URL_method="post">
 	<description>table browser</description>
-	<command interpreter="python">data_source.py $output</command>
+	<command interpreter="python">data_source.py $output $__app__.config.output_size_limit</command>
 	<inputs action="http://genome-test.cse.ucsc.edu/cgi-bin/hgTables" check_values="false" method="get">
 		<display>go to UCSC Table Browser $GALAXY_URL</display>
 		<param name="GALAXY_URL" type="baseurl" value="/tool_runner" />
diff -r 07dffb2735dd -r a7ba71cade35 tools/data_source/wormbase.xml
--- a/tools/data_source/wormbase.xml	Fri Jul 31 12:01:02 2009 -0400
+++ b/tools/data_source/wormbase.xml	Mon Aug 03 12:02:54 2009 -0400
@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 <tool name="Wormbase" id="wormbase" tool_type="data_source" URL_method="post">
     <description>server</description>
-    <command interpreter="python">data_source.py $output</command>
+    <command interpreter="python">data_source.py $output $__app__.config.output_size_limit</command>
     <inputs action="http://www.wormbase.org/db/seq/gbgff/c_elegans/" check_values="false" target="_top"> 
         <display>go to Wormbase server $GALAXY_URL</display>
         <param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=wormbase" />
diff -r 07dffb2735dd -r a7ba71cade35 tools/data_source/wormbase_test.xml
--- a/tools/data_source/wormbase_test.xml	Fri Jul 31 12:01:02 2009 -0400
+++ b/tools/data_source/wormbase_test.xml	Mon Aug 03 12:02:54 2009 -0400
@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 <tool name="Wormbase" id="wormbase_test" tool_type="data_source" URL_method="post">
 	<description>test server</description>
-	<command interpreter="python">data_source.py $output</command>
+	<command interpreter="python">data_source.py $output $__app__.config.output_size_limit</command>
 	<inputs action="http://dev.wormbase.org/db/seq/gbrowse/c_elegans/" check_values="false" target="_top"> 
 		<display>go to Wormbase test server $GALAXY_URL</display>
 		<param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=wormbase_test" />

    

Greg Von Kuster

tags

participants (1)