details: http://www.bx.psu.edu/hg/galaxy/rev/fd5943ff7b40 changeset: 2903:fd5943ff7b40 user: James Taylor <james@jamestaylor.org> date: Tue Oct 20 13:08:29 2009 -0400 description: New functional test harness. Update to recent nose. No longer rely on nose to manage the embedded web server (this is not done in functional_tests.py). Log page contents to a temporary file on check_page_for_string error. [mq]: functional-test-harness 7 file(s) affected in this change: eggs.ini lib/galaxy/eggs/__init__.py lib/galaxy/model/migrate/check.py run_functional_tests.sh scripts/functional_tests.py test/base/twilltestcase.py test/functional/__init__.py diffs (585 lines): diff -r 2f3c4dd6ff1a -r fd5943ff7b40 eggs.ini --- a/eggs.ini Wed Oct 21 14:19:45 2009 -0400 +++ b/eggs.ini Tue Oct 20 13:08:29 2009 -0400 @@ -34,8 +34,8 @@ ;lsprof - james Mako = 0.2.5 MyghtyUtils = 0.52 -nose = 0.9.1 -NoseHTML = 0.2 +nose = 0.11.1 +NoseHTML = 0.3 Paste = 1.5.1 PasteDeploy = 1.3.1 PasteScript = 1.3.6 @@ -60,7 +60,7 @@ python_lzo = _static flup = .dev_r2311 bx_python = _dev_r4bf1f32e6b76 -nose = .dev_r101 +; nose = .dev_r7156749efc58 ; source location, necessary for scrambling [source] @@ -82,8 +82,8 @@ lrucache = http://evan.prodromou.name/lrucache/lrucache-0.2.tar.gz Mako = http://www.makotemplates.org/downloads/Mako-0.2.5.tar.gz MyghtyUtils = http://cheeseshop.python.org/packages/source/M/MyghtyUtils/MyghtyUtils-0.52.... -nose = http://www.somethingaboutorange.com/mrl/projects/nose/nose-0.9.1.tar.gz -NoseHTML = http://dist.g2.bx.psu.edu/nosehtml-0.2.tar.bz2 +nose = http://pypi.python.org/packages/source/n/nose/nose-0.11.1.tar.gz +NoseHTML = http://bitbucket.org/james_taylor/nosehtml/get/c46a54d569ae.bz2 Paste = http://cheeseshop.python.org/packages/source/P/Paste/Paste-1.5.1.tar.gz PasteDeploy = http://cheeseshop.python.org/packages/source/P/PasteDeploy/PasteDeploy-1.3.1... PasteScript = http://cheeseshop.python.org/packages/source/P/PasteScript/PasteScript-1.3.6... diff -r 2f3c4dd6ff1a -r fd5943ff7b40 lib/galaxy/eggs/__init__.py --- a/lib/galaxy/eggs/__init__.py Wed Oct 21 14:19:45 2009 -0400 +++ b/lib/galaxy/eggs/__init__.py Tue Oct 20 13:08:29 2009 -0400 @@ -2,7 +2,7 @@ Manage Galaxy eggs """ -import os, sys, shutil, tarfile, zipfile, subprocess, ConfigParser, glob, urllib2 +import os, sys, shutil, tarfile, zipfile, zipimport, subprocess, ConfigParser, glob, urllib2, shutil from types import ModuleType import logging @@ -101,10 +101,17 @@ raise EggNotFetchable( self.name ) #if e.code == 404: # return False + self.unpack_if_needed() for doppelganger in self.doppelgangers: - os.unlink( doppelganger ) + remove_file_or_path( doppelganger ) log.debug( "Removed conflicting egg: %s" % doppelganger ) return True + def unpack_if_needed( self ): + meta = pkg_resources.EggMetadata( zipimport.zipimporter( self.path ) ) + if meta.has_metadata( 'not-zip-safe' ): + unpack_zipfile( self.path, self.path + "-tmp" ) + os.remove( self.path ) + os.rename( self.path + "-tmp", self.path ) def scramble( self ): if self.path is None: self.find() @@ -141,8 +148,9 @@ shutil.copyfile( new_egg, self.path ) log.warning( "scramble(): Copied egg to:" ) log.warning( " %s" % self.path ) + self.unpack_if_needed() for doppelganger in self.doppelgangers: - os.unlink( doppelganger ) + remove_file_or_path( doppelganger ) log.warning( "Removed conflicting egg: %s" % doppelganger ) return True # scramble helper methods @@ -584,4 +592,37 @@ else: return "%s-noplatform" % get_py() +def unpack_zipfile( filename, extract_dir): + z = zipfile.ZipFile(filename) + try: + for info in z.infolist(): + name = info.filename + # don't extract absolute paths or ones with .. in them + if name.startswith('/') or '..' in name: + continue + target = os.path.join(extract_dir, *name.split('/')) + if not target: + continue + if name.endswith('/'): + # directory + pkg_resources.ensure_directory(target) + else: + # file + pkg_resources.ensure_directory(target) + data = z.read(info.filename) + f = open(target,'wb') + try: + f.write(data) + finally: + f.close() + del data + finally: + z.close() + +def remove_file_or_path( f ): + if os.path.isdir( f ): + shutil.rmtree( f ) + else: + os.remove( f ) + pkg_resources.require = require diff -r 2f3c4dd6ff1a -r fd5943ff7b40 lib/galaxy/model/migrate/check.py --- a/lib/galaxy/model/migrate/check.py Wed Oct 21 14:19:45 2009 -0400 +++ b/lib/galaxy/model/migrate/check.py Tue Oct 20 13:08:29 2009 -0400 @@ -1,4 +1,4 @@ -import os.path, logging +import sys, os.path, logging from galaxy import eggs @@ -109,4 +109,18 @@ for ver, change in changeset: nextver = ver + changeset.step log.info( 'Migrating %s -> %s... ' % ( ver, nextver ) ) - schema.runchange( ver, change, changeset.step ) + old_stdout = sys.stdout + class FakeStdout( object ): + def __init__( self ): + self.buffer = [] + def write( self, s ): + self.buffer.append( s ) + def flush( self ): + pass + sys.stdout = FakeStdout() + try: + schema.runchange( ver, change, changeset.step ) + finally: + for message in "".join( sys.stdout.buffer ).split( "\n" ): + log.info( message ) + sys.stdout = old_stdout \ No newline at end of file diff -r 2f3c4dd6ff1a -r fd5943ff7b40 run_functional_tests.sh --- a/run_functional_tests.sh Wed Oct 21 14:19:45 2009 -0400 +++ b/run_functional_tests.sh Tue Oct 20 13:08:29 2009 -0400 @@ -5,7 +5,7 @@ rm -f run_functional_tests.log if [ ! $1 ]; then - python -ES ./scripts/nosetests.py -v -w test --with-nosehtml --html-report-file run_functional_tests.html --exclude="^get" functional + python -ES ./scripts/functional_tests.py -v --with-nosehtml --html-report-file run_functional_tests.html --exclude="^get" functional elif [ $1 = 'help' ]; then echo "'run_functional_tests.sh' for testing all the tools in functional directory" echo "'run_functional_tests.sh aaa' for testing one test case of 'aaa' ('aaa' is the file name with path)" @@ -13,16 +13,16 @@ echo "'run_functional_tests.sh -sid ccc' for testing one section with sid 'ccc' ('ccc' is the string after 'section::')" echo "'run_functional_tests.sh -list' for listing all the tool ids" elif [ $1 = '-id' ]; then - python -ES ./scripts/nosetests.py -v -w test functional.test_toolbox:TestForTool_$2 --with-nosehtml --html-report-file run_functional_tests.html + python -ES ./scripts/functional_tests.py -v functional.test_toolbox:TestForTool_$2 --with-nosehtml --html-report-file run_functional_tests.html elif [ $1 = '-sid' ]; then - python -ES ./scripts/nosetests.py --with-nosehtml --html-report-file run_functional_tests.html -v -w test `python tool_list.py $2` + python -ES ./scripts/functional_tests.py --with-nosehtml --html-report-file run_functional_tests.html -v `python tool_list.py $2` elif [ $1 = '-list' ]; then python tool_list.py echo "===========================================================================================================================================" echo "'run_functional_tests.sh -id bbb' for testing one tool with id 'bbb' ('bbb' is the tool id)" echo "'run_functional_tests.sh -sid ccc' for testing one section with sid 'ccc' ('ccc' is the string after 'section::')" else - python -ES ./scripts/nosetests.py -v -w test --with-nosehtml --html-report-file run_functional_tests.html $1 + python -ES ./scripts/functional_tests.py -v --with-nosehtml --html-report-file run_functional_tests.html $1 fi echo "'run_functional_tests.sh help' for help" diff -r 2f3c4dd6ff1a -r fd5943ff7b40 scripts/functional_tests.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/scripts/functional_tests.py Tue Oct 20 13:08:29 2009 -0400 @@ -0,0 +1,211 @@ +#!/usr/bin/env python + +import os, sys + +# Assume we are run from the galaxy root directory, add lib to the python path +cwd = os.getcwd() +new_path = [ os.path.join( cwd, "lib" ) ] +new_path.extend( sys.path[1:] ) +sys.path = new_path + +from galaxy import eggs + +eggs.require( "nose" ) +eggs.require( "NoseHTML" ) +eggs.require( "twill==0.9" ) +eggs.require( "Paste" ) +eggs.require( "PasteDeploy" ) +eggs.require( "Cheetah" ) + +import atexit, logging, os, sys, tempfile +import twill, unittest, time +import os, os.path, subprocess, sys, threading +import httplib +from paste import httpserver +import galaxy.app +from galaxy.app import UniverseApplication +from galaxy.web import buildapp +from galaxy import tools +from galaxy.util import bunch + +log = logging.getLogger( "functional_tests.py" ) + +default_galaxy_test_host = "localhost" +default_galaxy_test_port = "8777" +default_galaxy_locales = 'en' +default_galaxy_test_file_dir = "test-data" + +def main(): + + # ---- Configuration ------------------------------------------------------ + + galaxy_test_host = os.environ.get( 'GALAXY_TEST_HOST', default_galaxy_test_host ) + galaxy_test_port = os.environ.get( 'GALAXY_TEST_PORT', default_galaxy_test_port ) + if 'HTTP_ACCEPT_LANGUAGE' not in os.environ: + os.environ['HTTP_ACCEPT_LANGUAGE'] = default_galaxy_locales + galaxy_test_file_dir = os.environ.get( 'GALAXY_TEST_FILE_DIR', default_galaxy_test_file_dir ) + if not os.path.isabs( galaxy_test_file_dir ): + galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) + start_server = 'GALAXY_TEST_EXTERNAL' not in os.environ + if start_server: + if 'GALAXY_TEST_DBPATH' in os.environ: + db_path = os.environ['GALAXY_TEST_DBPATH'] + else: + tempdir = tempfile.mkdtemp() + db_path = os.path.join( tempdir, 'database' ) + file_path = os.path.join( db_path, 'files' ) + try: + os.makedirs( file_path ) + except OSError: + pass + if 'GALAXY_TEST_DBURI' in os.environ: + database_connection = os.environ['GALAXY_TEST_DBURI'] + else: + database_connection = 'sqlite:///' + os.path.join( db_path, 'universe.sqlite' ) + if 'GALAXY_TEST_RUNNERS' in os.environ: + start_job_runners = os.environ['GALAXY_TEST_RUNNERS'] + else: + start_job_runners = None + if 'GALAXY_TEST_DEF_RUNNER' in os.environ: + default_cluster_job_runner = os.environ['GALAXY_TEST_DEF_RUNNER'] + else: + default_cluster_job_runner = 'local:///' + + print "Database connection:", database_connection + + # What requires these? + os.environ['GALAXY_TEST_HOST'] = galaxy_test_host + os.environ['GALAXY_TEST_PORT'] = galaxy_test_port + os.environ['GALAXY_TEST_FILE_DIR'] = galaxy_test_file_dir + + # ---- Build Application -------------------------------------------------- + + app = None + + if start_server: + + # Build the Universe Application + app = UniverseApplication( job_queue_workers = 5, + start_job_runners = start_job_runners, + default_cluster_job_runner = default_cluster_job_runner, + id_secret = 'changethisinproductiontoo', + template_path = "templates", + database_connection = database_connection, + file_path = file_path, + tool_config_file = "tool_conf.xml.sample", + datatype_converters_config_file = "datatype_converters_conf.xml.sample", + tool_path = "tools", + tool_parse_help = False, + test_conf = "test.conf", + log_destination = "stdout", + use_heartbeat = False, + allow_user_creation = True, + allow_user_deletion = True, + admin_users = 'test@bx.psu.edu', + library_import_dir = galaxy_test_file_dir, + user_library_import_dir = os.path.join( galaxy_test_file_dir, 'users' ), + global_conf = { "__file__": "universe_wsgi.ini.sample" } ) + + log.info( "Embedded Universe application started" ); + + # ---- Run webserver ------------------------------------------------------ + + server = None + + if start_server: + + webapp = buildapp.app_factory( dict(), use_translogger = False, app=app ) + server = httpserver.serve( webapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False ) + t = threading.Thread( target=server.serve_forever ) + t.start() + + # Test if the server is up + for i in range( 10 ): + conn = httplib.HTTPConnection( galaxy_test_host, galaxy_test_port ) + conn.request( "GET", "/" ) + if conn.getresponse().status == 200: + break + time.sleep( 0.1 ) + else: + raise Exception( "Test HTTP server did not return '200 OK' after 10 tries" ) + + log.info( "Embedded web server started" ) + + + # ---- Load toolbox for generated tests ----------------------------------- + + # We don't add the tests to the path until everything is up and running + new_path = [ os.path.join( cwd, "test" ) ] + new_path.extend( sys.path[1:] ) + sys.path = new_path + + import functional.test_toolbox + + if app: + # TODO: provisions for loading toolbox from file when using external server + functional.test_toolbox.toolbox = app.toolbox + else: + # FIXME: This doesn't work at all now that toolbox requires an 'app' instance + # (to get at datatypes, might just pass a datatype registry directly) + my_app = bunch.Bunch( datatypes_registry = galaxy.datatypes.registry.Registry() ) + test_toolbox.toolbox = tools.ToolBox( 'tool_conf.xml.test', 'tools', my_app ) + + # ---- Find tests --------------------------------------------------------- + + log.info( "Functional tests will be run against %s:%s" % ( galaxy_test_host, galaxy_test_port ) ) + + rval = False + + try: + + import nose.core + import nose.config + import nose.loader + import nose.plugins.manager + + test_config = nose.config.Config( env = os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) + test_config.configure( sys.argv ) + + loader = nose.loader.TestLoader( config = test_config ) + + plug_loader = test_config.plugins.prepareTestLoader( loader ) + if plug_loader is not None: + loader = plug_loader + + tests = loader.loadTestsFromNames( test_config.testNames ) + + test_runner = nose.core.TextTestRunner( + stream = test_config.stream, + verbosity = test_config.verbosity, + config = test_config) + + plug_runner = test_config.plugins.prepareTestRunner( test_runner ) + if plug_runner is not None: + test_runner = plug_runner + + result = test_runner.run( tests ) + + rval = result.wasSuccessful() + + except: + log.exception( "Failure running tests" ) + + log.info( "Shutting down" ) + + # ---- Teardown ----------------------------------------------------------- + + if server: + log.info( "Shutting down embedded web server" ) + server.server_close() + server = None + log.info( "Embedded web server stopped" ) + if app: + log.info( "Shutting down app" ) + app.shutdown() + app = None + log.info( "Embedded Universe application stopped" ) + + return rval + +if __name__ == "__main__": + main() diff -r 2f3c4dd6ff1a -r fd5943ff7b40 test/base/twilltestcase.py --- a/test/base/twilltestcase.py Wed Oct 21 14:19:45 2009 -0400 +++ b/test/base/twilltestcase.py Tue Oct 20 13:08:29 2009 -0400 @@ -1,7 +1,7 @@ import pkg_resources pkg_resources.require( "twill==0.9" ) -import StringIO, os, sys, random, filecmp, time, unittest, urllib, logging, difflib, zipfile +import StringIO, os, sys, random, filecmp, time, unittest, urllib, logging, difflib, zipfile, tempfile from itertools import * import twill @@ -650,14 +650,24 @@ self.visit_page( "user/logout" ) self.check_page_for_string( "You are no longer logged in" ) self.home() + # Functions associated with browsers, cookies, HTML forms and page visits + def check_page_for_string( self, patt ): """Looks for 'patt' in the current browser page""" page = self.last_page() for subpatt in patt.split(): if page.find( patt ) == -1: - errmsg = "TwillAssertionError: no match to '%s'" %patt + fname = self.write_temp_file( page ) + errmsg = "no match to '%s'\npage content written to '%s'" % ( patt, fname ) raise AssertionError( errmsg ) + + def write_temp_file( self, content ): + fd, fname = tempfile.mkstemp( suffix='.html', prefix='twilltestcase-' ) + f = os.fdopen( fd, "w" ) + f.write( content ) + f.close() + return fname def clear_cookies( self ): tc.clear_cookies() @@ -825,7 +835,7 @@ def wait( self, maxiter=20 ): """Waits for the tools to finish""" count = 0 - sleep_amount = 1 + sleep_amount = 0.1 self.home() while count < maxiter: count += 1 @@ -833,7 +843,7 @@ page = tc.browser.get_html() if page.find( '<!-- running: do not change this comment, used by TwillTestCase.wait -->' ) > -1: time.sleep( sleep_amount ) - sleep_amount += 1 + sleep_amount *= 2 else: break self.assertNotEqual(count, maxiter) diff -r 2f3c4dd6ff1a -r fd5943ff7b40 test/functional/__init__.py --- a/test/functional/__init__.py Wed Oct 21 14:19:45 2009 -0400 +++ b/test/functional/__init__.py Tue Oct 20 13:08:29 2009 -0400 @@ -1,131 +1,3 @@ -import atexit, logging, os, sys, tempfile - -import pkg_resources - -pkg_resources.require( "twill==0.9" ) -pkg_resources.require( "Paste" ) -pkg_resources.require( "PasteDeploy" ) -pkg_resources.require( "Cheetah" ) - -import twill, unittest, time -import os, os.path, subprocess, sys, threading -import httplib -from paste import httpserver -import galaxy.app -from galaxy.app import UniverseApplication -from galaxy.web import buildapp -import test_toolbox -from galaxy import tools -from galaxy.util import bunch - -log = logging.getLogger( __name__ ) - -# TODO: should be able to override these, as well as prevent starting th -# server (for running the tests against a running instance) - -default_galaxy_test_host = "localhost" -default_galaxy_test_port = "8777" -default_galaxy_locales = 'en' -galaxy_test_file_dir = "test-data" -server = None -app = None - -def setup(): - """Start the web server for the tests""" - global server, app - - galaxy_test_host = os.environ.get( 'GALAXY_TEST_HOST', default_galaxy_test_host ) - galaxy_test_port = os.environ.get( 'GALAXY_TEST_PORT', default_galaxy_test_port ) - if 'HTTP_ACCEPT_LANGUAGE' not in os.environ: - os.environ['HTTP_ACCEPT_LANGUAGE'] = default_galaxy_locales - - start_server = 'GALAXY_TEST_EXTERNAL' not in os.environ - if start_server: - if 'GALAXY_TEST_DBPATH' in os.environ: - db_path = os.environ['GALAXY_TEST_DBPATH'] - else: - tempdir = tempfile.mkdtemp() - db_path = os.path.join( tempdir, 'database' ) - file_path = os.path.join( db_path, 'files' ) - try: - os.makedirs( file_path ) - except OSError: - pass - if 'GALAXY_TEST_DBURI' in os.environ: - database_connection = os.environ['GALAXY_TEST_DBURI'] - else: - database_connection = 'sqlite:///' + os.path.join( db_path, 'universe.sqlite' ) - if 'GALAXY_TEST_RUNNERS' in os.environ: - start_job_runners = os.environ['GALAXY_TEST_RUNNERS'] - else: - start_job_runners = None - if 'GALAXY_TEST_DEF_RUNNER' in os.environ: - default_cluster_job_runner = os.environ['GALAXY_TEST_DEF_RUNNER'] - else: - default_cluster_job_runner = 'local:///' - app = UniverseApplication( job_queue_workers = 5, - start_job_runners = start_job_runners, - default_cluster_job_runner = default_cluster_job_runner, - id_secret = 'changethisinproductiontoo', - template_path = "templates", - database_connection = database_connection, - file_path = file_path, - tool_config_file = "tool_conf.xml.sample", - datatype_converters_config_file = "datatype_converters_conf.xml.sample", - tool_path = "tools", - test_conf = "test.conf", - log_destination = "stdout", - use_heartbeat = False, - allow_user_creation = True, - allow_user_deletion = True, - admin_users = 'test@bx.psu.edu', - library_import_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ), - user_library_import_dir = os.path.join( os.getcwd(), galaxy_test_file_dir, 'users' ), - global_conf = { "__file__": "universe_wsgi.ini.sample" } ) - - log.info( "Embedded Universe application started" ) - - webapp = buildapp.app_factory( dict(), use_translogger = False, app=app ) - server = httpserver.serve( webapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False ) - - atexit.register( teardown ) - t = threading.Thread( target=server.serve_forever ) - t.start() - time.sleep( 2 ) - log.info( "Embedded web server started" ) - - if app: - # TODO: provisions for loading toolbox from file when using external server - test_toolbox.toolbox = app.toolbox - else: - # FIXME: This doesn't work at all now that toolbox requires an 'app' instance - # (to get at datatypes, might just pass a datatype registry directly) - my_app = bunch.Bunch( datatypes_registry = galaxy.datatypes.registry.Registry() ) - test_toolbox.toolbox = tools.ToolBox( 'tool_conf.xml.test', 'tools', my_app ) - - # Test if the server is up - conn = httplib.HTTPConnection( galaxy_test_host, galaxy_test_port ) - conn.request( "GET", "/" ) - assert conn.getresponse().status == 200, "Test HTTP server did not return '200 OK'" - - log.info( "Functional tests will be run against %s:%s" % ( galaxy_test_host, galaxy_test_port ) ) - - os.environ['GALAXY_TEST_HOST'] = galaxy_test_host - os.environ['GALAXY_TEST_PORT'] = galaxy_test_port - os.environ['GALAXY_TEST_FILE_DIR'] = galaxy_test_file_dir - -def teardown(): - global server, app - if server: - server.server_close() - server = None - log.info( "Embedded web server stopped" ) - if app: - app.shutdown() - app = None - log.info( "Embedded Universe application stopped" ) - time.sleep(2) - -# def test_toolbox(): -# for test in app.toolbox.get_tests(): -# yield ( test, ) +""" +Functional Tests +"""