3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6822f41bc9bb/ Changeset: 6822f41bc9bb Branch: stable User: Dave Bouvier Date: 2013-08-19 19:06:17 Summary: Fix for case where running functional tests might overwrite certain files in database/files. Affected #: 1 file diff -r d05bf67aefa670f4d004db3c1494e9052c0c0d9c -r 6822f41bc9bb2a2bf4673d6dcdeb1939730d970f scripts/functional_tests.py --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os, sys, shutil, tempfile, re +from ConfigParser import SafeConfigParser # Assume we are run from the galaxy root directory, add lib to the python path cwd = os.getcwd() @@ -86,6 +87,39 @@ global_conf.update( get_static_settings() ) return global_conf +def generate_config_file( input_filename, output_filename, config_items ): + ''' + Generate a config file with the configuration that has been defined for the embedded web application. + This is mostly relevant when setting metadata externally, since the script for doing that does not + have access to app.config. + ''' + cp = SafeConfigParser() + cp.read( input_filename ) + config_items_by_section = [] + for label, value in config_items: + found = False + # Attempt to determine the correct section for this configuration option. + for section in cp.sections(): + if cp.has_option( section, label ): + config_tuple = section, label, value + config_items_by_section.append( config_tuple ) + found = True + continue + # Default to app:main if no section was found. + if not found: + config_tuple = 'app:main', label, value + config_items_by_section.append( config_tuple ) + print( config_items_by_section ) + # Replace the default values with the provided configuration. + for section, label, value in config_items_by_section: + + if cp.has_option( section, label ): + cp.remove_option( section, label ) + cp.set( section, label, str( value ) ) + fh = open( output_filename, 'w' ) + cp.write( fh ) + fh.close() + def run_tests( test_config ): loader = nose.loader.TestLoader( config=test_config ) plug_loader = test_config.plugins.prepareTestLoader( loader ) @@ -145,6 +179,9 @@ shed_tool_data_table_config = 'shed_tool_data_table_conf.xml' tool_dependency_dir = os.environ.get( 'GALAXY_TOOL_DEPENDENCY_DIR', None ) use_distributed_object_store = os.environ.get( 'GALAXY_USE_DISTRIBUTED_OBJECT_STORE', False ) + galaxy_test_tmp_dir = os.environ.get( 'GALAXY_TEST_TMP_DIR', None ) + if galaxy_test_tmp_dir is None: + galaxy_test_tmp_dir = tempfile.mkdtemp() if start_server: psu_production = False @@ -193,13 +230,14 @@ default_cluster_job_runner = default_cluster_job_runner ) psu_production = True else: + # Configure the database path. if 'GALAXY_TEST_DBPATH' in os.environ: - db_path = os.environ['GALAXY_TEST_DBPATH'] + galaxy_db_path = os.environ[ 'GALAXY_TEST_DBPATH' ] else: - tempdir = tempfile.mkdtemp() - db_path = tempfile.mkdtemp( prefix='database_', dir=tempdir ) - # FIXME: This is a workaround for cases where metadata is being set externally. - file_path = os.path.join( 'database', 'files' ) + tempdir = tempfile.mkdtemp( dir=galaxy_test_tmp_dir ) + galaxy_db_path = os.path.join( tempdir, 'database' ) + # Configure the paths Galaxy needs to test tools. + file_path = os.path.join( galaxy_db_path, 'files' ) new_file_path = tempfile.mkdtemp( prefix='new_files_path_', dir=tempdir ) job_working_directory = tempfile.mkdtemp( prefix='job_working_directory_', dir=tempdir ) if 'GALAXY_TEST_DBURI' in os.environ: @@ -217,9 +255,32 @@ # ---- Build Application -------------------------------------------------- app = None if start_server: - global_conf = { '__file__' : 'universe_wsgi.ini.sample' } + kwargs = dict( admin_users = 'test@bx.psu.edu', + allow_library_path_paste = True, + allow_user_creation = True, + allow_user_deletion = True, + database_connection = database_connection, + datatype_converters_config_file = "datatype_converters_conf.xml.sample", + file_path = file_path, + id_secret = 'changethisinproductiontoo', + job_queue_workers = 5, + job_working_directory = job_working_directory, + library_import_dir = library_import_dir, + log_destination = "stdout", + new_file_path = new_file_path, + running_functional_tests = True, + shed_tool_data_table_config = shed_tool_data_table_config, + template_path = "templates", + test_conf = "test.conf", + tool_config_file = tool_config_file, + tool_data_table_config_path = tool_data_table_config_path, + tool_path = tool_path, + tool_parse_help = False, + update_integrated_tool_panel = False, + use_heartbeat = False, + user_library_import_dir = user_library_import_dir ) if psu_production: - global_conf = None + kwargs[ 'global_conf' ] = None if not database_connection.startswith( 'sqlite://' ): kwargs[ 'database_engine_option_max_overflow' ] = '20' kwargs[ 'database_engine_option_pool_size' ] = '10' @@ -228,40 +289,29 @@ if use_distributed_object_store: kwargs[ 'object_store' ] = 'distributed' kwargs[ 'distributed_object_store_config_file' ] = 'distributed_object_store_conf.xml.sample' + # If the user has passed in a path for the .ini file, do not overwrite it. + galaxy_config_file = os.environ.get( 'GALAXY_TEST_INI_FILE', None ) + if not galaxy_config_file: + galaxy_config_file = os.path.join( galaxy_test_tmp_dir, 'functional_tests_wsgi.ini' ) + config_items = [] + for label in kwargs: + config_tuple = label, kwargs[ label ] + config_items.append( config_tuple ) + # Write a temporary file, based on universe_wsgi.ini.sample, using the configuration options defined above. + generate_config_file( 'universe_wsgi.ini.sample', galaxy_config_file, config_items ) + # Set the global_conf[ '__file__' ] option to the location of the temporary .ini file, which gets passed to set_metadata.sh. + kwargs[ 'global_conf' ] = get_webapp_global_conf() + kwargs[ 'global_conf' ][ '__file__' ] = galaxy_config_file + kwargs[ 'config_file' ] = galaxy_config_file # Build the Universe Application - app = UniverseApplication( admin_users = 'test@bx.psu.edu', - allow_library_path_paste = True, - allow_user_creation = True, - allow_user_deletion = True, - database_connection = database_connection, - datatype_converters_config_file = "datatype_converters_conf.xml.sample", - file_path = file_path, - global_conf = global_conf, - id_secret = 'changethisinproductiontoo', - job_queue_workers = 5, - job_working_directory = job_working_directory, - library_import_dir = library_import_dir, - log_destination = "stdout", - new_file_path = new_file_path, - running_functional_tests = True, - shed_tool_data_table_config = shed_tool_data_table_config, - template_path = "templates", - test_conf = "test.conf", - tool_config_file = tool_config_file, - tool_data_table_config_path = tool_data_table_config_path, - tool_path = tool_path, - tool_parse_help = False, - update_integrated_tool_panel = False, - use_heartbeat = False, - user_library_import_dir = user_library_import_dir, - **kwargs ) + app = UniverseApplication( **kwargs ) log.info( "Embedded Universe application started" ) # ---- Run webserver ------------------------------------------------------ server = None if start_server: - webapp = buildapp.app_factory( get_webapp_global_conf(), app=app, + webapp = buildapp.app_factory( kwargs[ 'global_conf' ], app=app, use_translogger=False, static_enabled=STATIC_ENABLED ) if galaxy_test_port is not None: server = httpserver.serve( webapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False ) https://bitbucket.org/galaxy/galaxy-central/commits/2ca0b65d5005/ Changeset: 2ca0b65d5005 User: Dave Bouvier Date: 2013-08-19 19:06:42 Summary: Merge in bugfix from stable. Affected #: 2 files diff -r 2d64260eb6e31a992b78d82ee50f351bc3b95d07 -r 2ca0b65d5005f35df6b9c05ebcec64f9164ab972 .hgtags --- a/.hgtags +++ b/.hgtags @@ -3,3 +3,4 @@ 75f09617abaadbc8cc732bb8ee519decaeb56ea7 release_2013.04.01 2cc8d10988e03257dc7b97f8bb332c7df745d1dd security_2013.04.08 524f246ca85395082719ae7a6ff72260d7ad5612 release_2013.06.03 +1ae95b3aa98d1ccf15b243ac3ce6a895eb7efc53 release_2013.08.12 diff -r 2d64260eb6e31a992b78d82ee50f351bc3b95d07 -r 2ca0b65d5005f35df6b9c05ebcec64f9164ab972 scripts/functional_tests.py --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os, sys, shutil, tempfile, re +from ConfigParser import SafeConfigParser # Assume we are run from the galaxy root directory, add lib to the python path cwd = os.getcwd() @@ -86,6 +87,39 @@ global_conf.update( get_static_settings() ) return global_conf +def generate_config_file( input_filename, output_filename, config_items ): + ''' + Generate a config file with the configuration that has been defined for the embedded web application. + This is mostly relevant when setting metadata externally, since the script for doing that does not + have access to app.config. + ''' + cp = SafeConfigParser() + cp.read( input_filename ) + config_items_by_section = [] + for label, value in config_items: + found = False + # Attempt to determine the correct section for this configuration option. + for section in cp.sections(): + if cp.has_option( section, label ): + config_tuple = section, label, value + config_items_by_section.append( config_tuple ) + found = True + continue + # Default to app:main if no section was found. + if not found: + config_tuple = 'app:main', label, value + config_items_by_section.append( config_tuple ) + print( config_items_by_section ) + # Replace the default values with the provided configuration. + for section, label, value in config_items_by_section: + + if cp.has_option( section, label ): + cp.remove_option( section, label ) + cp.set( section, label, str( value ) ) + fh = open( output_filename, 'w' ) + cp.write( fh ) + fh.close() + def run_tests( test_config ): loader = nose.loader.TestLoader( config=test_config ) plug_loader = test_config.plugins.prepareTestLoader( loader ) @@ -145,6 +179,9 @@ shed_tool_data_table_config = 'shed_tool_data_table_conf.xml' tool_dependency_dir = os.environ.get( 'GALAXY_TOOL_DEPENDENCY_DIR', None ) use_distributed_object_store = os.environ.get( 'GALAXY_USE_DISTRIBUTED_OBJECT_STORE', False ) + galaxy_test_tmp_dir = os.environ.get( 'GALAXY_TEST_TMP_DIR', None ) + if galaxy_test_tmp_dir is None: + galaxy_test_tmp_dir = tempfile.mkdtemp() if start_server: psu_production = False @@ -193,13 +230,14 @@ default_cluster_job_runner = default_cluster_job_runner ) psu_production = True else: + # Configure the database path. if 'GALAXY_TEST_DBPATH' in os.environ: - db_path = os.environ['GALAXY_TEST_DBPATH'] + galaxy_db_path = os.environ[ 'GALAXY_TEST_DBPATH' ] else: - tempdir = tempfile.mkdtemp() - db_path = tempfile.mkdtemp( prefix='database_', dir=tempdir ) - # FIXME: This is a workaround for cases where metadata is being set externally. - file_path = os.path.join( 'database', 'files' ) + tempdir = tempfile.mkdtemp( dir=galaxy_test_tmp_dir ) + galaxy_db_path = os.path.join( tempdir, 'database' ) + # Configure the paths Galaxy needs to test tools. + file_path = os.path.join( galaxy_db_path, 'files' ) new_file_path = tempfile.mkdtemp( prefix='new_files_path_', dir=tempdir ) job_working_directory = tempfile.mkdtemp( prefix='job_working_directory_', dir=tempdir ) if 'GALAXY_TEST_DBURI' in os.environ: @@ -217,9 +255,32 @@ # ---- Build Application -------------------------------------------------- app = None if start_server: - global_conf = { '__file__' : 'universe_wsgi.ini.sample' } + kwargs = dict( admin_users = 'test@bx.psu.edu', + allow_library_path_paste = True, + allow_user_creation = True, + allow_user_deletion = True, + database_connection = database_connection, + datatype_converters_config_file = "datatype_converters_conf.xml.sample", + file_path = file_path, + id_secret = 'changethisinproductiontoo', + job_queue_workers = 5, + job_working_directory = job_working_directory, + library_import_dir = library_import_dir, + log_destination = "stdout", + new_file_path = new_file_path, + running_functional_tests = True, + shed_tool_data_table_config = shed_tool_data_table_config, + template_path = "templates", + test_conf = "test.conf", + tool_config_file = tool_config_file, + tool_data_table_config_path = tool_data_table_config_path, + tool_path = tool_path, + tool_parse_help = False, + update_integrated_tool_panel = False, + use_heartbeat = False, + user_library_import_dir = user_library_import_dir ) if psu_production: - global_conf = None + kwargs[ 'global_conf' ] = None if not database_connection.startswith( 'sqlite://' ): kwargs[ 'database_engine_option_max_overflow' ] = '20' kwargs[ 'database_engine_option_pool_size' ] = '10' @@ -228,40 +289,29 @@ if use_distributed_object_store: kwargs[ 'object_store' ] = 'distributed' kwargs[ 'distributed_object_store_config_file' ] = 'distributed_object_store_conf.xml.sample' + # If the user has passed in a path for the .ini file, do not overwrite it. + galaxy_config_file = os.environ.get( 'GALAXY_TEST_INI_FILE', None ) + if not galaxy_config_file: + galaxy_config_file = os.path.join( galaxy_test_tmp_dir, 'functional_tests_wsgi.ini' ) + config_items = [] + for label in kwargs: + config_tuple = label, kwargs[ label ] + config_items.append( config_tuple ) + # Write a temporary file, based on universe_wsgi.ini.sample, using the configuration options defined above. + generate_config_file( 'universe_wsgi.ini.sample', galaxy_config_file, config_items ) + # Set the global_conf[ '__file__' ] option to the location of the temporary .ini file, which gets passed to set_metadata.sh. + kwargs[ 'global_conf' ] = get_webapp_global_conf() + kwargs[ 'global_conf' ][ '__file__' ] = galaxy_config_file + kwargs[ 'config_file' ] = galaxy_config_file # Build the Universe Application - app = UniverseApplication( admin_users = 'test@bx.psu.edu', - allow_library_path_paste = True, - allow_user_creation = True, - allow_user_deletion = True, - database_connection = database_connection, - datatype_converters_config_file = "datatype_converters_conf.xml.sample", - file_path = file_path, - global_conf = global_conf, - id_secret = 'changethisinproductiontoo', - job_queue_workers = 5, - job_working_directory = job_working_directory, - library_import_dir = library_import_dir, - log_destination = "stdout", - new_file_path = new_file_path, - running_functional_tests = True, - shed_tool_data_table_config = shed_tool_data_table_config, - template_path = "templates", - test_conf = "test.conf", - tool_config_file = tool_config_file, - tool_data_table_config_path = tool_data_table_config_path, - tool_path = tool_path, - tool_parse_help = False, - update_integrated_tool_panel = False, - use_heartbeat = False, - user_library_import_dir = user_library_import_dir, - **kwargs ) + app = UniverseApplication( **kwargs ) log.info( "Embedded Universe application started" ) # ---- Run webserver ------------------------------------------------------ server = None if start_server: - webapp = buildapp.app_factory( get_webapp_global_conf(), app=app, + webapp = buildapp.app_factory( kwargs[ 'global_conf' ], app=app, use_translogger=False, static_enabled=STATIC_ENABLED ) if galaxy_test_port is not None: server = httpserver.serve( webapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False ) https://bitbucket.org/galaxy/galaxy-central/commits/e9ee9c5d30ae/ Changeset: e9ee9c5d30ae User: Dave Bouvier Date: 2013-08-19 19:08:45 Summary: Remove duplicate method from the install and test's functional_tests.py. Affected #: 1 file diff -r 2ca0b65d5005f35df6b9c05ebcec64f9164ab972 -r e9ee9c5d30aef4390a94823990221d3876785726 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -6,7 +6,6 @@ import os, sys, shutil, tempfile, re, string, urllib, platform from time import strftime -from ConfigParser import SafeConfigParser # Assume we are run from the galaxy root directory, add lib to the python path cwd = os.getcwd() @@ -21,10 +20,11 @@ default_galaxy_locales = 'en' default_galaxy_test_file_dir = "test-data" os.environ[ 'GALAXY_INSTALL_TEST_TMP_DIR' ] = galaxy_test_tmp_dir -new_path = [ os.path.join( cwd, "lib" ), os.path.join( cwd, 'test' ), os.path.join( cwd, 'scripts', 'api' ) ] +new_path = [ os.path.join( cwd, "scripts" ), os.path.join( cwd, "lib" ), os.path.join( cwd, 'test' ), os.path.join( cwd, 'scripts', 'api' ) ] new_path.extend( sys.path ) sys.path = new_path +from functional_tests import generate_config_file from galaxy import eggs eggs.require( "nose" ) @@ -254,36 +254,6 @@ success = result.wasSuccessful() return success -def generate_config_file( input_filename, output_filename, config_items ): - ''' - Generate a config file with the configuration that has been defined for the embedded web application. - This is mostly relevant when setting metadata externally, since the script for doing that does not - have access to app.config. - ''' - cp = SafeConfigParser() - cp.read( input_filename ) - config_items_by_section = [] - for label, value in config_items: - found = False - # Attempt to determine the correct section for this configuration option. - for section in cp.sections(): - if cp.has_option( section, label ): - config_tuple = section, label, value - config_items_by_section.append( config_tuple ) - found = True - continue - # Default to app:main if no section was found. - if not found: - config_tuple = 'app:main', label, value - config_items_by_section.append( config_tuple ) - # Replace the default values with the provided configuration. - for section, label, value in config_items_by_section: - cp.remove_option( section, label ) - cp.set( section, label, str( value ) ) - fh = open( output_filename, 'w' ) - cp.write( fh ) - fh.close() - def get_api_url( base, parts=[], params=None, key=None ): if 'api' in parts and parts.index( 'api' ) != 0: parts.pop( parts.index( 'api' ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.