1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/ae60aaaf6a13/ changeset: ae60aaaf6a13 user: inithello date: 2012-12-12 22:56:14 summary: Added functional tests to verify repository installation on the Galaxy side. affected #: 6 files diff -r dad76985fc571feed65cc63de37b2685312348e3 -r ae60aaaf6a139e88b2849abf97fd47af711341fd test/tool_shed/base/test_db_util.py --- a/test/tool_shed/base/test_db_util.py +++ b/test/tool_shed/base/test_db_util.py @@ -1,6 +1,8 @@ +import galaxy.model import galaxy.webapps.community.model as model from galaxy.model.orm import * from galaxy.webapps.community.model.mapping import context as sa_session +from galaxy.model.mapping import context as ga_session def delete_obj( obj ): sa_session.delete( obj ) @@ -12,6 +14,10 @@ def flush( obj ): sa_session.add( obj ) sa_session.flush() +def get_category_by_name( name ): + return sa_session.query( model.Category ) \ + .filter( model.Category.table.c.name == name ) \ + .first() def get_default_user_permissions_by_role( role ): return sa_session.query( model.DefaultUserPermissions ) \ .filter( model.DefaultUserPermissions.table.c.role_id == role.id ) \ @@ -20,6 +26,12 @@ return sa_session.query( model.DefaultUserPermissions ) \ .filter( model.DefaultUserPermissions.table.c.user_id==user.id ) \ .all() +def get_galaxy_repository_by_name_owner_changeset_revision( repository_name, owner, changeset_revision ): + return ga_session.query( galaxy.model.ToolShedRepository ) \ + .filter( and_( galaxy.model.ToolShedRepository.table.c.name == repository_name, + galaxy.model.ToolShedRepository.table.c.owner == owner, + galaxy.model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ + .first() def get_private_role( user ): for role in user.all_roles(): if role.name == user.email and role.description == 'Private Role for %s' % user.email: @@ -39,6 +51,17 @@ sa_session.flush() def refresh( obj ): sa_session.refresh( obj ) +def ga_refresh( obj ): + ga_session.refresh( obj ) +def get_galaxy_private_role( user ): + for role in user.all_roles(): + if role.name == user.email and role.description == 'Private Role for %s' % user.email: + return role + raise AssertionError( "Private role not found for user '%s'" % user.email ) +def get_galaxy_user( email ): + return ga_session.query( galaxy.model.User ) \ + .filter( galaxy.model.User.table.c.email==email ) \ + .first() def get_repository_by_name_and_owner( name, owner_username ): owner = get_user_by_name( owner_username ) repository = sa_session.query( model.Repository ) \ diff -r dad76985fc571feed65cc63de37b2685312348e3 -r ae60aaaf6a139e88b2849abf97fd47af711341fd test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -1,7 +1,9 @@ import galaxy.webapps.community.util.hgweb_config -import common, string, os +import galaxy.model as galaxy_model +import common, string, os, re from base.twilltestcase import tc, from_json_string, TwillTestCase, security -from test_db_util import get_repository_metadata_by_repository_id_changeset_revision +from test_db_util import get_repository_by_name_and_owner, get_repository_metadata_by_repository_id_changeset_revision, \ + get_galaxy_repository_by_name_owner_changeset_revision from galaxy import eggs eggs.require('mercurial') @@ -19,14 +21,25 @@ self.host = os.environ.get( 'TOOL_SHED_TEST_HOST' ) self.port = os.environ.get( 'TOOL_SHED_TEST_PORT' ) self.url = "http://%s:%s" % ( self.host, self.port ) + self.galaxy_host = os.environ.get( 'GALAXY_TEST_HOST' ) + self.galaxy_port = os.environ.get( 'GALAXY_TEST_PORT' ) + self.galaxy_url = "http://%s:%s" % ( self.galaxy_host, self.galaxy_port ) self.file_dir = os.environ.get( 'TOOL_SHED_TEST_FILE_DIR', None ) self.tool_shed_test_file = None self.shed_tools_dict = {} self.home() + def browse_category( self, category, strings_displayed=[], strings_not_displayed=[] ): + url = '/repository/browse_valid_categories?sort=name&operation=valid_repositories_by_category&id=%s' % \ + self.security.encode_id( category.id ) + self.visit_url( url ) + self.check_for_strings( strings_displayed, strings_not_displayed ) def browse_repository( self, repository, strings_displayed=[], strings_not_displayed=[] ): url = '/repository/browse_repository?id=%s' % self.security.encode_id( repository.id ) self.visit_url( url ) self.check_for_strings( strings_displayed, strings_not_displayed ) + def browse_tool_shed( self, url, strings_displayed=[], strings_not_displayed=[] ): + self.visit_galaxy_url( '/admin_toolshed/browse_tool_shed?tool_shed_url=%s' % url ) + self.check_for_strings( strings_displayed, strings_not_displayed ) def check_for_strings( self, strings_displayed=[], strings_not_displayed=[] ): if strings_displayed: for string in strings_displayed: @@ -122,6 +135,36 @@ tc.fv( "1", "category_id", "+%s" % category ) tc.submit( "create_repository_button" ) self.check_for_strings( strings_displayed, strings_not_displayed ) + def create_user_in_galaxy( self, cntrller='user', email='test@bx.psu.edu', password='testuser', username='admin-user', redirect='' ): + self.visit_galaxy_url( "/user/create?cntrller=%s&use_panels=False" % cntrller ) + tc.fv( '1', 'email', email ) + tc.fv( '1', 'redirect', redirect ) + tc.fv( '1', 'password', password ) + tc.fv( '1', 'confirm', password ) + tc.fv( '1', 'username', username ) + tc.submit( 'create_user_button' ) + previously_created = False + username_taken = False + invalid_username = False + try: + self.check_page_for_string( "Created new user account" ) + except: + try: + # May have created the account in a previous test run... + self.check_page_for_string( "User with that email already exists" ) + previously_created = True + except: + try: + self.check_page_for_string( 'Public name is taken; please choose another' ) + username_taken = True + except: + try: + # Note that we're only checking if the usr name is >< 4 chars here... + self.check_page_for_string( 'Public name must be at least 4 characters in length' ) + invalid_username = True + except: + pass + return previously_created, username_taken, invalid_username def delete_files_from_repository( self, repository, filenames=[], strings_displayed=[ 'were deleted from the repository' ], strings_not_displayed=[] ): files_to_delete = [] basepath = self.get_repo_path( repository ) @@ -212,6 +255,21 @@ else: string = string.replace( character, replacement ) return string + def galaxy_login( self, email='test@bx.psu.edu', password='testuser', username='admin-user', redirect='' ): + previously_created, username_taken, invalid_username = \ + self.create_user_in_galaxy( email=email, password=password, username=username, redirect=redirect ) + if previously_created: + self.visit_galaxy_url( "/user/login?use_panels=False" ) + tc.fv( '1', 'email', email ) + tc.fv( '1', 'redirect', redirect ) + tc.fv( '1', 'password', password ) + tc.submit( 'login_button' ) + def galaxy_logout( self ): + self.home() + self.visit_galaxy_url( "/user/logout" ) + self.check_page_for_string( "You have been logged out" ) + self.home() + def generate_repository_dependency_xml( self, repositories, xml_filename, dependency_description='' ): file_path = os.path.split( xml_filename )[0] if not os.path.exists( file_path ): @@ -239,9 +297,6 @@ return os.path.abspath( os.path.join( filepath, filename ) ) else: return os.path.abspath( os.path.join( self.file_dir, filename ) ) - def get_latest_repository_metadata_for_repository( self, repository ): - # TODO: This will not work as expected. Fix it. - return repository.metadata_revisions[ 0 ] def get_repo_path( self, repository ): # An entry in the hgweb.config file looks something like: repos/test/mira_assembler = database/community_files/000/repo_123 lhs = "repos/%s/%s" % ( repository.user.username, repository.name ) @@ -304,6 +359,29 @@ tc.fv( "3", "allow_push", '+%s' % username ) tc.submit( 'user_access_button' ) self.check_for_strings( strings_displayed, strings_not_displayed ) + def install_repository( self, name, owner, changeset_revision=None, strings_displayed=[], strings_not_displayed=[] ): + repository = get_repository_by_name_and_owner( name, owner ) + repository_id = self.security.encode_id( repository.id ) + if changeset_revision is None: + changeset_revision = self.get_repository_tip( repository ) + url = '/repository/install_repositories_by_revision?changeset_revisions=%s&repository_ids=%s&galaxy_url=%s' % \ + ( changeset_revision, repository_id, self.galaxy_url ) + self.visit_url( url ) + self.check_for_strings( strings_displayed, strings_not_displayed ) + tc.submit( 'select_tool_panel_section_button' ) + html = self.last_page() + # Since the installation process is by necessity asynchronous, we have to get the parameters to 'manually' initiate the + # installation process. This regex will return the tool shed repository IDs in group(1), the encoded_kwd parameter in + # group(2), and the reinstalling flag in group(3) and pass them to the manage_repositories method in the Galaxy + # admin_toolshed controller. + install_parameters = re.search( 'initiate_repository_installation\( "([^"]+)", "([^"]+)", "([^"]+)" \);', html ) + iri_ids = install_parameters.group(1) + encoded_kwd = install_parameters.group(2) + reinstalling = install_parameters.group(3) + url = '/admin_toolshed/manage_repositories?operation=install&tool_shed_repository_ids=%s&encoded_kwd=%s&reinstalling=%s' % \ + ( iri_ids, encoded_kwd, reinstalling ) + self.visit_galaxy_url( url ) + self.wait_for_repository_installation( repository ) def load_invalid_tool_page( self, repository, tool_xml, changeset_revision, strings_displayed=[], strings_not_displayed=[] ): url = '/repository/load_invalid_tool?repository_id=%s&tool_config=%s&changeset_revision=%s' % \ ( self.security.encode_id( repository.id ), tool_xml, changeset_revision ) @@ -314,6 +392,13 @@ ( self.security.encode_id( repository.id ), tool_xml_path, changeset_revision ) self.visit_url( url ) self.check_for_strings( strings_displayed, strings_not_displayed ) + def preview_repository_in_tool_shed( self, name, owner, changeset_revision=None, strings_displayed=[], strings_not_displayed=[] ): + repository = get_repository_by_name_and_owner( name, owner ) + if changeset_revision is None: + changeset_revision = self.get_repository_tip( repository ) + self.visit_url( '/repository/preview_tools_in_changeset?repository_id=%s&changeset_revision=%s' % \ + ( self.security.encode_id( repository.id ), changeset_revision ) ) + self.check_for_strings( strings_displayed, strings_not_displayed ) def repository_is_new( self, repository ): repo = hg.repository( ui.ui(), self.get_repo_path( repository ) ) tip_ctx = repo.changectx( repo.changelog.tip() ) @@ -361,3 +446,18 @@ tc.formfile( "1", "file_data", self.get_filename( filename, filepath ) ) tc.submit( "upload_button" ) self.check_for_strings( strings_displayed, strings_not_displayed ) + def visit_galaxy_url( self, url ): + url = '%s%s' % ( self.galaxy_url, url ) + self.visit_url( url ) + def wait_for_repository_installation( self, repository ): + final_states = [ galaxy_model.ToolShedRepository.installation_status.ERROR, + galaxy_model.ToolShedRepository.installation_status.INSTALLED, + galaxy_model.ToolShedRepository.installation_status.UNINSTALLED, + galaxy_model.ToolShedRepository.installation_status.DEACTIVATED ] + repository_name = repository.name + owner = repository.user.username + changeset_revision = self.get_repository_tip( repository ) + galaxy_repository = get_galaxy_repository_by_name_owner_changeset_revision( repository_name, owner, changeset_revision ) + while galaxy_repository.status not in final_states: + ga_refresh( galaxy_repository ) + time.sleep( 1 ) diff -r dad76985fc571feed65cc63de37b2685312348e3 -r ae60aaaf6a139e88b2849abf97fd47af711341fd test/tool_shed/functional/test_0030_repository_dependency_revisions.py --- a/test/tool_shed/functional/test_0030_repository_dependency_revisions.py +++ b/test/tool_shed/functional/test_0030_repository_dependency_revisions.py @@ -1,7 +1,7 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os from tool_shed.base.test_db_util import get_repository_by_name_and_owner, get_user, get_private_role -datatypes_repository_name = 'emboss_datatypes' +datatypes_repository_name = 'emboss_datatypes_0030' datatypes_repository_description = "Galaxy applicable data formats used by Emboss tools." datatypes_repository_long_description = "Galaxy applicable data formats used by Emboss tools. This repository contains no tools." @@ -29,7 +29,7 @@ """Create a category for this test suite""" self.create_category( 'Test 0030 Repository Dependency Revisions', 'Testing repository dependencies by revision.' ) def test_0010_create_repositories( self ): - '''Create the emboss_5_0030, emboss_6_0030, emboss_datatypes, and emboss repositories and populate the emboss_datatypes repository.''' + '''Create the emboss_5_0030, emboss_6_0030, emboss_datatypes_0030, and emboss_0030 repositories and populate the emboss_datatypes repository.''' self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) emboss_5_repository = get_repository_by_name_and_owner( emboss_5_repository_name, common.test_user_1_name ) diff -r dad76985fc571feed65cc63de37b2685312348e3 -r ae60aaaf6a139e88b2849abf97fd47af711341fd test/tool_shed/functional/test_0050_circular_n_levels.py --- a/test/tool_shed/functional/test_0050_circular_n_levels.py +++ b/test/tool_shed/functional/test_0050_circular_n_levels.py @@ -79,8 +79,8 @@ repository_long_description=filtering_repository_long_description, categories=[ default_category ], strings_displayed=[] ) - repository = get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) - self.upload_file( repository, + filtering_repository = get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) + self.upload_file( filtering_repository, 'filtering/filtering_1.1.0.tar', strings_displayed=[], commit_message='Uploaded filtering.tar.' ) @@ -89,7 +89,7 @@ self.generate_repository_dependency_xml( [ emboss_repository ], self.get_filename( 'repository_dependencies.xml', filepath=repository_dependencies_path ), dependency_description='Filtering depends on the emboss repository.' ) - self.upload_file( repository, + self.upload_file( filtering_repository, 'repository_dependencies.xml', filepath=repository_dependencies_path, commit_message='Uploaded dependency on emboss.' ) @@ -109,6 +109,13 @@ emboss_repository = get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) filtering_repository = get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) repository_dependencies_path = self.generate_temp_path( 'test_0050', additional_paths=[ 'freebayes' ] ) + self.generate_repository_dependency_xml( [ filtering_repository ], + self.get_filename( 'repository_dependencies.xml', filepath=repository_dependencies_path ), + dependency_description='Emboss depends on the filtering repository.' ) + self.upload_file( emboss_repository, + 'repository_dependencies.xml', + filepath=repository_dependencies_path, + commit_message='Uploaded dependency on filtering.' ) previous_tip = self.get_repository_tip( repository ) self.generate_repository_dependency_xml( [ emboss_datatypes_repository, emboss_repository, filtering_repository, repository ], self.get_filename( 'repository_dependencies.xml', filepath=repository_dependencies_path ), @@ -119,11 +126,24 @@ commit_message='Uploaded dependency on filtering.' ) self.display_manage_repository_page( repository, strings_not_displayed=[ previous_tip ] ) def test_0030_verify_repository_dependencies( self ): - '''Verify that the generated dependency circle does not cause an infinite loop.''' + '''Verify that the generated dependency circle does not cause an infinite loop. + + Expected structure: + + id: 2 key: http://localhost:8634__ESEP__freebayes_0050__ESEP__user1__ESEP__2e73d8e1b59d + ['http://localhost:8634', 'emboss_datatypes_0050', 'user1', '596029c334b1'] + ['http://localhost:8634', 'emboss_0050', 'user1', '9f1503046640'] + id: 3 key: http://localhost:8634__ESEP__filtering_0050__ESEP__user1__ESEP__eefdd8bc0db9 + ['http://localhost:8634', 'emboss_0050', 'user1', '9f1503046640'] + id: 4 key: http://localhost:8634__ESEP__emboss_0050__ESEP__user1__ESEP__9f1503046640 + ['http://localhost:8634', 'emboss_datatypes_0050', 'user1', '596029c334b1'] + ''' emboss_datatypes_repository = get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) emboss_repository = get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) filtering_repository = get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) freebayes_repository = get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) for repository in [ emboss_datatypes_repository, emboss_repository, filtering_repository ]: self.check_repository_dependency( freebayes_repository, repository, self.get_repository_tip( repository ) ) + for changeset_revision in self.get_repository_metadata_revisions( emboss_repository ): + self.check_repository_dependency( freebayes_repository, emboss_repository, changeset_revision ) self.display_manage_repository_page( freebayes_repository, strings_displayed=[ 'Freebayes depends on the filtering repository.' ] ) diff -r dad76985fc571feed65cc63de37b2685312348e3 -r ae60aaaf6a139e88b2849abf97fd47af711341fd test/tool_shed/functional/test_1000_install_filtering_repository.py --- /dev/null +++ b/test/tool_shed/functional/test_1000_install_filtering_repository.py @@ -0,0 +1,31 @@ +from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os +from tool_shed.base.test_db_util import get_repository_by_name_and_owner, get_galaxy_user, get_galaxy_private_role, get_category_by_name + +class BasicToolShedFeatures( ShedTwillTestCase ): + '''Test installing a basic repository.''' + def test_0000_initiate_users( self ): + """Create necessary user accounts.""" + self.galaxy_logout() + self.galaxy_login( email=common.test_user_1_email, username=common.test_user_1_name ) + test_user_1 = get_galaxy_user( common.test_user_1_email ) + assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email + test_user_1_private_role = get_galaxy_private_role( test_user_1 ) + self.galaxy_logout() + self.galaxy_login( email=common.admin_email, username=common.admin_username ) + admin_user = get_galaxy_user( common.admin_email ) + assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email + admin_user_private_role = get_galaxy_private_role( admin_user ) + def test_0005_browse_tool_sheds( self ): + """Browse the available tool sheds in this Galaxy instance.""" + self.visit_galaxy_url( '/admin_toolshed/browse_tool_sheds' ) + self.check_page_for_string( 'Embedded tool shed for functional tests' ) + self.browse_tool_shed( url=self.url, strings_displayed=[ 'Test 0000 Basic Repository Features 1', 'Test 0000 Basic Repository Features 2' ] ) + def test_0010_browse_test_0000_category( self ): + '''Browse the category created in test 0000. It should contain the filtering_0000 repository also created in that test.''' + category = get_category_by_name( 'Test 0000 Basic Repository Features 1' ) + self.browse_category( category, strings_displayed=[ 'filtering_0000' ] ) + def test_0015_preview_filtering_repository( self ): + '''Load the preview page for the filtering_0000 repository in the tool shed.''' + self.preview_repository_in_tool_shed( 'filtering_0000', common.test_user_1_name, strings_displayed=[ 'filtering_0000', 'Valid tools' ] ) + def test_0020_install_filtering_repository( self ): + self.install_repository( 'filtering_0000', common.test_user_1_name ) diff -r dad76985fc571feed65cc63de37b2685312348e3 -r ae60aaaf6a139e88b2849abf97fd47af711341fd test/tool_shed/functional_tests.py --- a/test/tool_shed/functional_tests.py +++ b/test/tool_shed/functional_tests.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -import os, sys, shutil, tempfile, re +import os, sys, shutil, tempfile, re, string # Assume we are run from the galaxy root directory, add lib to the python path cwd = os.getcwd() @@ -33,9 +33,14 @@ import sys, threading, random import httplib, socket from paste import httpserver +# This is for the tool shed application. import galaxy.webapps.community.app -from galaxy.webapps.community.app import UniverseApplication -from galaxy.webapps.community import buildapp +from galaxy.webapps.community.app import UniverseApplication as ToolshedUniverseApplication +from galaxy.webapps.community import buildapp as toolshedbuildapp +# This is for the galaxy application. +import galaxy.app +from galaxy.app import UniverseApplication as GalaxyUniverseApplication +from galaxy.web import buildapp as galaxybuildapp import nose.core import nose.config @@ -46,8 +51,22 @@ default_tool_shed_test_host = "localhost" default_tool_shed_test_port_min = 8000 -default_tool_shed_test_port_max = 9999 +default_tool_shed_test_port_max = 8999 default_tool_shed_locales = 'en' +default_galaxy_test_port_min = 9000 +default_galaxy_test_port_max = 9999 +default_galaxy_test_host = 'localhost' + +tool_sheds_conf_xml_template = '''<?xml version="1.0"?> +<tool_sheds> + <tool_shed name="Embedded tool shed for functional tests" url="http://${shed_url}:${shed_port}/"/> +</tool_sheds> +''' + +shed_tool_conf_xml_template = '''<?xml version="1.0"?> +<toolbox tool_path="${shed_tool_path}"> +</toolbox> +''' def run_tests( test_config ): loader = nose.loader.TestLoader( config=test_config ) @@ -67,6 +86,8 @@ # ---- Configuration ------------------------------------------------------ tool_shed_test_host = os.environ.get( 'TOOL_SHED_TEST_HOST', default_tool_shed_test_host ) tool_shed_test_port = os.environ.get( 'TOOL_SHED_TEST_PORT', None ) + galaxy_test_host = os.environ.get( 'GALAXY_TEST_HOST', default_galaxy_test_host ) + galaxy_test_port = os.environ.get( 'GALAXY_TEST_PORT', None ) tool_path = os.environ.get( 'TOOL_SHED_TEST_TOOL_PATH', 'tools' ) if 'HTTP_ACCEPT_LANGUAGE' not in os.environ: os.environ[ 'HTTP_ACCEPT_LANGUAGE' ] = default_tool_shed_locales @@ -85,18 +106,33 @@ if not os.path.isdir( tool_shed_test_tmp_dir ): os.mkdir( tool_shed_test_tmp_dir ) tool_shed_test_proxy_port = None + galaxy_test_proxy_port = None if 'TOOL_SHED_TEST_DBPATH' in os.environ: - db_path = os.environ[ 'TOOL_SHED_TEST_DBPATH' ] + shed_db_path = os.environ[ 'TOOL_SHED_TEST_DBPATH' ] else: tempdir = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir ) - db_path = os.path.join( tempdir, 'database' ) - file_path = os.path.join( db_path, 'files' ) + shed_db_path = os.path.join( tempdir, 'database' ) + galaxy_shed_tool_conf_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_tool_conf.xml' ) ) + galaxy_tool_sheds_conf_file = os.environ.get( 'GALAXY_TEST_SHED_TOOLS_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_sheds_conf.xml' ) ) + if 'GALAXY_TEST_DBPATH' in os.environ: + galaxy_db_path = os.environ[ 'GALAXY_TEST_DBPATH' ] + else: + tempdir = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir ) + galaxy_db_path = os.path.join( tempdir, 'database' ) + shed_file_path = os.path.join( shed_db_path, 'files' ) + galaxy_file_path = os.path.join( galaxy_db_path, 'files' ) hgweb_config_file_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir ) new_repos_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir ) + galaxy_shed_tool_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir ) + galaxy_tool_dependency_dir = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir ) if 'TOOL_SHED_TEST_DBURI' in os.environ: - database_connection = os.environ[ 'TOOL_SHED_TEST_DBURI' ] + toolshed_database_connection = os.environ[ 'TOOL_SHED_TEST_DBURI' ] else: - database_connection = 'sqlite:///' + os.path.join( db_path, 'universe.sqlite' ) + toolshed_database_connection = 'sqlite:///' + os.path.join( shed_db_path, 'community_test.sqlite' ) + if 'GALAXY_TEST_DBURI' in os.environ: + galaxy_database_connection = os.environ[ 'GALAXY_TEST_DBURI' ] + else: + galaxy_database_connection = 'sqlite:///' + os.path.join( galaxy_db_path, 'universe_test.sqlite' ) kwargs = {} for dir in [ tool_shed_test_tmp_dir ]: try: @@ -104,30 +140,31 @@ except OSError: pass - print "Database connection:", database_connection + print "Tool shed database connection:", toolshed_database_connection + print "Galaxy database connection:", galaxy_database_connection hgweb_config_dir = hgweb_config_file_path os.environ[ 'TEST_HG_WEB_CONFIG_DIR' ] = hgweb_config_dir print "Directory location for hgweb.config:", hgweb_config_dir - # ---- Build Application -------------------------------------------------- - app = None + # ---- Build Tool Shed Application -------------------------------------------------- + toolshedapp = None global_conf = { '__file__' : 'community_wsgi.ini.sample' } - if not database_connection.startswith( 'sqlite://' ): - kwargs[ 'database_engine_option_max_overflow' ] = '20' +# if not toolshed_database_connection.startswith( 'sqlite://' ): +# kwargs[ 'database_engine_option_max_overflow' ] = '20' if tool_dependency_dir is not None: kwargs[ 'tool_dependency_dir' ] = tool_dependency_dir if use_distributed_object_store: kwargs[ 'object_store' ] = 'distributed' kwargs[ 'distributed_object_store_config_file' ] = 'distributed_object_store_conf.xml.sample' - app = UniverseApplication( job_queue_workers = 5, + toolshedapp = ToolshedUniverseApplication( job_queue_workers = 5, id_secret = 'changethisinproductiontoo', template_path = 'templates', - database_connection = database_connection, + database_connection = toolshed_database_connection, database_engine_option_pool_size = '10', - file_path = file_path, + file_path = shed_file_path, new_file_path = new_repos_path, tool_path=tool_path, datatype_converters_config_file = 'datatype_converters_conf.xml.sample', @@ -144,23 +181,23 @@ hgweb_config_dir = hgweb_config_dir, **kwargs ) - log.info( "Embedded Universe application started" ) + log.info( "Embedded Toolshed application started" ) - # ---- Run webserver ------------------------------------------------------ - server = None - webapp = buildapp.app_factory( dict( database_file=database_connection ), - use_translogger=False, - static_enabled=False, - app=app ) + # ---- Run tool shed webserver ------------------------------------------------------ + tool_shed_server = None + toolshedwebapp = toolshedbuildapp.app_factory( dict( database_file=toolshed_database_connection ), + use_translogger=False, + static_enabled=False, + app=toolshedapp ) if tool_shed_test_port is not None: - server = httpserver.serve( webapp, host=tool_shed_test_host, port=tool_shed_test_port, start_loop=False ) + tool_shed_server = httpserver.serve( toolshedwebapp, host=tool_shed_test_host, port=tool_shed_test_port, start_loop=False ) else: random.seed() for i in range( 0, 9 ): try: tool_shed_test_port = str( random.randint( default_tool_shed_test_port_min, default_tool_shed_test_port_max ) ) log.debug( "Attempting to serve app on randomly chosen port: %s" % tool_shed_test_port ) - server = httpserver.serve( webapp, host=tool_shed_test_host, port=tool_shed_test_port, start_loop=False ) + tool_shed_server = httpserver.serve( toolshedwebapp, host=tool_shed_test_host, port=tool_shed_test_port, start_loop=False ) break except socket.error, e: if e[0] == 98: @@ -172,7 +209,7 @@ os.environ[ 'TOOL_SHED_TEST_PORT' ] = tool_shed_test_proxy_port else: os.environ[ 'TOOL_SHED_TEST_PORT' ] = tool_shed_test_port - t = threading.Thread( target=server.serve_forever ) + t = threading.Thread( target=tool_shed_server.serve_forever ) t.start() # Test if the server is up for i in range( 10 ): @@ -185,6 +222,90 @@ else: raise Exception( "Test HTTP server did not return '200 OK' after 10 tries" ) log.info( "Embedded web server started" ) + + # ---- Optionally start up a Galaxy instance ------------------------------------------------------ + if 'TEST_TOOL_SHED_START_GALAXY' in os.environ: + # Generate the shed_tool_conf.xml and tool_sheds_conf.xml files + tool_sheds_conf_template_parser = string.Template( tool_sheds_conf_xml_template ) + tool_sheds_conf_xml = tool_sheds_conf_template_parser.safe_substitute( shed_url=tool_shed_test_host, shed_port=tool_shed_test_port ) + file( galaxy_tool_sheds_conf_file, 'w' ).write( tool_sheds_conf_xml ) + shed_tool_conf_template_parser = string.Template( shed_tool_conf_xml_template ) + shed_tool_conf_xml = shed_tool_conf_template_parser.safe_substitute( shed_tool_path=galaxy_shed_tool_path ) + file( galaxy_shed_tool_conf_file, 'w' ).write( shed_tool_conf_xml ) + + # ---- Build Galaxy Application -------------------------------------------------- + galaxy_global_conf = { '__file__' : 'universe_wsgi.ini.sample' } + if not galaxy_database_connection.startswith( 'sqlite://' ): + kwargs[ 'database_engine_option_max_overflow' ] = '20' + galaxyapp = GalaxyUniverseApplication( job_queue_workers = 5, + id_secret = 'changethisinproductiontoo', + template_path = "templates", + database_connection = galaxy_database_connection, + database_engine_option_pool_size = '10', + file_path = galaxy_file_path, + tool_path = tool_path, + tool_dependency_dir=galaxy_tool_dependency_dir, + shed_tool_path=galaxy_shed_tool_path, + update_integrated_tool_panel = False, + tool_config_file = galaxy_shed_tool_conf_file, + tool_sheds_config_file = galaxy_tool_sheds_conf_file, + datatype_converters_config_file = "datatype_converters_conf.xml.sample", + tool_parse_help = False, + tool_data_table_config_path = tool_data_table_config_path, + shed_tool_data_table_config = shed_tool_data_table_config, + log_destination = "stdout", + use_heartbeat = False, + allow_user_creation = True, + allow_user_deletion = True, + admin_users = 'test@bx.psu.edu', + allow_library_path_paste = True, + global_conf = global_conf, + running_functional_tests=True, + **kwargs ) + + log.info( "Embedded Galaxy application started" ) + + # ---- Run galaxy webserver ------------------------------------------------------ + galaxy_server = None + galaxywebapp = galaxybuildapp.app_factory( dict( database_file=galaxy_database_connection ), + use_translogger=False, + static_enabled=False, + app=galaxyapp ) + + if galaxy_test_port is not None: + galaxy_server = httpserver.serve( galaxywebapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False ) + else: + random.seed() + for i in range( 0, 9 ): + try: + galaxy_test_port = str( random.randint( default_galaxy_test_port_min, default_galaxy_test_port_max ) ) + log.debug( "Attempting to serve app on randomly chosen port: %s" % galaxy_test_port ) + galaxy_server = httpserver.serve( galaxywebapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False ) + break + except socket.error, e: + if e[0] == 98: + continue + raise + else: + raise Exception( "Unable to open a port between %s and %s to start Galaxy server" % \ + ( default_galaxy_test_port_min, default_galaxy_test_port_max ) ) + if galaxy_test_proxy_port: + os.environ[ 'GALAXY_TEST_PORT' ] = galaxy_test_proxy_port + else: + os.environ[ 'GALAXY_TEST_PORT' ] = galaxy_test_port + t = threading.Thread( target=galaxy_server.serve_forever ) + t.start() + # Test if the server is up + for i in range( 10 ): + # Directly test the app, not the proxy. + conn = httplib.HTTPConnection( galaxy_test_host, galaxy_test_port ) + conn.request( "GET", "/" ) + if conn.getresponse().status == 200: + break + time.sleep( 0.1 ) + else: + raise Exception( "Test HTTP server did not return '200 OK' after 10 tries" ) + log.info( "Embedded galaxy web server started" ) # We don't add the tests to the path until everything is up and running new_path = [ os.path.join( cwd, 'test' ) ] new_path.extend( sys.path[1:] ) @@ -194,10 +315,15 @@ log.info( "Functional tests will be run against %s:%s" % ( tool_shed_test_host, tool_shed_test_proxy_port ) ) else: log.info( "Functional tests will be run against %s:%s" % ( tool_shed_test_host, tool_shed_test_port ) ) + if galaxy_test_proxy_port: + log.info( "Galaxy tests will be run against %s:%s" % ( galaxy_test_host, galaxy_test_proxy_port ) ) + else: + log.info( "Galaxy tests will be run against %s:%s" % ( galaxy_test_host, galaxy_test_port ) ) success = False try: # Pass in through script set env, will leave a copy of ALL test validate files. os.environ[ 'TOOL_SHED_TEST_HOST' ] = tool_shed_test_host + os.environ[ 'GALAXY_TEST_HOST' ] = galaxy_test_host if tool_shed_test_file_dir: os.environ[ 'TOOL_SHED_TEST_FILE_DIR' ] = tool_shed_test_file_dir test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() ) @@ -210,16 +336,26 @@ log.info( "Shutting down" ) # ---- Tear down ----------------------------------------------------------- - if server: + if tool_shed_server: log.info( "Shutting down embedded web server" ) - server.server_close() - server = None + tool_shed_server.server_close() + tool_shed_server = None log.info( "Embedded web server stopped" ) - if app: - log.info( "Shutting down app" ) - app.shutdown() - app = None - log.info( "Embedded Universe application stopped" ) + if toolshedapp: + log.info( "Shutting down tool shed app" ) + toolshedapp.shutdown() + toolshedapp = None + log.info( "Embedded tool shed application stopped" ) + if galaxy_server: + log.info( "Shutting down galaxy web server" ) + galaxy_server.server_close() + galaxy_server = None + log.info( "Embedded galaxy server stopped" ) + if galaxyapp: + log.info( "Shutting down galaxy app" ) + galaxyapp.shutdown() + galaxyapp = None + log.info( "Embedded galaxy application stopped" ) if 'TOOL_SHED_TEST_NO_CLEANUP' not in os.environ: try: for dir in [ tool_shed_test_tmp_dir ]: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.