1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/commits/0de3c7e7d632/
Changeset: 0de3c7e7d632
User: greg
Date: 2013-12-20 17:13:20
Summary: Eliminate the need for deactivating or uninstalling repositories in order to manage the tools loaded into app.toolbox within the tool shed's install and test framework.
Affected #: 5 files
diff -r 0fb6c5d7312aafece427c4e9b19dc5e5b991c6d8 -r 0de3c7e7d6327e1af295fc4d81419efd8b5f41af test/install_and_test_tool_shed_repositories/base/twilltestcase.py
--- a/test/install_and_test_tool_shed_repositories/base/twilltestcase.py
+++ b/test/install_and_test_tool_shed_repositories/base/twilltestcase.py
@@ -128,17 +128,6 @@
del( kwd[ field_name ] )
return kwd
- def uninstall_repository( self, repository ):
- """Uninstall a repository."""
- # A repository can be uninstalled only if no dependent repositories are installed.
- url = '/admin_toolshed/deactivate_or_uninstall_repository?id=%s' % self.security.encode_id( repository.id )
- self.visit_url( url )
- tc.fv ( 1, "remove_from_disk", 'true' )
- tc.submit( 'deactivate_or_uninstall_repository_button' )
- strings_displayed = [ 'The repository named' ]
- strings_displayed.append( 'has been uninstalled' )
- self.check_for_strings( strings_displayed, strings_not_displayed=[] )
-
def visit_url( self, url, allowed_codes=[ 200 ] ):
new_url = tc.go( url )
return_code = tc.browser.get_code()
diff -r 0fb6c5d7312aafece427c4e9b19dc5e5b991c6d8 -r 0de3c7e7d6327e1af295fc4d81419efd8b5f41af test/install_and_test_tool_shed_repositories/base/util.py
--- a/test/install_and_test_tool_shed_repositories/base/util.py
+++ b/test/install_and_test_tool_shed_repositories/base/util.py
@@ -518,8 +518,6 @@
repository_dict,
params,
can_update_tool_shed )
- # Uninstall this repository since it is missing dependencies.
- uninstall_repository_and_repository_dependencies( app, repository_dict )
def initialize_install_and_test_statistics_dict( test_framework ):
# Initialize a dictionary for the summary that will be printed to stdout.
@@ -591,7 +589,7 @@
log.exception( error_message )
return repository, error_message
-def is_excluded( exclude_list_dicts, name, owner, changeset_revision ):
+def is_excluded( exclude_list_dicts, name, owner, changeset_revision, encoded_repository_metadata_id ):
"""
Return True if the repository defined by the received name, owner, changeset_revision should
be excluded from testing for any reason.
@@ -606,7 +604,7 @@
return True, reason
# Skip this repository if it has a repository dependency that is in the exclude list.
repository_dependency_dicts, error_message = \
- get_repository_dependencies_for_changeset_revision( install_and_test_base_util.galaxy_tool_shed_url,
+ get_repository_dependencies_for_changeset_revision( galaxy_tool_shed_url,
encoded_repository_metadata_id )
if error_message:
log.debug( 'Error getting repository dependencies for revision %s of repository %s owned by %s:' % \
@@ -800,96 +798,3 @@
test_runner = plug_runner
result = test_runner.run( tests )
return result, test_config.plugins._plugins
-
-def uninstall_repository_and_repository_dependencies( app, repository_dict ):
- """Uninstall a repository and all of its repository dependencies."""
- # This method assumes that the repositor defined by the received repository_dict is not a repository
- # dependency of another repository.
- sa_session = app.install_model.context
- # The dict contains the only repository the app should have installed at this point.
- name = str( repository_dict[ 'name' ] )
- owner = str( repository_dict[ 'owner' ] )
- changeset_revision = str( repository_dict[ 'changeset_revision' ] )
- # Since this install and test framework uninstalls repositories immediately after installing and testing
- # them, the values of repository.installed_changeset_revision and repository.changeset_revision should be
- # the same.
- repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision )
- if repository.can_uninstall( app ):
- # A repository can be uninstalled only if no dependent repositories are installed. So uninstallation order
- # id critical. A repository is always uninstalled first, and the each of its dependencies is checked to see
- # if it can be uninstalled.
- uninstall_repository_dict = dict( name=name,
- owner=owner,
- changeset_revision=changeset_revision )
- log.debug( 'Revision %s of repository %s owned by %s selected for uninstallation.' % ( changeset_revision, name, owner ) )
- try:
- test_install_repositories.generate_uninstall_method( uninstall_repository_dict )
- # Set up nose to run the generated uninstall method as a functional test.
- test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() )
- test_config.configure( sys.argv )
- # Run the uninstall method. This method uses the Galaxy web interface to uninstall the previously installed
- # repository and all of its repository dependencies, deleting each of them from disk.
- result, _ = run_tests( test_config )
- repository_uninstall_successful = result.wasSuccessful()
- except Exception, e:
- repository_uninstall_successful = False
- log.exception( 'Uninstallation of revision %s of repository %s owned by %s failed: %s.' % \
- ( rd_changeset_revision, rd_name, rd_owner, str( e ) ) )
- if repository_uninstall_successful:
- # Now that the repository is uninstalled we can attempt to uninstall each of its repository dependencies.
- # We have to do this through Twill in order to maintain app.toolbox and shed_tool_conf.xml in a state that
- # is valid for future tests. Since some of the repository's repository dependencies may require other of
- # the repository's repository dependencies, we'll keep track of the repositories we've been able to unistall.
- processed_repository_dependency_ids = []
- while len( processed_repository_dependency_ids ) < len( repository.repository_dependencies ):
- for repository_dependency in repository.repository_dependencies:
- if repository_dependency.id not in processed_repository_dependency_ids and repository_dependency.can_uninstall( app ):
- processed_repository_dependency_ids.append( repository_dependency.id )
- rd_name = str( repository_dependency.name )
- rd_owner = str( repository_dependency.owner )
- rd_changeset_revision = str( repository_dependency.changeset_revision )
- uninstall_repository_dict = dict( name=rd_name,
- owner=rd_owner,
- changeset_revision=rd_changeset_revision )
- log.debug( 'Revision %s of repository dependency %s owned by %s selected for uninstallation.' % \
- ( rd_changeset_revision, rd_name, rd_owner ) )
- # Generate a test method to uninstall the repository dependency through the embedded Galaxy application's
- # web interface.
- try:
- test_install_repositories.generate_uninstall_method( uninstall_repository_dict )
- # Set up nose to run the generated uninstall method as a functional test.
- test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() )
- test_config.configure( sys.argv )
- # Run the uninstall method.
- result, _ = run_tests( test_config )
- if not result.wasSuccessful():
- # We won't set ok here because we'll continue to uninstall whatever we can.
- log.debug( 'Uninstallation of revision %s of repository %s owned by %s failed.' % \
- ( rd_changeset_revision, rd_name, rd_owner ) )
- except Exception, e:
- log.exception( 'Uninstallation of revision %s of repository %s owned by %s failed: %s.' % \
- ( rd_changeset_revision, rd_name, rd_owner, str( e ) ) )
- else:
- log.debug( 'Uninstallation of revision %s of repository %s owned by %s failed.' % ( changeset_revision, name, owner ) )
- else:
- log_reason_repository_cannot_be_uninstalled( app, repository )
-
-def uninstall_tool_dependency( app, tool_dependency ):
- """Attempt to uninstall a tool dependency."""
- sa_session = app.install_model.context
- # Clean out any generated tests. This is necessary for Twill.
- tool_dependency_install_path = tool_dependency.installation_directory( app )
- uninstalled, error_message = tool_dependency_util.remove_tool_dependency( app, tool_dependency )
- if error_message:
- log.debug( 'Error attempting to remove directory: %s' % str( tool_dependency_install_path ) )
- log.debug( error_message )
- else:
- log.debug( 'Successfully removed tool dependency installation directory: %s' % str( tool_dependency_install_path ) )
- if not uninstalled or tool_dependency.status != app.model.ToolDependency.installation_status.UNINSTALLED:
- tool_dependency.status = app.model.ToolDependency.installation_status.UNINSTALLED
- sa_session.add( tool_dependency )
- sa_session.flush()
- if os.path.exists( tool_dependency_install_path ):
- log.debug( 'Uninstallation of tool dependency succeeded, but the installation path still exists on the filesystem. It is now being explicitly deleted.')
- suc.remove_dir( tool_dependency_install_path )
-
diff -r 0fb6c5d7312aafece427c4e9b19dc5e5b991c6d8 -r 0de3c7e7d6327e1af295fc4d81419efd8b5f41af test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py
--- a/test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py
+++ b/test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py
@@ -9,21 +9,6 @@
class InstallTestRepositories( InstallTestRepository ):
"""Abstract test case that installs and uninstalls a predefined list of repositories."""
- def do_deactivate( self, repository_dict ):
- self.logout()
- self.login( email='test(a)bx.psu.edu', username='test' )
- admin_user = test_db_util.get_user( 'test(a)bx.psu.edu' )
- assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email
- # Get the repository defined by the received repository_dict along with all of its repository dependencies
- # from the database.
- name = repository_dict[ 'name' ]
- owner = repository_dict[ 'owner' ]
- changeset_revision = repository_dict[ 'changeset_revision' ]
- repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision )
- admin_user_private_role = test_db_util.get_private_role( admin_user )
- # Uninstall the repository through the web interface using twill.
- self.deactivate_repository( repository )
-
def do_install( self, repository_dict ):
self.logout()
self.login( email='test(a)bx.psu.edu', username='test' )
@@ -34,50 +19,6 @@
# actually install more than this singe repository because repository dependencies can be installed.
self.install_repository( repository_dict )
- def do_uninstall( self, repository_dict ):
- self.logout()
- self.login( email='test(a)bx.psu.edu', username='test' )
- admin_user = test_db_util.get_user( 'test(a)bx.psu.edu' )
- assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email
- # Get the repository defined by the received repository_dict along with all of its repository dependencies
- # from the database.
- name = repository_dict[ 'name' ]
- owner = repository_dict[ 'owner' ]
- changeset_revision = repository_dict[ 'changeset_revision' ]
- repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision )
- admin_user_private_role = test_db_util.get_private_role( admin_user )
- # Uninstall the repository and all of its repository dependencies through the Galaxy web interface using twill.
- self.uninstall_repository( repository )
-
-def generate_deactivate_method( repository_dict=None ):
- """Generate abstract test cases for the received repository_dict."""
- if repository_dict is None:
- return
- # Push all the toolbox tests to module level
- G = globals()
- # Eliminate all previous tests from G.
- for key, val in G.items():
- if key.startswith( 'TestInstallRepository_' ) or key.startswith( 'TestForTool_' ):
- del G[ key ]
- # Create a new subclass with a method named install_repository_XXX that deactivates the
- # repository specified by the received repository_dict.
- name = "TestDeactivateRepository_%s_%s" % \
- ( str( repository_dict[ 'name' ] ), str( repository_dict[ 'changeset_revision' ] ) )
- baseclasses = ( InstallTestRepositories, )
- namespace = dict()
- def make_deactivate_method( repository_dict ):
- def test_deactivate_repository( self ):
- self.do_deactivate( repository_dict )
- return test_deactivate_repository
- test_method = make_deactivate_method( repository_dict )
- test_method.__doc__ = "Deactivate the repository %s." % str( repository_dict[ 'name' ] )
- namespace[ 'uninstall_repository_%s_%s' % ( str( repository_dict[ 'name' ] ),
- str( repository_dict[ 'changeset_revision' ] ) ) ] = test_method
- # The new.classobj function returns a new class object, with name name, derived
- # from baseclasses (which should be a tuple of classes) and with namespace dict.
- new_class_obj = new.classobj( str( name ), baseclasses, namespace )
- G[ name ] = new_class_obj
-
def generate_install_method( repository_dict=None ):
"""Generate abstract test cases for the defined list of repositories."""
if repository_dict is None:
@@ -104,32 +45,3 @@
# from baseclasses (which should be a tuple of classes) and with namespace dict.
new_class_obj = new.classobj( str( name ), baseclasses, namespace )
G[ name ] = new_class_obj
-
-def generate_uninstall_method( repository_dict=None ):
- """Generate abstract test cases for the received repository_dict."""
- if repository_dict is None:
- return
- # Push all the toolbox tests to module level
- G = globals()
- # Eliminate all previous tests from G.
- for key, val in G.items():
- if key.startswith( 'TestInstallRepository_' ) or key.startswith( 'TestForTool_' ):
- del G[ key ]
- # Create a new subclass with a method named install_repository_XXX that installs the repository
- # specified by the received repository_dict.
- name = "TestUninstallRepository_%s_%s" % \
- ( str( repository_dict[ 'name' ] ), str( repository_dict[ 'changeset_revision' ] ) )
- baseclasses = ( InstallTestRepositories, )
- namespace = dict()
- def make_uninstall_method( repository_dict ):
- def test_uninstall_repository( self ):
- self.do_uninstall( repository_dict )
- return test_uninstall_repository
- test_method = make_uninstall_method( repository_dict )
- test_method.__doc__ = "Uninstall the repository %s." % repository_dict[ 'name' ]
- namespace[ 'uninstall_repository_%s_%s' % ( str( repository_dict[ 'name' ] ),
- str( repository_dict[ 'changeset_revision' ] ) ) ] = test_method
- # The new.classobj function returns a new class object, with name name, derived
- # from baseclasses (which should be a tuple of classes) and with namespace dict.
- new_class_obj = new.classobj( str( name ), baseclasses, namespace )
- G[ name ] = new_class_obj
diff -r 0fb6c5d7312aafece427c4e9b19dc5e5b991c6d8 -r 0de3c7e7d6327e1af295fc4d81419efd8b5f41af test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py
--- a/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py
+++ b/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py
@@ -76,7 +76,7 @@
can_update_tool_shed = True
test_framework = install_and_test_base_util.REPOSITORIES_WITH_TOOLS
-
+
def get_failed_test_dicts( test_result, from_tool_test=True ):
"""Extract any useful data from the test_result.failures and test_result.errors attributes."""
failed_test_dicts = []
@@ -177,7 +177,11 @@
log.debug( error_message )
else:
tool_test_results_dict = install_and_test_base_util.get_tool_test_results_dict( tool_test_results_dicts )
- is_excluded, reason = install_and_test_base_util.is_excluded( exclude_list_dicts, name, owner, changeset_revision )
+ is_excluded, reason = install_and_test_base_util.is_excluded( exclude_list_dicts,
+ name,
+ owner,
+ changeset_revision,
+ encoded_repository_metadata_id )
if is_excluded:
# If this repository is being skipped, register the reason.
tool_test_results_dict[ 'not_tested' ] = dict( reason=reason )
@@ -193,28 +197,15 @@
( changeset_revision, name, owner ) )
else:
tool_test_results_dict = install_and_test_base_util.initialize_tool_tests_results_dict( app, tool_test_results_dict )
- # Explicitly clear tests from twill's test environment.
- remove_generated_tests( app )
# Proceed with installing repositories and testing contained tools.
repository, error_message = install_and_test_base_util.install_repository( app, repository_dict )
install_and_test_statistics_dict[ 'total_repositories_processed' ] += 1
if error_message:
# The repository installation failed.
- log.debug( 'Installation failed for revision %s of repository %s owned by %s.' % \
- ( changeset_revision, name, owner ) )
+ log.debug( 'Installation failed for revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) )
install_and_test_statistics_dict[ 'repositories_with_installation_error' ].append( repository_identifier_dict )
tool_test_results_dict[ 'installation_errors' ][ 'current_repository' ] = error_message
- # Even if the repository failed to install, execute the uninstall method, in case a dependency did succeed.
- log.debug( 'Attempting to uninstall revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) )
- try:
- repository = \
- test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision )
- except Exception, e:
- error_message = 'Unable to find revision %s of repository %s owned by %s: %s.' % \
- ( changeset_revision, name, owner, str( e ) )
- log.exception( error_message )
- params = dict( test_install_error=True,
- do_not_test=False )
+ params = dict( test_install_error=True )
# TODO: do something useful with response_dict
response_dict = install_and_test_base_util.register_test_result( install_and_test_base_util.galaxy_tool_shed_url,
tool_test_results_dicts,
@@ -222,19 +213,10 @@
repository_dict,
params,
can_update_tool_shed )
- try:
- # We are uninstalling this repository and all of its repository dependencies.
- install_and_test_base_util.uninstall_repository_and_repository_dependencies( app, repository_dict )
- except Exception, e:
- log.exception( 'Error attempting to uninstall revision %s of repository %s owned by %s: %s' % \
- ( changeset_revision, name, owner, str( e ) ) )
- # Clean out any generated tests. This is necessary for Twill.
- remove_generated_tests( app )
- install_and_test_statistics_dict[ 'repositories_with_installation_error' ].append( repository_identifier_dict )
+ else:
+ # The repository was successfully installed.
log.debug( 'Installation succeeded for revision %s of repository %s owned by %s.' % \
( changeset_revision, name, owner ) )
- else:
- # The repository was successfully installed.
params, install_and_test_statistics_dict, tool_test_results_dict = \
install_and_test_base_util.register_installed_and_missing_dependencies( app,
repository,
@@ -257,26 +239,23 @@
tool_test_results_dict=tool_test_results_dict,
params=params,
can_update_tool_shed=can_update_tool_shed )
- # Set the test_toolbox.toolbox module-level variable to the new app.toolbox.
- test_toolbox.toolbox = app.toolbox
else:
- # This repository and all of its dependencies were successfully installed.
- # Configure and run functional tests for this repository. This is equivalent to
+ log.debug( 'Installation of %s succeeded, running all defined functional tests.' % str( repository.name ) )
+ # Generate the shed_tools_dict that specifies the location of test data contained within this repository.
+ # and configure and run functional tests for this repository. This is equivalent to
# sh run_functional_tests.sh -installed
- remove_install_tests()
- log.debug( 'Installation of %s succeeded, running all defined functional tests.' % str( repository.name ) )
- # Generate the shed_tools_dict that specifies the location of test data contained within this repository. If the repository
- # does not have a test-data directory, this will return has_test_data = False, and we will set the do_not_test flag to True,
- # and the tools_functionally_correct flag to False, as well as updating tool_test_results.
file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( {} ) )
+ # Find the path to the test-data directory within the installed repository.
has_test_data, shed_tools_dict = \
parse_tool_panel_config( galaxy_shed_tool_conf_file,
from_json_string( file( galaxy_shed_tools_dict, 'r' ).read() ) )
- # If the repository has a test-data directory we write the generated shed_tools_dict to a file, so the functional
- # test framework can find it.
+ # If the repository has a test-data directory we write the generated shed_tools_dict to a temporary
+ # file so the functional test framework can find it.
file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( shed_tools_dict ) )
log.debug( 'Saved generated shed_tools_dict to %s\nContents: %s' % ( str( galaxy_shed_tools_dict ),
str( shed_tools_dict ) ) )
+ # Remove twills' old generated test before building the new tests for the current tools.
+ remove_tests( app )
try:
install_and_test_statistics_dict = test_repository_tools( app,
repository,
@@ -441,6 +420,8 @@
config_items.append( config_tuple )
# Write a temporary file, based on universe_wsgi.ini.sample, using the configuration options defined above.
generate_config_file( 'universe_wsgi.ini.sample', galaxy_config_file, config_items )
+ # kwargs must be a list when passed to the Galaxy webapp (i.e., UniverseApplication).
+ # The following is used to set metadata externally.
kwargs[ 'tool_config_file' ] = [ galaxy_tool_conf_file, galaxy_shed_tool_conf_file ]
# Set the global_conf[ '__file__' ] option to the location of the temporary .ini file, which gets passed to set_metadata.sh.
kwargs[ 'global_conf' ] = install_and_test_base_util.get_webapp_global_conf()
@@ -581,46 +562,45 @@
# Return a "successful" response to buildbot.
return 0
-def remove_generated_tests( app ):
+def remove_tests( app ):
"""
Delete any configured tool functional tests from the test_toolbox.__dict__, otherwise nose will find them
and try to re-run the tests after uninstalling the repository, which will cause false failure reports,
since the test data has been deleted from disk by now.
"""
tests_to_delete = []
- tools_to_delete = []
+ tools_to_delete_by_id = []
global test_toolbox
+ # Find all tests previously generated by twill.
for key in test_toolbox.__dict__:
if key.startswith( 'TestForTool_' ):
- log.debug( 'Tool test found in test_toolbox, deleting: %s' % str( key ) )
+ log.debug( 'Tool test %s discovered in test_toolbox.' % str( key ) )
# We can't delete this test just yet, we're still iterating over __dict__.
tests_to_delete.append( key )
tool_id = key.replace( 'TestForTool_', '' )
- for tool in app.toolbox.tools_by_id:
- if tool.replace( '_', ' ' ) == tool_id.replace( '_', ' ' ):
- tools_to_delete.append( tool )
+ for app_tool_id in app.toolbox.tools_by_id:
+ if app_tool_id.replace( '_', ' ' ) == tool_id.replace( '_', ' ' ):
+ tools_to_delete_by_id.append( tool_id )
+ # Delete the discovered twill-generated tests.
for key in tests_to_delete:
- # Now delete the tests found in the previous loop.
- del test_toolbox.__dict__[ key ]
- for tool in tools_to_delete:
- del app.toolbox.tools_by_id[ tool ]
-
-def remove_install_tests():
- """
- Delete any configured repository installation tests from the test_toolbox.__dict__, otherwise nose will find them
- and try to install the repository again while running tool functional tests.
- """
- tests_to_delete = []
- global test_toolbox
+ if key in test_toolbox.__dict__:
+ log.debug( 'Deleting test %s from test_toolbox.' % str( key ) )
+ del test_toolbox.__dict__[ key ]
+ for tool_id in tools_to_delete_by_id:
+ if tool_id in app.toolbox.tools_by_id:
+ log.debug( 'Deleting tool id %s from app.toolbox[ tools_by_id ].' % str( tool_id ) )
+ del app.toolbox.tools_by_id[ tool_id ]
# Push all the toolbox tests to module level
for key in test_install_repositories.__dict__:
if key.startswith( 'TestInstallRepository_' ):
- log.debug( 'Repository installation process found, deleting: %s' % str( key ) )
+ log.debug( 'Repository installation process test %s discovered.' % str( key ) )
# We can't delete this test just yet, we're still iterating over __dict__.
tests_to_delete.append( key )
+ # Delete the discovered twill-generated tests.
for key in tests_to_delete:
- # Now delete the tests found in the previous loop.
- del test_install_repositories.__dict__[ key ]
+ if key in test_install_repositories.__dict__:
+ log.debug( 'Deleting test %s from test_toolbox.' % str( key ) )
+ del test_install_repositories.__dict__[ key ]
def test_repository_tools( app, repository, repository_dict, tool_test_results_dicts, tool_test_results_dict,
install_and_test_statistics_dict ):
@@ -629,7 +609,9 @@
owner = str( repository.owner )
changeset_revision = str( repository.changeset_revision )
repository_identifier_dict = dict( name=name, owner=owner, changeset_revision=changeset_revision )
- # Set the module-level variable 'toolbox', so that test.functional.test_toolbox will generate the appropriate test methods.
+ # Set the module-level variable 'toolbox', so that test.functional.test_toolbox will generate the
+ # appropriate test methods. At this point, app.toolbox contains the upload tool and all tools contained
+ # in the repository.
test_toolbox.toolbox = app.toolbox
# Generate the test methods for this installed repository. We need to pass in True here, or it will look
# in $GALAXY_HOME/test-data for test data, which may result in missing or invalid test files.
@@ -692,15 +674,6 @@
can_update_tool_shed )
log.debug( 'Revision %s of repository %s owned by %s installed successfully but did not pass functional tests.' % \
( changeset_revision, name, owner ) )
- # Run the uninstall method. This removes tool functional test methods from the test_toolbox module and uninstalls the
- # repository using Twill.
- log.debug( 'Uninstalling changeset revision %s of repository %s' % ( str( changeset_revision ), str( name ) ) )
- # We are uninstalling this repository and all of its repository dependencies.
- install_and_test_base_util.uninstall_repository_and_repository_dependencies( app, repository_dict )
- # Clean out any generated tests. This is necessary for Twill.
- remove_generated_tests( app )
- # Set the test_toolbox.toolbox module-level variable to the new app.toolbox.
- test_toolbox.toolbox = app.toolbox
return install_and_test_statistics_dict
if __name__ == "__main__":
diff -r 0fb6c5d7312aafece427c4e9b19dc5e5b991c6d8 -r 0de3c7e7d6327e1af295fc4d81419efd8b5f41af test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py
--- a/test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py
+++ b/test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py
@@ -115,7 +115,11 @@
log.debug( error_message )
else:
tool_test_results_dict = install_and_test_base_util.get_tool_test_results_dict( tool_test_results_dicts )
- is_excluded, reason = install_and_test_base_util.is_excluded( exclude_list_dicts, name, owner, changeset_revision )
+ is_excluded, reason = install_and_test_base_util.is_excluded( exclude_list_dicts,
+ name,
+ owner,
+ changeset_revision,
+ encoded_repository_metadata_id )
if this_repository_is_in_the_exclude_list or requires_excluded:
# If this repository is being skipped, register the reason.
tool_test_results_dict[ 'not_tested' ] = dict( reason=reason )
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/commits/bedb1e48e5c7/
Changeset: bedb1e48e5c7
User: inithello
Date: 2013-12-20 15:55:14
Summary: Fix for install_and_test_statistics_dict being set to None when a repository's tool dependencies were determined to be unsuccessfully installed.
Affected #: 1 file
diff -r c77a66edf25294e1743cc0a3886f8062db5c1fff -r bedb1e48e5c724d817f1a695ed068e0745f066cd test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py
--- a/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py
+++ b/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py
@@ -300,15 +300,14 @@
tool_test_results_dict[ 'missing_test_components' ] = []
missing_tool_dependencies = install_and_test_base_util.get_missing_tool_dependencies( repository )
if missing_tool_dependencies or repository.missing_repository_dependencies:
- install_and_test_statistics_dict = \
- install_and_test_base_util.handle_missing_dependencies( app=app,
- repository=repository,
- missing_tool_dependencies=missing_tool_dependencies,
- repository_dict=repository_dict,
- tool_test_results_dicts=tool_test_results_dicts,
- tool_test_results_dict=tool_test_results_dict,
- params=params,
- can_update_tool_shed=can_update_tool_shed )
+ install_and_test_base_util.handle_missing_dependencies( app=app,
+ repository=repository,
+ missing_tool_dependencies=missing_tool_dependencies,
+ repository_dict=repository_dict,
+ tool_test_results_dicts=tool_test_results_dicts,
+ tool_test_results_dict=tool_test_results_dict,
+ params=params,
+ can_update_tool_shed=can_update_tool_shed )
# Set the test_toolbox.toolbox module-level variable to the new app.toolbox.
test_toolbox.toolbox = app.toolbox
else:
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/commits/c77a66edf252/
Changeset: c77a66edf252
User: greg
Date: 2013-12-20 15:50:03
Summary: Make the tool shed's install and test exclude list locally accessible rather than stored in cfengine.
Affected #: 4 files
diff -r 3e2b61de5a2322563c1ffa49e03c118144f5068c -r c77a66edf25294e1743cc0a3886f8062db5c1fff lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py
--- a/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py
+++ b/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py
@@ -197,7 +197,16 @@
test_environment_dict[ 'tool_shed_mercurial_version' ] = __version__.version
test_environment_dict[ 'tool_shed_revision' ] = get_repository_current_revision( os.getcwd() )
tool_test_results_dict[ 'test_environment' ] = test_environment_dict
- repository_metadata.tool_test_results = tool_test_results_dict
+ # Store only the configured number of test runs.
+ num_tool_test_results_saved = int( app.config.num_tool_test_results_saved )
+ if len( tool_test_results_dicts ) >= num_tool_test_results_saved:
+ test_results_index = num_tool_test_results_saved - 1
+ new_tool_test_results_dicts = tool_test_results_dicts[ :test_results_index ]
+ else:
+ new_tool_test_results_dicts = [ d for d in tool_test_results_dicts ]
+ # Insert the new element into the first position in the list.
+ new_tool_test_results_dicts.insert( 0, tool_test_results_dict )
+ repository_metadata.tool_test_results = new_tool_test_results_dicts
app.sa_session.add( repository_metadata )
app.sa_session.flush()
stop = time.time()
diff -r 3e2b61de5a2322563c1ffa49e03c118144f5068c -r c77a66edf25294e1743cc0a3886f8062db5c1fff test/install_and_test_tool_shed_repositories/base/twilltestcase.py
--- a/test/install_and_test_tool_shed_repositories/base/twilltestcase.py
+++ b/test/install_and_test_tool_shed_repositories/base/twilltestcase.py
@@ -41,16 +41,6 @@
self.shed_tools_dict = {}
self.home()
- def deactivate_repository( self, repository ):
- """Deactivate a repository."""
- url = '/admin_toolshed/deactivate_or_uninstall_repository?id=%s' % self.security.encode_id( repository.id )
- self.visit_url( url )
- tc.fv ( 1, "remove_from_disk", 'false' )
- tc.submit( 'deactivate_or_uninstall_repository_button' )
- strings_displayed = [ 'The repository named' ]
- strings_displayed.append( 'has been deactivated' )
- self.check_for_strings( strings_displayed, strings_not_displayed=[] )
-
def initiate_installation_process( self,
install_tool_dependencies=False,
install_repository_dependencies=True,
diff -r 3e2b61de5a2322563c1ffa49e03c118144f5068c -r c77a66edf25294e1743cc0a3886f8062db5c1fff test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py
--- a/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py
+++ b/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py
@@ -63,12 +63,11 @@
# Here's the directory where everything happens. Temporary directories are created within this directory to contain
# the database, new repositories, etc.
galaxy_test_tmp_dir = os.path.join( test_home_directory, 'tmp' )
+# File containing information about problematic repositories to exclude from test runs.
+exclude_list_file = os.path.join( test_home_directory, 'exclude.xml' )
default_galaxy_locales = 'en'
default_galaxy_test_file_dir = "test-data"
os.environ[ 'GALAXY_INSTALL_TEST_TMP_DIR' ] = galaxy_test_tmp_dir
-# This file is copied to the Galaxy root directory by buildbot.
-# It is managed by cfengine and is not locally available.
-exclude_list_file = os.environ.get( 'GALAXY_INSTALL_TEST_EXCLUDE_REPOSITORIES', 'repositories_with_tools_exclude.xml' )
# This script can be run in such a way that no Tool Shed database records should be changed.
if '-info_only' in sys.argv or 'GALAXY_INSTALL_TEST_INFO_ONLY' in os.environ:
@@ -138,25 +137,20 @@
return tool_id, tool_version
def install_and_test_repositories( app, galaxy_shed_tools_dict, galaxy_shed_tool_conf_file ):
+ # Initialize a dictionary for the summary that will be printed to stdout.
install_and_test_statistics_dict = install_and_test_base_util.initialize_install_and_test_statistics_dict( test_framework )
error_message = ''
- # Initialize a dictionary for the summary that will be printed to stdout.
- total_repositories_processed = install_and_test_statistics_dict[ 'total_repositories_processed' ]
repositories_to_install, error_message = \
install_and_test_base_util.get_repositories_to_install( install_and_test_base_util.galaxy_tool_shed_url, test_framework )
if error_message:
return None, error_message
# Handle repositories not to be tested.
if os.path.exists( exclude_list_file ):
- # Entries in the exclude_list look something like this.
- # { 'reason': The default reason or the reason specified in this section,
- # 'repositories':
- # [( name, owner, changeset revision if changeset revision else None ),
- # ( name, owner, changeset revision if changeset revision else None )] }
- # If changeset revision is None, that means the entire repository is excluded from testing, otherwise only the specified
- # revision should be skipped.
- log.debug( 'Loading the list of repositories excluded from testing from the file %s...' % \
- str( exclude_list_file ) )
+ log.debug( 'Loading the list of repositories excluded from testing from the file %s...' % str( exclude_list_file ) )
+ # The following exclude_list will look something like this:
+ # [{ 'reason': The default reason or the reason specified in this section,
+ # 'repositories': [( name, owner, changeset_revision if changeset_revision else None ),
+ # ( name, owner, changeset_revision if changeset_revision else None )]}]
exclude_list = install_and_test_base_util.parse_exclude_list( exclude_list_file )
else:
exclude_list = []
diff -r 3e2b61de5a2322563c1ffa49e03c118144f5068c -r c77a66edf25294e1743cc0a3886f8062db5c1fff test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py
--- a/test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py
+++ b/test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py
@@ -61,12 +61,11 @@
# Here's the directory where everything happens. Temporary directories are created within this directory to contain
# the database, new repositories, etc.
galaxy_test_tmp_dir = os.path.join( test_home_directory, 'tmp' )
+# File containing information about problematic repositories to exclude from test runs.
+exclude_list_file = os.path.join( test_home_directory, 'exclude.xml' )
default_galaxy_locales = 'en'
default_galaxy_test_file_dir = "test-data"
os.environ[ 'GALAXY_INSTALL_TEST_TMP_DIR' ] = galaxy_test_tmp_dir
-# This file is copied to the Galaxy root directory by buildbot.
-# It is managed by cfengine and is not locally available.
-exclude_list_file = os.environ.get( 'GALAXY_INSTALL_TEST_EXCLUDE_REPOSITORIES', 'tool_dependency_definition_exclude.xml' )
# This script can be run in such a way that no Tool Shed database records should be changed.
if '-info_only' in sys.argv or 'GALAXY_INSTALL_TEST_INFO_ONLY' in os.environ:
@@ -77,29 +76,15 @@
test_framework = install_and_test_base_util.TOOL_DEPENDENCY_DEFINITIONS
def install_and_test_repositories( app, galaxy_shed_tools_dict, galaxy_shed_tool_conf_file ):
+ # Initialize a dictionary for the summary that will be printed to stdout.
install_and_test_statistics_dict = install_and_test_base_util.initialize_install_and_test_statistics_dict( test_framework )
error_message = ''
- # Initialize a dictionary for the summary that will be printed to stdout.
- total_repositories_processed = install_and_test_statistics_dict[ 'total_repositories_processed' ]
repositories_to_install, error_message = \
install_and_test_base_util.get_repositories_to_install( install_and_test_base_util.galaxy_tool_shed_url, test_framework )
if error_message:
return None, error_message
- # Handle repositories not to be tested.
if os.path.exists( exclude_list_file ):
- # Entries in the exclude_list look something like this.
- # { 'reason': The default reason or the reason specified in this section,
- # 'repositories':
- # [( name, owner, changeset revision if changeset revision else None ),
- # ( name, owner, changeset revision if changeset revision else None )] }
- # If changeset revision is None, that means the entire repository is excluded from testing, otherwise only the specified
- # revision should be skipped.
- # We are testing deprecated repositories because it is possible that a deprecated repository contains valid tools that
- # someone has previously installed. Deleted repositories have never been installed, so should not be tested. If they are
- # undeleted, this script will then test them the next time it runs. We don't need to check if a repository has been deleted
- # here because our call to the Tool Shed API filters by downloadable='true', in which case deleted will always be False.
- log.debug( 'Loading the list of repositories excluded from testing from the file %s...' % \
- str( exclude_list_file ) )
+ log.debug( 'Loading the list of repositories excluded from testing from the file %s...' % str( exclude_list_file ) )
# The following exclude_list will look something like this:
# [{ 'reason': The default reason or the reason specified in this section,
# 'repositories': [( name, owner, changeset_revision if changeset_revision else None ),
@@ -201,19 +186,9 @@
install_and_test_statistics_dict[ 'total_repositories_processed' ] += 1
if error_message:
# The repository installation failed.
- log.debug( 'Installation failed for revision %s of repository %s owned by %s.' % \
- ( changeset_revision, name, owner ) )
+ log.debug( 'Installation failed for revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) )
install_and_test_statistics_dict[ 'repositories_with_installation_error' ].append( repository_identifier_dict )
tool_test_results_dict[ 'installation_errors' ][ 'current_repository' ] = error_message
- # Even if the repository failed to install, execute the uninstall method, in case a dependency did succeed.
- log.debug( 'Attempting to uninstall revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) )
- try:
- repository = \
- test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision )
- except Exception, e:
- error_message = 'Unable to find revision %s of repository %s owned by %s: %s.' % \
- ( changeset_revision, name, owner, str( e ) )
- log.exception( error_message )
params = dict( test_install_error=True,
do_not_test=False )
# TODO: do something useful with response_dict
@@ -223,14 +198,6 @@
repository_dict,
params,
can_update_tool_shed )
- try:
- # We are uninstalling this repository and all of its repository dependencies.
- install_and_test_base_util.uninstall_repository_and_repository_dependencies( app, repository_dict )
- except Exception, e:
- log.exception( 'Error attempting to uninstall revision %s of repository %s owned by %s: %s' % \
- ( changeset_revision, name, owner, str( e ) ) )
- log.debug( 'Installation failed for revision %s of repository %s owned by %s.' % \
- ( changeset_revision, name, owner ) )
else:
# The repository was successfully installed.
params, install_and_test_statistics_dict, tool_test_results_dict = \
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/commits/3e2b61de5a23/
Changeset: 3e2b61de5a23
User: greg
Date: 2013-12-20 03:20:57
Summary: Fix for testing repositories with tools using the tool shed's install and test framework.
Affected #: 1 file
diff -r 9e9746198fd8c49380d42a2b4657ef241dda8f6a -r 3e2b61de5a2322563c1ffa49e03c118144f5068c test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py
--- a/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py
+++ b/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py
@@ -155,10 +155,6 @@
# ( name, owner, changeset revision if changeset revision else None )] }
# If changeset revision is None, that means the entire repository is excluded from testing, otherwise only the specified
# revision should be skipped.
- # We are testing deprecated repositories because it is possible that a deprecated repository contains valid tools that
- # someone has previously installed. Deleted repositories have never been installed, so should not be tested. If they are
- # undeleted, this script will then test them the next time it runs. We don't need to check if a repository has been deleted
- # here because our call to the Tool Shed API filters by downloadable='true', in which case deleted will always be False.
log.debug( 'Loading the list of repositories excluded from testing from the file %s...' % \
str( exclude_list_file ) )
exclude_list = install_and_test_base_util.parse_exclude_list( exclude_list_file )
@@ -739,7 +735,9 @@
tool_test_results_dict[ 'failed_tests' ] = failed_test_dicts
failed_repository_dict = repository_identifier_dict
install_and_test_statistics_dict[ 'at_least_one_test_failed' ].append( failed_repository_dict )
- set_do_not_test = not is_latest_downloadable_revision( install_and_test_base_util.galaxy_tool_shed_url, repository_dict )
+ set_do_not_test = \
+ not install_and_test_base_util.is_latest_downloadable_revision( install_and_test_base_util.galaxy_tool_shed_url,
+ repository_dict )
params = dict( tools_functionally_correct=False,
test_install_error=False,
do_not_test=str( set_do_not_test ) )
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/commits/9e9746198fd8/
Changeset: 9e9746198fd8
User: greg
Date: 2013-12-19 23:30:29
Summary: Re-add changeset_revision to the information returned from the tool shed's repository_dependencies api method.
Affected #: 1 file
diff -r 7701d6b5e1b62f501e827a3cc6ee880e8955f036 -r 9e9746198fd8c49380d42a2b4657ef241dda8f6a lib/galaxy/webapps/tool_shed/api/repository_revisions.py
--- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py
+++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py
@@ -91,6 +91,8 @@
tool_shed, name, owner, changeset_revision = rd_tup[ 0:4 ]
repository_dependency = suc.get_repository_by_name_and_owner( trans.app, name, owner )
repository_dependency_dict = repository_dependency.to_dict( view='element', value_mapper=value_mapper )
+ # We have to add the changeset_revision of of the repository dependency.
+ repository_dependency_dict[ 'changeset_revision' ] = changeset_revision
repository_dependency_dict[ 'url' ] = web.url_for( controller='repositories',
action='show',
id=trans.security.encode_id( repository_dependency.id ) )
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.