[galaxy-commits] commit/galaxy-central: 3 new changesets

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/079ce0488cdc/ Changeset: 079ce0488cdc User: Dave Bouvier Date: 2013-09-24 15:10:13 Summary: Fix for backwards compatibility with a Galaxy instance running the latest revision installing repositories from a tool shed running the stable branch. Affected #: 5 files diff -r fd7f14161d3537abf1f676f14ec763d150ee611f -r 079ce0488cdcf4ee882ba3cd4e5413dcab3f9412 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -100,19 +100,14 @@ continue # rd_key is something like: 'http://localhost:9009__ESEP__package_rdkit_2012_12__ESEP__test__ESEP__d635ff...' # rd_val is something like: [['http://localhost:9009', 'package_numpy_1_7', 'test', 'cddd64ecd985', 'True']] - try: - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - container_util.get_components_from_key( rd_key ) - except: - tool_shed, name, owner, changeset_revision = container_util.get_components_from_key( rd_val ) + repository_components_tuple = container_util.get_components_from_key( rd_key ) + components_list = suc.extract_components_from_tuple( repository_components_tuple ) + tool_shed, name, owner, changeset_revision = components_list[ 0:4 ] installed_repository = suc.get_tool_shed_repository_by_shed_name_owner_changeset_revision( trans.app, tool_shed, name, owner, changeset_revision ) if installed_repository not in installed_repositories: installed_repositories.append( installed_repository ) for rd_val in rd_vals: - try: - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = rd_val - except: - tool_shed, name, owner, changeset_revision = rd_val + tool_shed, name, owner, changeset_revision = rd_val[ 0:4 ] installed_repository = suc.get_tool_shed_repository_by_shed_name_owner_changeset_revision( trans.app, tool_shed, name, owner, changeset_revision ) if installed_repository not in installed_repositories: installed_repositories.append( installed_repository ) @@ -631,13 +626,10 @@ # Change the folder id so it won't confict with others being merged. old_container_repository_dependencies_folder.id = folder_id folder_id += 1 + repository_components_tuple = container_util.get_components_from_key( old_container_repository_dependencies_folder.key ) + components_list = suc.extract_components_from_tuple( repository_components_tuple ) + name = components_list[ 1 ] # Generate the label by retrieving the repository name. - try: - toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - container_util.get_components_from_key( old_container_repository_dependencies_folder.key ) - except ValueError: - # For backward compatibility to the 12/20/12 Galaxy release. - toolshed, name, owner, changeset_revision = container_util.get_components_from_key( old_container_repository_dependencies_folder.key ) old_container_repository_dependencies_folder.label = str( name ) repository_dependencies_folder.folders.append( old_container_repository_dependencies_folder ) # Merge tool_dependencies. diff -r fd7f14161d3537abf1f676f14ec763d150ee611f -r 079ce0488cdcf4ee882ba3cd4e5413dcab3f9412 lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -352,19 +352,20 @@ for key, val in repository_dependencies.items(): if key in [ 'root_key', 'description' ]: continue - try: - toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - container_util.get_components_from_key( key ) - components_list = [ toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td ] - except ValueError: - # For backward compatibility to the 12/20/12 Galaxy release, default prior_installation_required and only_if_compiling_contained_td - # to False in the caller. - toolshed, name, owner, changeset_revision = container_util.get_components_from_key( key ) - components_list = [ toolshed, name, owner, changeset_revision ] - only_if_compiling_contained_td = 'False' + repository_components_tuple = container_util.get_components_from_key( key ) + components_list = suc.extract_components_from_tuple( repository_components_tuple ) # Skip listing a repository dependency if it is required only to compile a tool dependency defined for the dependent repository since # in this case, the repository dependency is really a dependency of the dependent repository's contained tool dependency, and only if # that tool dependency requires compilation. + # For backward compatibility to the 12/20/12 Galaxy release. + prior_installation_required = 'False' + only_if_compiling_contained_td = 'False' + if len( components_list ) == 4: + prior_installation_required = 'False' + only_if_compiling_contained_td = 'False' + elif len( components_list ) == 5: + prior_installation_required = components_list[ 4 ] + only_if_compiling_contained_td = 'False' if not util.asbool( only_if_compiling_contained_td ): if components_list not in required_repository_tups: required_repository_tups.append( components_list ) diff -r fd7f14161d3537abf1f676f14ec763d150ee611f -r 079ce0488cdcf4ee882ba3cd4e5413dcab3f9412 lib/tool_shed/util/container_util.py --- a/lib/tool_shed/util/container_util.py +++ b/lib/tool_shed/util/container_util.py @@ -855,7 +855,7 @@ folder_id, data_managers_root_folder = build_invalid_data_managers_folder( trans, folder_id, data_managers, error_messages, label="Invalid Data Managers" ) containers_dict[ 'invalid_data_managers' ] = data_managers_root_folder except Exception, e: - log.debug( "Exception in build_repository_containers_for_tool_shed: %s" % str( e ) ) + log.exception( "Exception in build_repository_containers_for_tool_shed: %s" % str( e ) ) finally: lock.release() return containers_dict @@ -1325,29 +1325,31 @@ repository_name = items[ 1 ] repository_owner = items[ 2 ] changeset_revision = items[ 3 ] - if len( items ) >= 5: - try: - prior_installation_required = items[ 4 ] - except: - prior_installation_required = 'False' - try: - only_if_compiling_contained_td = items[ 5 ] - except: - only_if_compiling_contained_td = 'False' + if len( items ) == 5: + prior_installation_required = items[ 4 ] + return toolshed_base_url, repository_name, repository_owner, changeset_revision, prior_installation_required + elif len( items ) == 6: + prior_installation_required = items[ 4 ] + only_if_compiling_contained_td = items[ 5 ] return toolshed_base_url, repository_name, repository_owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td else: # For backward compatibility to the 12/20/12 Galaxy release we have to return the following, and callers must handle exceptions. return toolshed_base_url, repository_name, repository_owner, changeset_revision def handle_repository_dependencies_container_entry( trans, repository_dependencies_folder, rd_key, rd_value, folder_id, repository_dependency_id, folder_keys ): - try: - toolshed, repository_name, repository_owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - get_components_from_key( rd_key ) - except ValueError: - # For backward compatibility to the 12/20/12 Galaxy release, default prior_installation_required and only_if_compiling_contained_td to 'False'. - toolshed, repository_name, repository_owner, changeset_revision = get_components_from_key( rd_key ) + repository_components_tuple = get_components_from_key( rd_key ) + components_list = suc.extract_components_from_tuple( repository_components_tuple ) + toolshed, repository_name, repository_owner, changeset_revision = components_list[ 0:4 ] + # For backward compatibility to the 12/20/12 Galaxy release. + if len( components_list ) == 4: prior_installation_required = 'False' only_if_compiling_contained_td = 'False' + elif len( components_list ) == 5: + prior_installation_required = components_list[ 4 ] + only_if_compiling_contained_td = 'False' + elif len( components_list ) == 6: + prior_installation_required = components_list[ 4 ] + only_if_compiling_contained_td = components_list[ 5 ] folder = get_folder( repository_dependencies_folder, rd_key ) label = generate_repository_dependencies_folder_label_from_key( repository_name, repository_owner, @@ -1416,14 +1418,19 @@ return False def key_is_current_repositorys_key( repository_name, repository_owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, key ): - try: - toolshed_base_url, key_name, key_owner, key_changeset_revision, key_prior_installation_required, key_only_if_compiling_contained_td = \ - get_components_from_key( key ) - except ValueError: - # For backward compatibility to the 12/20/12 Galaxy release, default key_prior_installation_required to False. - toolshed_base_url, key_name, key_owner, key_changeset_revision = get_components_from_key( key ) + repository_components_tuple = get_components_from_key( key ) + components_list = suc.extract_components_from_tuple( repository_components_tuple ) + toolshed, key_name, key_owner, key_changeset_revision = components_list[ 0:4 ] + # For backward compatibility to the 12/20/12 Galaxy release. + if len( components_list ) == 4: key_prior_installation_required = 'False' key_only_if_compiling_contained_td = 'False' + elif len( components_list ) == 5: + key_prior_installation_required = components_list[ 4 ] + key_only_if_compiling_contained_td = 'False' + elif len( components_list ) == 6: + key_prior_installation_required = components_list[ 4 ] + key_only_if_compiling_contained_td = components_list[ 5 ] if repository_name == key_name and \ repository_owner == key_owner and \ changeset_revision == key_changeset_revision and \ diff -r fd7f14161d3537abf1f676f14ec763d150ee611f -r 079ce0488cdcf4ee882ba3cd4e5413dcab3f9412 lib/tool_shed/util/repository_dependency_util.py --- a/lib/tool_shed/util/repository_dependency_util.py +++ b/lib/tool_shed/util/repository_dependency_util.py @@ -36,12 +36,9 @@ if key in [ 'root_key', 'description' ]: continue d_repository = None - try: - d_toolshed, d_name, d_owner, d_changeset_revision, d_prior_installation_required, d_only_if_compiling_contained_td = \ - container_util.get_components_from_key( key ) - except ValueError: - # For backward compatibility to the 12/20/12 Galaxy release. - d_toolshed, d_name, d_owner, d_changeset_revision = container_util.get_components_from_key( key ) + repository_components_tuple = container_util.get_components_from_key( key ) + components_list = suc.extract_components_from_tuple( repository_components_tuple ) + d_toolshed, d_name, d_owner, d_changeset_revision = components_list[ 0:4 ] for tsr in tool_shed_repositories: # Get the the tool_shed_repository defined by name, owner and changeset_revision. This is the repository that will be # dependent upon each of the tool shed repositories contained in val. We'll need to check tool_shed_repository.tool_shed @@ -437,22 +434,22 @@ # We have the updated changset revision. updated_key_rd_dicts.append( new_key_rd_dict ) else: - try: - toolshed, repository_name, repository_owner, repository_changeset_revision, prior_installation_required, rd_only_if_compiling_contained_td = \ - container_util.get_components_from_key( key ) - except ValueError: - # For backward compatibility to the 12/20/12 Galaxy release. - toolshed, repository_name, repository_owner, repository_changeset_revision = container_util.get_components_from_key( key ) + repository_components_tuple = container_util.get_components_from_key( key ) + components_list = suc.extract_components_from_tuple( repository_components_tuple ) + toolshed, repository_name, repository_owner, repository_changeset_revision = components_list[ 0:4 ] + # For backward compatibility to the 12/20/12 Galaxy release. + if len( components_list ) == 4: + prior_installation_required = 'False' + rd_only_if_compiling_contained_td = 'False' + elif len( components_list ) == 5: + rd_only_if_compiling_contained_td = 'False' message = "The revision %s defined for repository %s owned by %s is invalid, so repository dependencies defined for repository %s will be ignored." % \ ( str( rd_changeset_revision ), str( rd_name ), str( rd_owner ), str( repository_name ) ) log.debug( message ) else: - try: - toolshed, repository_name, repository_owner, repository_changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - container_util.get_components_from_key( key ) - except ValueError: - # For backward compatibility to the 12/20/12 Galaxy release. - toolshed, repository_name, repository_owner, repository_changeset_revision = container_util.get_components_from_key( key ) + repository_components_tuple = container_util.get_components_from_key( key ) + components_list = suc.extract_components_from_tuple( repository_components_tuple ) + toolshed, repository_name, repository_owner, repository_changeset_revision = components_list[ 0:4 ] message = "The revision %s defined for repository %s owned by %s is invalid, so repository dependencies defined for repository %s will be ignored." % \ ( str( rd_changeset_revision ), str( rd_name ), str( rd_owner ), str( repository_name ) ) log.debug( message ) diff -r fd7f14161d3537abf1f676f14ec763d150ee611f -r 079ce0488cdcf4ee882ba3cd4e5413dcab3f9412 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -307,6 +307,21 @@ sa_session.flush() return tool_shed_repository +def extract_components_from_tuple( repository_components_tuple ): + '''Extract the repository components from the provided tuple in a backward-compatible manner.''' + toolshed = repository_components_tuple[ 0 ] + name = repository_components_tuple[ 1 ] + owner = repository_components_tuple[ 2 ] + changeset_revision = repository_components_tuple[ 3 ] + components_list = [ toolshed, name, owner, changeset_revision ] + if len( repository_components_tuple ) == 5: + toolshed, name, owner, changeset_revision, prior_installation_required = repository_components_tuple + components_list = [ toolshed, name, owner, changeset_revision, prior_installation_required ] + elif len( repository_components_tuple ) == 6: + toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = repository_components_tuple + components_list = [ toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td ] + return components_list + def generate_clone_url_for_installed_repository( app, repository ): """Generate the URL for cloning a repository that has been installed into a Galaxy instance.""" tool_shed_url = get_url_from_tool_shed( app, repository.tool_shed ) https://bitbucket.org/galaxy/galaxy-central/commits/78836810e69e/ Changeset: 78836810e69e User: Dave Bouvier Date: 2013-09-24 15:21:41 Summary: Update functional tests to reflect changes in HTML output. Add the ability to browse a running test instance of Galaxy or the tool shed with styles and javascript enabled. Affected #: 9 files diff -r 079ce0488cdcf4ee882ba3cd4e5413dcab3f9412 -r 78836810e69e016a56597f8e273c9baeb13e94dc test/tool_shed/functional/test_0000_basic_repository_features.py --- a/test/tool_shed/functional/test_0000_basic_repository_features.py +++ b/test/tool_shed/functional/test_0000_basic_repository_features.py @@ -38,7 +38,7 @@ self.login( email=common.test_user_1_email, username=common.test_user_1_name ) category = test_db_util.get_category_by_name( 'Test 0000 Basic Repository Features 1' ) strings_displayed = [ 'Repository %s' % "'%s'" % repository_name, - 'Repository %s has been created' % "'%s'" % repository_name ] + 'Repository %s has been created' % "<b>%s</b>" % repository_name ] self.get_or_create_repository( name=repository_name, description=repository_description, long_description=repository_long_description, diff -r 079ce0488cdcf4ee882ba3cd4e5413dcab3f9412 -r 78836810e69e016a56597f8e273c9baeb13e94dc test/tool_shed/functional/test_0120_simple_repository_dependency_multiple_owners.py --- a/test/tool_shed/functional/test_0120_simple_repository_dependency_multiple_owners.py +++ b/test/tool_shed/functional/test_0120_simple_repository_dependency_multiple_owners.py @@ -58,7 +58,7 @@ self.logout() self.login( email=common.test_user_2_email, username=common.test_user_2_name ) strings_displayed = [ 'Repository %s' % "'%s'" % datatypes_repository_name, - 'Repository %s has been created' % "'%s'" % datatypes_repository_name ] + 'Repository %s has been created' % "<b>%s</b>" % datatypes_repository_name ] repository = self.get_or_create_repository( name=datatypes_repository_name, description=datatypes_repository_description, long_description=datatypes_repository_long_description, @@ -98,7 +98,7 @@ self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) strings_displayed = [ 'Repository %s' % "'%s'" % tool_repository_name, - 'Repository %s has been created' % "'%s'" % tool_repository_name ] + 'Repository %s has been created' % "<b>%s</b>" % tool_repository_name ] repository = self.get_or_create_repository( name=tool_repository_name, description=tool_repository_description, long_description=tool_repository_long_description, diff -r 079ce0488cdcf4ee882ba3cd4e5413dcab3f9412 -r 78836810e69e016a56597f8e273c9baeb13e94dc test/tool_shed/functional/test_0400_repository_component_reviews.py --- a/test/tool_shed/functional/test_0400_repository_component_reviews.py +++ b/test/tool_shed/functional/test_0400_repository_component_reviews.py @@ -93,7 +93,7 @@ self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) strings_displayed = [ 'Repository %s' % "'%s'" % repository_name, - 'Repository %s has been created' % "'%s'" % repository_name ] + 'Repository %s has been created' % "<b>%s</b>" % repository_name ] repository = self.get_or_create_repository( name=repository_name, description=repository_description, long_description=repository_long_description, diff -r 079ce0488cdcf4ee882ba3cd4e5413dcab3f9412 -r 78836810e69e016a56597f8e273c9baeb13e94dc test/tool_shed/functional/test_0410_repository_component_review_access_control.py --- a/test/tool_shed/functional/test_0410_repository_component_review_access_control.py +++ b/test/tool_shed/functional/test_0410_repository_component_review_access_control.py @@ -72,7 +72,7 @@ self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) strings_displayed = [ 'Repository %s' % "'%s'" % repository_name, - 'Repository %s has been created' % "'%s'" % repository_name ] + 'Repository %s has been created' % "<b>%s</b>" % repository_name ] repository = self.get_or_create_repository( name=repository_name, description=repository_description, long_description=repository_long_description, diff -r 079ce0488cdcf4ee882ba3cd4e5413dcab3f9412 -r 78836810e69e016a56597f8e273c9baeb13e94dc test/tool_shed/functional/test_0420_citable_urls_for_repositories.py --- a/test/tool_shed/functional/test_0420_citable_urls_for_repositories.py +++ b/test/tool_shed/functional/test_0420_citable_urls_for_repositories.py @@ -52,7 +52,7 @@ self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) strings_displayed = [ 'Repository %s' % "'%s'" % repository_name, - 'Repository %s has been created' % "'%s'" % repository_name ] + 'Repository %s has been created' % "<b>%s</b>" % repository_name ] repository = self.get_or_create_repository( name=repository_name, description=repository_description, long_description=repository_long_description, diff -r 079ce0488cdcf4ee882ba3cd4e5413dcab3f9412 -r 78836810e69e016a56597f8e273c9baeb13e94dc test/tool_shed/functional/test_0430_browse_utilities.py --- a/test/tool_shed/functional/test_0430_browse_utilities.py +++ b/test/tool_shed/functional/test_0430_browse_utilities.py @@ -57,7 +57,7 @@ self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) strings_displayed = [ 'Repository %s' % "'%s'" % datatypes_repository_name, - 'Repository %s has been created' % "'%s'" % datatypes_repository_name ] + 'Repository %s has been created' % "<b>%s</b>" % datatypes_repository_name ] repository = self.get_or_create_repository( name=datatypes_repository_name, description=datatypes_repository_description, long_description=datatypes_repository_long_description, @@ -85,7 +85,7 @@ self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) strings_displayed = [ 'Repository %s' % "'%s'" % emboss_repository_name, - 'Repository %s has been created' % "'%s'" % emboss_repository_name ] + 'Repository %s has been created' % "<b>%s</b>" % emboss_repository_name ] emboss_repository = self.get_or_create_repository( name=emboss_repository_name, description=emboss_repository_description, long_description=emboss_repository_long_description, @@ -122,7 +122,7 @@ self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) strings_displayed = [ 'Repository %s' % "'%s'" % freebayes_repository_name, - 'Repository %s has been created' % "'%s'" % freebayes_repository_name ] + 'Repository %s has been created' % "<b>%s</b>" % freebayes_repository_name ] repository = self.get_or_create_repository( name=freebayes_repository_name, description=freebayes_repository_description, long_description=freebayes_repository_long_description, diff -r 079ce0488cdcf4ee882ba3cd4e5413dcab3f9412 -r 78836810e69e016a56597f8e273c9baeb13e94dc test/tool_shed/functional/test_0450_skip_tool_tests.py --- a/test/tool_shed/functional/test_0450_skip_tool_tests.py +++ b/test/tool_shed/functional/test_0450_skip_tool_tests.py @@ -76,7 +76,7 @@ self.login( email=common.test_user_1_email, username=common.test_user_1_name ) category = test_db_util.get_category_by_name( category_name ) strings_displayed = [ 'Repository %s' % "'%s'" % repository_name, - 'Repository %s has been created' % "'%s'" % repository_name ] + 'Repository %s has been created' % "<b>%s</b>" % repository_name ] repository = self.get_or_create_repository( name=repository_name, description=repository_description, long_description=repository_long_description, diff -r 079ce0488cdcf4ee882ba3cd4e5413dcab3f9412 -r 78836810e69e016a56597f8e273c9baeb13e94dc test/tool_shed/functional/test_1120_simple_repository_dependency_multiple_owners.py --- a/test/tool_shed/functional/test_1120_simple_repository_dependency_multiple_owners.py +++ b/test/tool_shed/functional/test_1120_simple_repository_dependency_multiple_owners.py @@ -63,7 +63,7 @@ self.logout() self.login( email=common.test_user_2_email, username=common.test_user_2_name ) strings_displayed = [ 'Repository %s' % "'%s'" % datatypes_repository_name, - 'Repository %s has been created' % "'%s'" % datatypes_repository_name ] + 'Repository %s has been created' % "<b>%s</b>" % datatypes_repository_name ] repository = self.get_or_create_repository( name=datatypes_repository_name, description=datatypes_repository_description, long_description=datatypes_repository_long_description, @@ -105,7 +105,7 @@ self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) strings_displayed = [ 'Repository %s' % "'%s'" % tool_repository_name, - 'Repository %s has been created' % "'%s'" % tool_repository_name ] + 'Repository %s has been created' % "<b>%s</b>" % tool_repository_name ] repository = self.get_or_create_repository( name=tool_repository_name, description=tool_repository_description, long_description=tool_repository_long_description, diff -r 079ce0488cdcf4ee882ba3cd4e5413dcab3f9412 -r 78836810e69e016a56597f8e273c9baeb13e94dc test/tool_shed/functional_tests.py --- a/test/tool_shed/functional_tests.py +++ b/test/tool_shed/functional_tests.py @@ -56,6 +56,40 @@ default_galaxy_test_port_max = 9999 default_galaxy_test_host = 'localhost' +# should this serve static resources (scripts, images, styles, etc.) +STATIC_ENABLED = True + +def get_static_settings(): + """Returns dictionary of the settings necessary for a galaxy App + to be wrapped in the static middleware. + + This mainly consists of the filesystem locations of url-mapped + static resources. + """ + cwd = os.getcwd() + static_dir = os.path.join( cwd, 'static' ) + #TODO: these should be copied from universe_wsgi.ini + return dict( + #TODO: static_enabled needed here? + static_enabled = True, + static_cache_time = 360, + static_dir = static_dir, + static_images_dir = os.path.join( static_dir, 'images', '' ), + static_favicon_dir = os.path.join( static_dir, 'favicon.ico' ), + static_scripts_dir = os.path.join( static_dir, 'scripts', '' ), + static_style_dir = os.path.join( static_dir, 'june_2007_style', 'blue' ), + static_robots_txt = os.path.join( static_dir, 'robots.txt' ), + ) + +def get_webapp_global_conf(): + """Get the global_conf dictionary sent as the first argument to app_factory. + """ + # (was originally sent 'dict()') - nothing here for now except static settings + global_conf = dict() + if STATIC_ENABLED: + global_conf.update( get_static_settings() ) + return global_conf + tool_sheds_conf_xml_template = '''<?xml version="1.0"?><tool_sheds><tool_shed name="Embedded tool shed for functional tests" url="http://${shed_url}:${shed_port}/"/> @@ -144,6 +178,9 @@ galaxy_migrated_tool_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir ) galaxy_tool_dependency_dir = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir ) os.environ[ 'GALAXY_TEST_TOOL_DEPENDENCY_DIR' ] = galaxy_tool_dependency_dir + hgweb_config_dir = hgweb_config_file_path + os.environ[ 'TEST_HG_WEB_CONFIG_DIR' ] = hgweb_config_dir + print "Directory location for hgweb.config:", hgweb_config_dir if 'TOOL_SHED_TEST_DBURI' in os.environ: toolshed_database_connection = os.environ[ 'TOOL_SHED_TEST_DBURI' ] else: @@ -152,7 +189,29 @@ galaxy_database_connection = os.environ[ 'GALAXY_TEST_DBURI' ] else: galaxy_database_connection = 'sqlite:///' + os.path.join( galaxy_db_path, 'universe_test.sqlite' ) - kwargs = {} + tool_shed_global_conf = get_webapp_global_conf() + tool_shed_global_conf[ '__file__' ] = 'tool_shed_wsgi.ini.sample' + kwargs = dict( admin_users = 'test@bx.psu.edu', + allow_user_creation = True, + allow_user_deletion = True, + database_connection = toolshed_database_connection, + datatype_converters_config_file = 'datatype_converters_conf.xml.sample', + file_path = shed_file_path, + global_conf = tool_shed_global_conf, + hgweb_config_dir = hgweb_config_dir, + job_queue_workers = 5, + id_secret = 'changethisinproductiontoo', + log_destination = "stdout", + new_file_path = new_repos_path, + running_functional_tests = True, + shed_tool_data_table_config = shed_tool_data_table_conf_file, + smtp_server = 'smtp.dummy.string.tld', + email_from = 'functional@localhost', + template_path = 'templates', + tool_path=tool_path, + tool_parse_help = False, + tool_data_table_config_path = galaxy_tool_data_table_conf_file, + use_heartbeat = False ) for dir in [ tool_shed_test_tmp_dir ]: try: os.makedirs( dir ) @@ -162,11 +221,6 @@ print "Tool shed database connection:", toolshed_database_connection print "Galaxy database connection:", galaxy_database_connection - hgweb_config_dir = hgweb_config_file_path - os.environ[ 'TEST_HG_WEB_CONFIG_DIR' ] = hgweb_config_dir - - print "Directory location for hgweb.config:", hgweb_config_dir - # Generate the tool_data_table_conf.xml file. file( galaxy_tool_data_table_conf_file, 'w' ).write( tool_data_table_conf_xml_template ) # Generate the shed_tool_data_table_conf.xml file. @@ -174,7 +228,6 @@ os.environ[ 'TOOL_SHED_TEST_TOOL_DATA_TABLE_CONF' ] = shed_tool_data_table_conf_file # ---- Build Tool Shed Application -------------------------------------------------- toolshedapp = None - global_conf = { '__file__' : 'tool_shed_wsgi.ini.sample' } # if not toolshed_database_connection.startswith( 'sqlite://' ): # kwargs[ 'database_engine_option_max_overflow' ] = '20' if tool_dependency_dir is not None: @@ -183,41 +236,22 @@ kwargs[ 'object_store' ] = 'distributed' kwargs[ 'distributed_object_store_config_file' ] = 'distributed_object_store_conf.xml.sample' + kwargs[ 'global_conf' ] = tool_shed_global_conf if not toolshed_database_connection.startswith( 'sqlite://' ): kwargs[ 'database_engine_option_pool_size' ] = '10' - toolshedapp = ToolshedUniverseApplication( admin_users = 'test@bx.psu.edu', - allow_user_creation = True, - allow_user_deletion = True, - database_connection = toolshed_database_connection, - datatype_converters_config_file = 'datatype_converters_conf.xml.sample', - file_path = shed_file_path, - global_conf = global_conf, - hgweb_config_dir = hgweb_config_dir, - job_queue_workers = 5, - id_secret = 'changethisinproductiontoo', - log_destination = "stdout", - new_file_path = new_repos_path, - running_functional_tests = True, - shed_tool_data_table_config = shed_tool_data_table_conf_file, - smtp_server = 'smtp.dummy.string.tld', - email_from = 'functional@localhost', - template_path = 'templates', - tool_path=tool_path, - tool_parse_help = False, - tool_data_table_config_path = galaxy_tool_data_table_conf_file, - use_heartbeat = False, - **kwargs ) + toolshedapp = ToolshedUniverseApplication( **kwargs ) log.info( "Embedded Toolshed application started" ) # ---- Run tool shed webserver ------------------------------------------------------ tool_shed_server = None - toolshedwebapp = toolshedbuildapp.app_factory( dict( database_connection=toolshed_database_connection ), - use_translogger=False, - static_enabled=False, - app=toolshedapp ) + tool_shed_global_conf[ 'database_connection' ] = toolshed_database_connection + toolshedwebapp = toolshedbuildapp.app_factory( tool_shed_global_conf, + use_translogger=False, + static_enabled=True, + app=toolshedapp ) if tool_shed_test_port is not None: tool_shed_server = httpserver.serve( toolshedwebapp, host=tool_shed_test_host, port=tool_shed_test_port, start_loop=False ) else: @@ -268,50 +302,53 @@ migrated_tool_conf_xml = shed_tool_conf_template_parser.safe_substitute( shed_tool_path=galaxy_migrated_tool_path ) file( galaxy_migrated_tool_conf_file, 'w' ).write( migrated_tool_conf_xml ) os.environ[ 'GALAXY_TEST_SHED_TOOL_CONF' ] = galaxy_shed_tool_conf_file + galaxy_global_conf = get_webapp_global_conf() + galaxy_global_conf[ '__file__' ] = 'universe_wsgi.ini.sample' + + kwargs = dict( allow_user_creation = True, + allow_user_deletion = True, + admin_users = 'test@bx.psu.edu', + allow_library_path_paste = True, + database_connection = galaxy_database_connection, + datatype_converters_config_file = "datatype_converters_conf.xml.sample", + enable_tool_shed_check = True, + file_path = galaxy_file_path, + global_conf = galaxy_global_conf, + hours_between_check = 0.001, + id_secret = 'changethisinproductiontoo', + job_queue_workers = 5, + log_destination = "stdout", + migrated_tools_config = galaxy_migrated_tool_conf_file, + new_file_path = galaxy_tempfiles, + running_functional_tests=True, + shed_tool_data_table_config = shed_tool_data_table_conf_file, + shed_tool_path = galaxy_shed_tool_path, + template_path = "templates", + tool_data_path = tool_data_path, + tool_dependency_dir = galaxy_tool_dependency_dir, + tool_path = tool_path, + tool_config_file = [ galaxy_tool_conf_file, galaxy_shed_tool_conf_file ], + tool_sheds_config_file = galaxy_tool_sheds_conf_file, + tool_parse_help = False, + tool_data_table_config_path = galaxy_tool_data_table_conf_file, + update_integrated_tool_panel = False, + use_heartbeat = False ) # ---- Build Galaxy Application -------------------------------------------------- - galaxy_global_conf = { '__file__' : 'universe_wsgi.ini.sample' } if not galaxy_database_connection.startswith( 'sqlite://' ): kwargs[ 'database_engine_option_pool_size' ] = '10' kwargs[ 'database_engine_option_max_overflow' ] = '20' - galaxyapp = GalaxyUniverseApplication( allow_user_creation = True, - allow_user_deletion = True, - admin_users = 'test@bx.psu.edu', - allow_library_path_paste = True, - database_connection = galaxy_database_connection, - datatype_converters_config_file = "datatype_converters_conf.xml.sample", - enable_tool_shed_check = True, - file_path = galaxy_file_path, - global_conf = global_conf, - hours_between_check = 0.001, - id_secret = 'changethisinproductiontoo', - job_queue_workers = 5, - log_destination = "stdout", - migrated_tools_config = galaxy_migrated_tool_conf_file, - new_file_path = galaxy_tempfiles, - running_functional_tests=True, - shed_tool_data_table_config = shed_tool_data_table_conf_file, - shed_tool_path = galaxy_shed_tool_path, - template_path = "templates", - tool_data_path = tool_data_path, - tool_dependency_dir = galaxy_tool_dependency_dir, - tool_path = tool_path, - tool_config_file = [ galaxy_tool_conf_file, galaxy_shed_tool_conf_file ], - tool_sheds_config_file = galaxy_tool_sheds_conf_file, - tool_parse_help = False, - tool_data_table_config_path = galaxy_tool_data_table_conf_file, - update_integrated_tool_panel = False, - use_heartbeat = False, - **kwargs ) + galaxyapp = GalaxyUniverseApplication( **kwargs ) log.info( "Embedded Galaxy application started" ) # ---- Run galaxy webserver ------------------------------------------------------ galaxy_server = None - galaxywebapp = galaxybuildapp.app_factory( dict( database_file=galaxy_database_connection ), - use_translogger=False, - static_enabled=False, - app=galaxyapp ) + galaxy_global_conf[ 'database_file' ] = galaxy_database_connection + galaxywebapp = galaxybuildapp.app_factory( galaxy_global_conf, + use_translogger=False, + static_enabled=True, + app=galaxyapp ) if galaxy_test_port is not None: galaxy_server = httpserver.serve( galaxywebapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False ) @@ -412,4 +449,8 @@ return 1 if __name__ == "__main__": - sys.exit( main() ) + try: + sys.exit( main() ) + except Exception, e: + log.exception( str( e ) ) + exit(1) https://bitbucket.org/galaxy/galaxy-central/commits/7df8d81544c3/ Changeset: 7df8d81544c3 User: Dave Bouvier Date: 2013-09-24 15:23:44 Summary: Update for automated repository installation and testing framework to record cases where a repository installation fails before a record is created in the embedded Galaxy instance's database. Affected #: 1 file diff -r 78836810e69e016a56597f8e273c9baeb13e94dc -r 7df8d81544c39cde769a5ead49be182294ba58b1 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -254,6 +254,49 @@ success = result.wasSuccessful() return success +def extract_log_data( test_result, from_tool_test=True ): + '''Extract any useful data from the test_result.failures and test_result.errors attributes.''' + for failure in test_result.failures + test_result.errors: + # Record the twill test identifier and information about the tool, so the repository owner can discover which test is failing. + test_id = str( failure[0] ) + if not from_tool_test: + tool_id = None + tool_version = None + else: + tool_id, tool_version = get_tool_info_from_test_id( test_id ) + test_status = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version ) + log_output = failure[1].replace( '\\n', '\n' ) + # Remove debug output that the reviewer or owner doesn't need. + log_output = re.sub( r'control \d+:.+', r'', log_output ) + log_output = re.sub( r'\n+', r'\n', log_output ) + appending_to = 'output' + tmp_output = {} + output = {} + # Iterate through the functional test output and extract only the important data. Captured logging and stdout are not recorded. + for line in log_output.split( '\n' ): + if line.startswith( 'Traceback' ): + appending_to = 'traceback' + elif '>> end captured' in line or '>> end tool' in line: + continue + elif 'request returned None from get_history' in line: + continue + elif '>> begin captured logging <<' in line: + appending_to = 'logging' + continue + elif '>> begin captured stdout <<' in line: + appending_to = 'stdout' + continue + elif '>> begin captured stderr <<' in line or '>> begin tool stderr <<' in line: + appending_to = 'stderr' + continue + if appending_to not in tmp_output: + tmp_output[ appending_to ] = [] + tmp_output[ appending_to ].append( line ) + for output_type in [ 'stderr', 'traceback' ]: + if output_type in tmp_output: + test_status[ output_type ] = '\n'.join( tmp_output[ output_type ] ) + return test_status + def get_api_url( base, parts=[], params=None, key=None ): if 'api' in parts and parts.index( 'api' ) != 0: parts.pop( parts.index( 'api' ) ) @@ -1054,43 +1097,7 @@ params ) log.debug( 'Revision %s of repository %s installed and passed functional tests.', changeset_revision, name ) else: - # If the functional tests fail, log the output and update the failed changeset revision's metadata record in the tool shed via the API. - for failure in result.failures + result.errors: - # Record the twill test identifier and information about the tool, so the repository owner can discover which test is failing. - test_id = str( failure[0] ) - tool_id, tool_version = get_tool_info_from_test_id( test_id ) - test_status = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version ) - log_output = failure[1].replace( '\\n', '\n' ) - # Remove debug output that the reviewer or owner doesn't need. - log_output = re.sub( r'control \d+:.+', r'', log_output ) - log_output = re.sub( r'\n+', r'\n', log_output ) - appending_to = 'output' - tmp_output = {} - output = {} - # Iterate through the functional test output and extract only the important data. Captured logging and stdout are not recorded. - for line in log_output.split( '\n' ): - if line.startswith( 'Traceback' ): - appending_to = 'traceback' - elif '>> end captured' in line or '>> end tool' in line: - continue - elif 'request returned None from get_history' in line: - continue - elif '>> begin captured logging <<' in line: - appending_to = 'logging' - continue - elif '>> begin captured stdout <<' in line: - appending_to = 'stdout' - continue - elif '>> begin captured stderr <<' in line or '>> begin tool stderr <<' in line: - appending_to = 'stderr' - continue - if appending_to not in tmp_output: - tmp_output[ appending_to ] = [] - tmp_output[ appending_to ].append( line ) - for output_type in [ 'stderr', 'traceback' ]: - if output_type in tmp_output: - test_status[ output_type ] = '\n'.join( tmp_output[ output_type ] ) - repository_status[ 'failed_tests' ].append( test_status ) + repository_status[ 'failed_tests' ] = extract_log_data( result, from_tool_test=True ) # Call the register_test_result method, which executes a PUT request to the repository_revisions API controller with the outcome # of the tests, and updates tool_test_results with the relevant log data. # This also sets the do_not_test and tools_functionally correct flags to the appropriate values, and updates the time_last_tested @@ -1129,7 +1136,7 @@ name=repository.name, owner=repository.owner, changeset_revision=repository.changeset_revision, - error_message=repository.error_message ) + error_message=extract_log_data( result, from_tool_test=False ) ) repository_status[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) params[ 'tools_functionally_correct' ] = False params[ 'test_install_error' ] = True Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.