- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: greg: Tool shed install and test framework changes to enable soon coming testing of repositories of type tool_dependency_definition.
by commits-noreply＠bitbucket.org 17 Dec '13

17 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/43b23f550399/ Changeset: 43b23f550399 User: greg Date: 2013-12-17 20:59:47 Summary: Tool shed install and test framework changes to enable soon coming testing of repositories of type tool_dependency_definition. Affected #: 5 files diff -r b1e82182414d359f6f43e117e92ec6d6638878fe -r 43b23f5503997727debf8232239434974491e55e install_and_test_tool_shed_repositories.sh --- a/install_and_test_tool_shed_repositories.sh +++ b/install_and_test_tool_shed_repositories.sh @@ -2,14 +2,16 @@ # A good place to look for nose info: http://somethingaboutorange.com/mrl/projects/nose/ -# The test/install_and_test_tool_shed_repositories/functional_tests.py can not be executed directly, because it must have certain functional test definitions -# in sys.argv. Running it through this shell script is the best way to ensure that it has the required definitions. +# The test/install_and_test_tool_shed_repositories/functional_tests.py cannot be executed directly because it must +# have certain functional test definitions in sys.argv. Running it through this shell script is the best way to +# ensure that it has the required definitions. -# This script requires the following environment variables: +# This script requires setting of the following environment variables: # GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY - must be set to the API key for the tool shed that is being checked. # GALAXY_INSTALL_TEST_TOOL_SHED_URL - must be set to a URL that the tool shed is listening on. -# If the tool shed url is not specified in tool_sheds_conf.xml, GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF must be set to a tool sheds configuration file -# that does specify that url, otherwise repository installation will fail. + +# If the tool shed url is not specified in tool_sheds_conf.xml, GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF must be set to +# a tool sheds configuration file that does specify that url or repository installation will fail. if [ -z $GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY ] ; then echo "This script requires the GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY environment variable to be set and non-empty." @@ -37,7 +39,11 @@ fi fi -python test/install_and_test_tool_shed_repositories/functional_tests.py $* -v --with-nosehtml --html-report-file \ - test/install_and_test_tool_shed_repositories/run_functional_tests.html \ +# Test installation of repositories of type tool_dependency_definition. +# ...coming soon! + +# Test installation of repositories that contain valid tools with defined functional tests and a test-data directory containing test files. +python test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py $* -v --with-nosehtml --html-report-file \ + test/install_and_test_tool_shed_repositories/repositories_with_tools/run_functional_tests.html \ test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py \ test/functional/test_toolbox.py diff -r b1e82182414d359f6f43e117e92ec6d6638878fe -r 43b23f5503997727debf8232239434974491e55e test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ /dev/null @@ -1,1492 +0,0 @@ -#!/usr/bin/env python -""" -This script cannot be run directly, because it needs to have test/functional/test_toolbox.py in sys.argv in -order to run functional tests on repository tools after installation. The install_and_test_tool_shed_repositories.sh -will execute this script with the appropriate parameters. -""" -import os -import sys -# Assume we are run from the galaxy root directory, add lib to the python path -cwd = os.getcwd() -sys.path.append( cwd ) -new_path = [ os.path.join( cwd, "scripts" ), - os.path.join( cwd, "lib" ), - os.path.join( cwd, 'test' ), - os.path.join( cwd, 'scripts', 'api' ) ] -new_path.extend( sys.path ) -sys.path = new_path - -from galaxy import eggs -eggs.require( "nose" ) -eggs.require( "Paste" ) -eggs.require( 'mercurial' ) -# This should not be required, but it is under certain conditions thanks to this bug: -# http://code.google.com/p/python-nose/issues/detail?id=284 -eggs.require( "pysqlite" ) - -test_toolbox = None -import httplib -import install_and_test_tool_shed_repositories.base.test_db_util as test_db_util -import install_and_test_tool_shed_repositories.functional.test_install_repositories as test_install_repositories -import logging -import nose -import platform -import random -import re -import shutil -import socket -import string -import tempfile -import time -import threading -import tool_shed.util.shed_util_common as suc -import urllib - -from base.tool_shed_util import log_reason_repository_cannot_be_uninstalled -from base.tool_shed_util import parse_tool_panel_config -from install_and_test_tool_shed_repositories.base.util import get_database_version -from install_and_test_tool_shed_repositories.base.util import get_repository_current_revision -from common import update -from datetime import datetime -from galaxy.app import UniverseApplication -from galaxy.util import asbool -from galaxy.util import listify -from galaxy.util import unicodify -from galaxy.util.json import from_json_string -from galaxy.util.json import to_json_string -from galaxy.web import buildapp -from galaxy.web.framework.helpers import time_ago -from functional_tests import generate_config_file -from mercurial import __version__ -from nose.plugins import Plugin -from paste import httpserver -from tool_shed.util import tool_dependency_util -from tool_shed.util.xml_util import parse_xml - -from functional import database_contexts - -log = logging.getLogger( 'install_and_test_repositories' ) - -assert sys.version_info[ :2 ] >= ( 2, 6 ) -test_home_directory = os.path.join( cwd, 'test', 'install_and_test_tool_shed_repositories' ) -default_test_file_dir = os.path.join( test_home_directory, 'test_data' ) - -# Here's the directory where everything happens. Temporary directories are created within this directory to contain -# the database, new repositories, etc. -galaxy_test_tmp_dir = os.path.join( test_home_directory, 'tmp' ) -default_galaxy_locales = 'en' -default_galaxy_test_file_dir = "test-data" -os.environ[ 'GALAXY_INSTALL_TEST_TMP_DIR' ] = galaxy_test_tmp_dir - -default_galaxy_test_port_min = 10000 -default_galaxy_test_port_max = 10999 -default_galaxy_test_host = '127.0.0.1' -# The following should be an actual value (not None). If developers manually specify their -# tests to use the API it will not work unless a master API key is specified. -default_galaxy_master_api_key = 123456 - -# This script can be run in such a way that no Tool Shed database records should be changed. -if '-info_only' in sys.argv or 'GALAXY_INSTALL_TEST_INFO_ONLY' in os.environ: - can_update_tool_shed = False -else: - can_update_tool_shed = True - -# Should this serve static resources (scripts, images, styles, etc.)? -STATIC_ENABLED = True - -# Set up a job_conf.xml that explicitly limits jobs to 10 minutes. -job_conf_xml = '''<?xml version="1.0"?> - -<job_conf> - <plugins> - <plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner" workers="4"/> - </plugins> - <handlers> - <handler id="main"/> - </handlers> - <destinations> - <destination id="local" runner="local"/> - </destinations> - <limits> - <limit type="walltime">00:10:00</limit> - </limits> -</job_conf> -''' - -# Create a blank shed_tool_conf.xml to define the installed repositories. -shed_tool_conf_xml_template = '''<?xml version="1.0"?> -<toolbox tool_path="${shed_tool_path}"> -</toolbox> -''' - -# Since we will be running functional tests we'll need the upload tool, but the rest can be omitted. -tool_conf_xml = '''<?xml version="1.0"?> -<toolbox> - <section name="Get Data" id="getext"> - <tool file="data_source/upload.xml"/> - </section> -</toolbox> -''' - -# Set up an empty shed_tool_data_table_conf.xml. -tool_data_table_conf_xml_template = '''<?xml version="1.0"?> -<tables> -</tables> -''' - -# Optionally set the environment variable GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF to the location of a -# tool shed's configuration file that includes the tool shed from which repositories will be installed. -tool_sheds_conf_xml = '''<?xml version="1.0"?> -<tool_sheds> - <tool_shed name="Galaxy main tool shed" url="http://toolshed.g2.bx.psu.edu/"/> - <tool_shed name="Galaxy test tool shed" url="http://testtoolshed.g2.bx.psu.edu/"/> -</tool_sheds> -''' - -# If we have a tool_data_table_conf.test.xml, set it up to be loaded when the UniverseApplication is started. -# This allows one to specify a set of tool data that is used exclusively for testing, and not loaded into any -# Galaxy instance. By default, this will be in the test-data-repo/location directory generated by buildbot_setup.sh. -if os.path.exists( 'tool_data_table_conf.test.xml' ): - additional_tool_data_tables = 'tool_data_table_conf.test.xml' - additional_tool_data_path = os.environ.get( 'GALAXY_INSTALL_TEST_EXTRA_TOOL_DATA_PATH', - os.path.join( 'test-data-repo', 'location' ) ) -else: - additional_tool_data_tables = None - additional_tool_data_path = None - -# Set up default tool data tables. -if os.path.exists( 'tool_data_table_conf.xml' ): - tool_data_table_conf = 'tool_data_table_conf.xml' -elif os.path.exists( 'tool_data_table_conf.xml.sample' ): - tool_data_table_conf = 'tool_data_table_conf.xml.sample' -else: - tool_data_table_conf = None - -# The GALAXY_INSTALL_TEST_TOOL_SHED_URL and GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY environment variables must be -# set for this script to work correctly. If the value of GALAXY_INSTALL_TEST_TOOL_SHED_URL does not refer to one -# of the defaults, the GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF must refer to a tool shed configuration file that contains -# a definition for that tool shed. -galaxy_tool_shed_url = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_URL', None ) -tool_shed_api_key = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY', None ) -exclude_list_file = os.environ.get( 'GALAXY_INSTALL_TEST_EXCLUDE_REPOSITORIES', 'install_test_exclude.xml' ) - -if 'GALAXY_INSTALL_TEST_SECRET' not in os.environ: - galaxy_encode_secret = 'changethisinproductiontoo' - os.environ[ 'GALAXY_INSTALL_TEST_SECRET' ] = galaxy_encode_secret -else: - galaxy_encode_secret = os.environ[ 'GALAXY_INSTALL_TEST_SECRET' ] - -testing_single_repository_dict = {} -if 'repository_name' in os.environ and 'repository_owner' in os.environ: - testing_single_repository_dict[ 'name' ] = str( os.environ[ 'repository_name' ] ) - testing_single_repository_dict[ 'owner' ] = str( os.environ[ 'repository_owner' ] ) - if 'repository_revision' in os.environ: - testing_single_repository_dict[ 'changeset_revision' ] = str( os.environ[ 'repository_revision' ] ) - else: - testing_single_repository_dict[ 'changeset_revision' ] = None - - -class ReportResults( Plugin ): - '''Simple Nose plugin to record the IDs of all tests run, regardless of success.''' - name = "reportresults" - passed = {} - - def options( self, parser, env=os.environ ): - super( ReportResults, self ).options( parser, env=env ) - - def configure(self, options, conf): - super( ReportResults, self ).configure( options, conf ) - if not self.enabled: - return - - def addSuccess( self, test ): - '''Only record test IDs that correspond to tool functional tests.''' - if 'TestForTool' in test.id(): - test_id = test.id() - # Rearrange the test ID to match the format that is produced in test_results.failures - test_id_parts = test_id.split( '.' ) - fixed_test_id = '%s (%s)' % ( test_id_parts[ -1 ], '.'.join( test_id_parts[ :-1 ] ) ) - test_parts = fixed_test_id.split( '/' ) - owner = test_parts[ -4 ] - name = test_parts[ -3 ] - test_identifier = '%s/%s' % ( owner, name ) - if test_identifier not in self.passed: - self.passed[ test_identifier ] = [] - self.passed[ test_identifier ].append( fixed_test_id ) - - def getTestStatus( self, test_identifier ): - if test_identifier in self.passed: - passed_tests = self.passed[ test_identifier ] - del self.passed[ test_identifier ] - return passed_tests - return [] - -def deactivate_repository( app, repository_dict ): - sa_session = app.install_model.context - # Clean out any generated tests. This is necessary for Twill. - remove_generated_tests( app ) - # The dict contains the only repository the app should have installed at this point. - name = str( repository_dict[ 'name' ] ) - owner = str( repository_dict[ 'owner' ] ) - changeset_revision = str( repository_dict[ 'changeset_revision' ] ) - repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) - repository_dict_for_deactivation = dict( name=name, - owner=owner, - changeset_revision=changeset_revision ) - log.debug( 'Changeset revision %s of repository %s owned by %s selected for deactivation.' % ( changeset_revision, name, owner ) ) - test_install_repositories.generate_deactivate_method( repository_dict_for_deactivation ) - # Set up nose to run the generated uninstall method as a functional test. - test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) - test_config.configure( sys.argv ) - # Run the deactivate method which uses the Galaxy web interface to deactivate the repository. - result, _ = run_tests( test_config ) - success = result.wasSuccessful() - if not success: - log.debug( 'Deactivation of revision %s repository %s owned by %s failed.' % ( changeset_revision, name, owner ) ) - -def get_api_url( base, parts=[], params=None ): - if 'api' in parts and parts.index( 'api' ) != 0: - parts.pop( parts.index( 'api' ) ) - parts.insert( 0, 'api' ) - elif 'api' not in parts: - parts.insert( 0, 'api' ) - url = suc.url_join( base, *parts ) - if params is not None: - query_string = urllib.urlencode( params ) - url += '?%s' % query_string - return url - -def get_failed_test_dicts( test_result, from_tool_test=True ): - """Extract any useful data from the test_result.failures and test_result.errors attributes.""" - failed_test_dicts = [] - for failure in test_result.failures + test_result.errors: - # Record the twill test identifier and information about the tool so the repository owner - # can discover which test is failing. - test_id = str( failure[ 0 ] ) - if not from_tool_test: - tool_id = None - tool_version = None - else: - tool_id, tool_version = get_tool_info_from_test_id( test_id ) - test_status_dict = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version ) - log_output = failure[ 1 ].replace( '\\n', '\n' ) - # Remove debug output. - log_output = re.sub( r'control \d+:.+', r'', log_output ) - log_output = re.sub( r'\n+', r'\n', log_output ) - appending_to = 'output' - tmp_output = {} - output = {} - # Iterate through the functional test output and extract only the important data. Captured - # logging and stdout are not recorded. - for line in log_output.split( '\n' ): - if line.startswith( 'Traceback' ): - appending_to = 'traceback' - elif '>> end captured' in line or '>> end tool' in line: - continue - elif 'request returned None from get_history' in line: - continue - elif '>> begin captured logging <<' in line: - appending_to = 'logging' - continue - elif '>> begin captured stdout <<' in line: - appending_to = 'stdout' - continue - elif '>> begin captured stderr <<' in line or '>> begin tool stderr <<' in line: - appending_to = 'stderr' - continue - if appending_to in tmp_output: - tmp_output[ appending_to ].append( line ) - else: - tmp_output[ appending_to ] = [ line ] - for output_type in [ 'stderr', 'traceback' ]: - if output_type in tmp_output: - test_status_dict[ output_type ] = '\n'.join( tmp_output[ output_type ] ) - failed_test_dicts.append( test_status_dict ) - return failed_test_dicts - -def get_latest_downloadable_changeset_revision( url, name, owner ): - error_message = '' - parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ] - params = dict( name=name, owner=owner ) - api_url = get_api_url( base=url, parts=parts, params=params ) - changeset_revisions, error_message = json_from_url( api_url ) - if error_message: - return None, error_message - if changeset_revisions: - return changeset_revisions[ -1 ], error_message - else: - return suc.INITIAL_CHANGELOG_HASH, error_message - -def get_missing_tool_dependencies( repository ): - log.debug( 'Checking %s repository %s for missing tool dependencies.' % ( repository.status, repository.name ) ) - missing_tool_dependencies = repository.missing_tool_dependencies - for tool_dependency in repository.tool_dependencies: - log.debug( 'Tool dependency %s version %s has status %s.' % ( tool_dependency.name, tool_dependency.version, tool_dependency.status ) ) - for repository_dependency in repository.repository_dependencies: - if repository_dependency.includes_tool_dependencies: - missing_tool_dependencies.extend( get_missing_tool_dependencies( repository_dependency ) ) - return missing_tool_dependencies - -def get_repository_dict( url, repository_dict ): - error_message = '' - parts = [ 'api', 'repositories', repository_dict[ 'repository_id' ] ] - api_url = get_api_url( base=url, parts=parts ) - extended_dict, error_message = json_from_url( api_url ) - if error_message: - return None, error_message - name = str( extended_dict[ 'name' ] ) - owner = str( extended_dict[ 'owner' ] ) - latest_changeset_revision, error_message = get_latest_downloadable_changeset_revision( url, name, owner ) - if error_message: - return None, error_message - extended_dict[ 'latest_revision' ] = str( latest_changeset_revision ) - return extended_dict, error_message - -def get_repository_tuple_from_elem( elem ): - attributes = elem.attrib - name = attributes.get( 'name', None ) - owner = attributes.get( 'owner', None ) - changeset_revision = attributes.get( 'changeset_revision', None ) - return ( name, owner, changeset_revision ) - -def get_repositories_to_install( tool_shed_url ): - """ - Get a list of repository info dicts to install. This method expects a json list of dicts with the following structure: - [{ "changeset_revision": <revision>, - "encoded_repository_id": <encoded repository id from the tool shed>, - "name": <name>, - "owner": <owner>, - "tool_shed_url": <url> }] - """ - error_message = '' - latest_revision_only = '-check_all_revisions' not in sys.argv - if latest_revision_only: - log.debug( 'Testing is restricted to the latest downloadable revision in this test run.' ) - repository_dicts = [] - parts = [ 'repository_revisions' ] - # We'll filter out deprecated repositories from testing since testing them is necessary only if reproducibility - # is guaranteed and we currently do not guarantee reproducibility. - params = dict( do_not_test='false', - downloadable='true', - includes_tools='true', - malicious='false', - missing_test_components='false', - skip_tool_test='false' ) - api_url = get_api_url( base=tool_shed_url, parts=parts, params=params ) - baseline_repository_dicts, error_message = json_from_url( api_url ) - if error_message: - return None, error_message - for baseline_repository_dict in baseline_repository_dicts: - # We need to get some details from the tool shed API, such as repository name and owner, to pass on to the - # module that will generate the install methods. - repository_dict, error_message = get_repository_dict( galaxy_tool_shed_url, baseline_repository_dict ) - if error_message: - log.debug( 'Error getting additional details from the API: %s' % str( error_message ) ) - else: - # Don't test deprecated repositories since testing them is necessary only if reproducibility is guaranteed - # and we are not currently guaranteeing reproducibility. - deprecated = asbool( repository_dict.get( 'deprecated', False ) ) - if not deprecated: - # Don't test empty repositories. - changeset_revision = baseline_repository_dict[ 'changeset_revision' ] - if changeset_revision != suc.INITIAL_CHANGELOG_HASH: - # Merge the dictionary returned from /api/repository_revisions with the detailed repository_dict and - # append it to the list of repository_dicts to install and test. - if latest_revision_only: - latest_revision = repository_dict[ 'latest_revision' ] - if changeset_revision == latest_revision: - repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) ) - else: - repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) ) - if testing_single_repository_dict: - tsr_name = testing_single_repository_dict[ 'name' ] - tsr_owner = testing_single_repository_dict[ 'owner' ] - tsr_changeset_revision = testing_single_repository_dict[ 'changeset_revision' ] - log.debug( 'Testing single repository with name %s and owner %s.' % ( str( tsr_name ), str( tsr_owner ) ) ) - for repository_to_install in repository_dicts: - rti_name = repository_to_install[ 'name' ] - rti_owner = repository_to_install[ 'owner' ] - rti_changeset_revision = repository_to_install[ 'changeset_revision' ] - if rti_name == tsr_name and rti_owner == tsr_owner: - if tsr_changeset_revision is None: - return [ repository_to_install ], error_message - else: - if tsr_changeset_revision == rti_changeset_revision: - return repository_dicts, error_message - return repository_dicts, error_message - # Get a list of repositories to test from the tool shed specified in the GALAXY_INSTALL_TEST_TOOL_SHED_URL - # environment variable. - log.debug( "The Tool Shed's API url...\n%s" % str( api_url ) ) - log.debug( "...retrieved %d repository revisions for testing." % len( repository_dicts ) ) - log.debug( "Repository revisions for testing:" ) - for repository_dict in repository_dicts: - name = str( repository_dict.get( 'name', None ) ) - owner = str( repository_dict.get( 'owner', None ) ) - changeset_revision = str( repository_dict.get( 'changeset_revision', None ) ) - log.debug( "Revision %s of repository %s owned by %s" % ( changeset_revision, name, owner ) ) - return repository_dicts, error_message - -def get_static_settings(): - """ - Return a dictionary of the settings necessary for a Galaxy application to be wrapped in the static - middleware. This mainly consists of the file system locations of url-mapped static resources. - """ - cwd = os.getcwd() - static_dir = os.path.join( cwd, 'static' ) - #TODO: these should be copied from universe_wsgi.ini - #TODO: static_enabled needed here? - return dict( static_enabled = True, - static_cache_time = 360, - static_dir = static_dir, - static_images_dir = os.path.join( static_dir, 'images', '' ), - static_favicon_dir = os.path.join( static_dir, 'favicon.ico' ), - static_scripts_dir = os.path.join( static_dir, 'scripts', '' ), - static_style_dir = os.path.join( static_dir, 'june_2007_style', 'blue' ), - static_robots_txt = os.path.join( static_dir, 'robots.txt' ) ) - -def get_tool_info_from_test_id( test_id ): - """ - Test IDs come in the form test_tool_number - (functional.test_toolbox.TestForTool_toolshed_url/repos/owner/repository_name/tool_id/tool_version) - We want the tool ID and tool version. - """ - parts = test_id.replace( ')', '' ).split( '/' ) - tool_version = parts[ -1 ] - tool_id = parts[ -2 ] - return tool_id, tool_version - -def get_tool_test_results_dicts( tool_shed_url, encoded_repository_metadata_id ): - """ - Return the list of dictionaries contained in the Tool Shed's repository_metadata.tool_test_results - column via the Tool Shed API. - """ - error_message = '' - parts = [ 'api', 'repository_revisions', encoded_repository_metadata_id ] - api_url = get_api_url( base=tool_shed_url, parts=parts ) - repository_metadata, error_message = json_from_url( api_url ) - if error_message: - return None, error_message - # The tool_test_results used to be stored as a single dictionary rather than a list, but we currently - # return a list. - tool_test_results = listify( repository_metadata.get( 'tool_test_results', [] ) ) - return tool_test_results, error_message - -def get_webapp_global_conf(): - """Return the global_conf dictionary sent as the first argument to app_factory.""" - global_conf = {} - if STATIC_ENABLED: - global_conf.update( get_static_settings() ) - return global_conf - -def handle_missing_dependencies( app, repository, missing_tool_dependencies, repository_dict, - tool_test_results_dicts, tool_test_results_dict, results_dict ): - """Handle missing repository or tool dependencies for an installed repository.""" - # If a tool dependency fails to install correctly, this should be considered an installation error, - # and functional tests should be skipped, since the tool dependency needs to be correctly installed - # for the test to be considered reliable. - log.debug( 'The following dependencies of this repository are missing, so skipping functional tests.' ) - # In keeping with the standard display layout, add the error message to the dict for each tool individually. - for dependency in repository.missing_tool_dependencies: - name = str( dependency.name ) - type = str( dependency.type ) - version = str( dependency.version ) - error_message = unicodify( dependency.error_message ) - log.debug( 'Missing tool dependency %s of type %s version %s: %s' % ( name, type, version, error_message ) ) - test_result = dict( type=dependency.type, - name=dependency.name, - version=dependency.version, - error_message=dependency.error_message ) - tool_test_results_dict[ 'installation_errors' ][ 'tool_dependencies' ].append( test_result ) - for dependency in repository.missing_repository_dependencies: - name = str( dependency.name ) - owner = str( dependency.owner ) - changeset_revision = str( dependency.changeset_revision ) - error_message = unicodify( dependency.error_message ) - log.debug( 'Missing repository dependency %s changeset revision %s owned by %s: %s' % \ - ( name, changeset_revision, owner, error_message ) ) - test_result = dict( tool_shed=dependency.tool_shed, - name=dependency.name, - owner=dependency.owner, - changeset_revision=dependency.changeset_revision, - error_message=dependency.error_message ) - tool_test_results_dict[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) - # Record the status of this repository in the tool shed. - params = dict( tools_functionally_correct=False, - do_not_test=False, - test_install_error=True ) - # TODO: do something useful with response_dict - response_dict = register_test_result( galaxy_tool_shed_url, - tool_test_results_dicts, - tool_test_results_dict, - repository_dict, - params ) - # Since this repository is missing components, we do not want to test it, so deactivate it or uninstall it. - # The deactivate flag is set to True if the environment variable GALAXY_INSTALL_TEST_KEEP_TOOL_DEPENDENCIES - # is set to 'true'. - deactivate = asbool( os.environ.get( 'GALAXY_INSTALL_TEST_KEEP_TOOL_DEPENDENCIES', False ) ) - if deactivate: - for missing_tool_dependency in missing_tool_dependencies: - uninstall_tool_dependency( app, missing_tool_dependency ) - # We are deactivating this repository and all of its repository dependencies. - deactivate_repository( app, repository_dict ) - else: - # We are uninstalling this repository and all of its repository dependencies. - uninstall_repository_and_repository_dependencies( app, repository_dict ) - results_dict[ 'repositories_failed_install' ].append( dict( name=str( repository.name ), - owner=str( repository.owner ), - changeset_revision=str( repository.changeset_revision ) ) ) - # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. - test_toolbox.toolbox = app.toolbox - return results_dict - -def initialize_results_dict(): - # Initialize a dictionary for the summary that will be printed to stdout. - results_dict = {} - results_dict[ 'total_repositories_tested' ] = 0 - results_dict[ 'all_tests_passed' ] = [] - results_dict[ 'at_least_one_test_failed' ] = [] - results_dict[ 'repositories_failed_install' ] = [] - return results_dict - -def install_repository( app, repository_dict ): - """Install a repository defined by the received repository_dict from the tool shed into Galaxy.""" - name = str( repository_dict[ 'name' ] ) - owner = str( repository_dict[ 'owner' ] ) - changeset_revision = str( repository_dict[ 'changeset_revision' ] ) - error_message = '' - repository = None - log.debug( "Installing revision %s of repository %s owned by %s." % ( changeset_revision, name, owner ) ) - # Explicitly clear tests from twill's test environment. - remove_generated_tests( app ) - # Use the repository information dictionary to generate an install method that will install the repository into the - # embedded Galaxy application, with tool dependencies and repository dependencies, if any. - test_install_repositories.generate_install_method( repository_dict ) - # Configure nose to run the install method as a test. - test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) - test_config.configure( sys.argv ) - # Run the configured install method as a test. This method uses the embedded Galaxy application's web interface to - # install the specified repository with tool and repository dependencies also selected for installation. - result, _ = run_tests( test_config ) - try: - repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) - except Exception, e: - error_message = 'Error getting revision %s of repository %s owned by %s: %s' % ( changeset_revision, name, owner, str( e ) ) - log.exception( error_message ) - return repository, error_message - -def install_and_test_repositories( app, galaxy_shed_tools_dict, galaxy_shed_tool_conf_file ): - results_dict = initialize_results_dict() - error_message = '' - # Allow the option to keep or delete tool dependencies when a repository has been tested. - deactivate = asbool( os.environ.get( 'GALAXY_INSTALL_TEST_KEEP_TOOL_DEPENDENCIES', False ) ) - # Initialize a dictionary for the summary that will be printed to stdout. - total_repositories_tested = results_dict[ 'total_repositories_tested' ] - repositories_to_install, error_message = get_repositories_to_install( galaxy_tool_shed_url ) - if error_message: - return None, error_message - # Handle repositories not to be tested. - if os.path.exists( exclude_list_file ): - # Entries in the exclude_list look something like this. - # { 'reason': The default reason or the reason specified in this section, - # 'repositories': - # [( name, owner, changeset revision if changeset revision else None ), - # ( name, owner, changeset revision if changeset revision else None )] } - # If changeset revision is None, that means the entire repository is excluded from testing, otherwise only the specified - # revision should be skipped. - # We are testing deprecated repositories because it is possible that a deprecated repository contains valid tools that - # someone has previously installed. Deleted repositories have never been installed, so should not be tested. If they are - # undeleted, this script will then test them the next time it runs. We don't need to check if a repository has been deleted - # here because our call to the Tool Shed API filters by downloadable='true', in which case deleted will always be False. - log.debug( 'Loading the list of repositories excluded from testing from the file %s...', exclude_list_file ) - exclude_list = parse_exclude_list( exclude_list_file ) - else: - exclude_list = [] - # Generate a test method that will use Twill to install each repository into the embedded Galaxy application that was - # started up, installing repository and tool dependencies. Upon successful installation, generate a test case for each - # functional test defined for each tool in the repository and execute the test cases. Record the result of the tests. - # The traceback and captured output of the tool that was run will be recored for test failures. After all tests have - # completed, the repository is uninstalled, so test cases don't interfere with the next repository's functional tests. - for repository_dict in repositories_to_install: - # Each repository_dict looks something like: - # { "changeset_revision": "13fa22a258b5", - # "contents_url": "/api/repositories/529fd61ab1c6cc36/contents", - # "deleted": false, - # "deprecated": false, - # "description": "Convert column case.", - # "downloadable": true, - # "id": "529fd61ab1c6cc36", - # "long_description": "This tool takes the specified columns and converts them to uppercase or lowercase.", - # "malicious": false, - # "name": "change_case", - # "owner": "test", - # "private": false, - # "repository_id": "529fd61ab1c6cc36", - # "times_downloaded": 0, - # "tool_shed_url": "http://toolshed.local:10001", - # "url": "/api/repository_revisions/529fd61ab1c6cc36", - # "user_id": "529fd61ab1c6cc36" } - encoded_repository_metadata_id = repository_dict.get( 'id', None ) - # Add the URL for the tool shed we're installing from, so the automated installation methods go to the right place. - repository_dict[ 'tool_shed_url' ] = galaxy_tool_shed_url - # Get the name and owner out of the repository info dict. - name = str( repository_dict[ 'name' ] ) - owner = str( repository_dict[ 'owner' ] ) - changeset_revision = str( repository_dict[ 'changeset_revision' ] ) - log.debug( "Processing revision %s of repository %s owned by %s..." % ( changeset_revision, name, owner ) ) - repository_identifier_dict = dict( name=name, owner=owner, changeset_revision=changeset_revision ) - # Retrieve the stored list of tool_test_results_dicts. - tool_test_results_dicts, error_message = get_tool_test_results_dicts( galaxy_tool_shed_url, encoded_repository_metadata_id ) - if error_message: - log.debug( error_message ) - else: - if tool_test_results_dicts: - # Inspect the tool_test_results_dict for the last test run to make sure it contains only a test_environment - # entry. If it contains more entries, then the script ~/tool_shed/api/check_repositories_for_functional_tests.py - # was not executed in preparation for this script's execution, so we'll just create an empty dictionary. - tool_test_results_dict = tool_test_results_dicts[ 0 ] - if len( tool_test_results_dict ) <= 1: - # We can re-use the mostly empty tool_test_results_dict for this run because it is either empty or it contains only - # a test_environment entry. If we use it we need to temporarily eliminate it from the list of tool_test_results_dicts - # since it will be re-inserted later. - tool_test_results_dict = tool_test_results_dicts.pop( 0 ) - elif len( tool_test_results_dict ) == 2 and \ - 'test_environment' in tool_test_results_dict and 'missing_test_components' in tool_test_results_dict: - # We can re-use tool_test_results_dict if its only entries are "test_environment" and "missing_test_components". - # In this case, some tools are missing tests components while others are not. - tool_test_results_dict = tool_test_results_dicts.pop( 0 ) - else: - # The latest tool_test_results_dict has been populated with the results of a test run, so it cannot be used. - tool_test_results_dict = {} - else: - # Create a new dictionary for this test test run, - tool_test_results_dict = {} - # See if this repository should be skipped for any reason. - this_repository_is_in_the_exclude_list = False - skip_reason = None - for exclude_dict in exclude_list: - reason = exclude_dict[ 'reason' ] - exclude_repositories = exclude_dict[ 'repositories' ] - if ( name, owner, changeset_revision ) in exclude_repositories or ( name, owner, None ) in exclude_repositories: - this_repository_is_in_the_exclude_list = True - skip_reason = reason - break - if this_repository_is_in_the_exclude_list: - tool_test_results_dict[ 'not_tested' ] = dict( reason=skip_reason ) - params = dict( tools_functionally_correct=False, - do_not_test=False ) - # TODO: do something useful with response_dict - response_dict = register_test_result( galaxy_tool_shed_url, tool_test_results_dicts, tool_test_results_dict, repository_dict, params ) - log.debug( "Not testing revision %s of repository %s owned by %s because it is in the exclude list for this test run." % \ - ( changeset_revision, name, owner ) ) - else: - test_environment_dict = tool_test_results_dict.get( 'test_environment', {} ) - if len( test_environment_dict ) == 0: - # Set information about the tool shed to nothing since we cannot currently determine it from here. - # We could eventually add an API method... - test_environment_dict = dict( tool_shed_database_version='', - tool_shed_mercurial_version='', - tool_shed_revision='' ) - # Add the current time as the approximate time that this test run occurs. A similar value will also be - # set to the repository_metadata.time_last_tested column, but we also store it here because the Tool Shed - # may be configured to store multiple test run results, so each must be associated with a time stamp. - now = time.strftime( "%Y-%m-%d %H:%M:%S" ) - # Add information about the current platform. - test_environment_dict[ 'time_tested' ] = now - test_environment_dict[ 'python_version' ] = platform.python_version() - test_environment_dict[ 'architecture' ] = platform.machine() - operating_system, hostname, operating_system_version, uname, arch, processor = platform.uname() - test_environment_dict[ 'system' ] = '%s %s' % ( operating_system, operating_system_version ) - # Add information about the current Galaxy environment. - test_environment_dict[ 'galaxy_database_version' ] = get_database_version( app ) - test_environment_dict[ 'galaxy_revision' ] = get_repository_current_revision( os.getcwd() ) - # Initialize and populate the tool_test_results_dict. - tool_test_results_dict[ 'test_environment' ] = test_environment_dict - tool_test_results_dict[ 'passed_tests' ] = [] - tool_test_results_dict[ 'failed_tests' ] = [] - tool_test_results_dict[ 'installation_errors' ] = dict( current_repository=[], repository_dependencies=[], tool_dependencies=[] ) - # Proceed with installing repositories and testing contained tools. - repository, error_message = install_repository( app, repository_dict ) - if error_message: - tool_test_results_dict[ 'installation_errors' ][ 'current_repository' ] = error_message - # Even if the repository failed to install, execute the uninstall method, in case a dependency did succeed. - log.debug( 'Attempting to uninstall repository %s owned by %s.' % ( name, owner ) ) - try: - repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) - except Exception, e: - error_message = 'Unable to find installed repository %s owned by %s: %s.' % ( name, owner, str( e ) ) - log.exception( error_message ) - test_result = dict( tool_shed=galaxy_tool_shed_url, - name=name, - owner=owner, - changeset_revision=changeset_revision, - error_message=error_message ) - tool_test_results_dict[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) - params = dict( tools_functionally_correct=False, - test_install_error=True, - do_not_test=False ) - # TODO: do something useful with response_dict - response_dict = register_test_result( galaxy_tool_shed_url, - tool_test_results_dicts, - tool_test_results_dict, - repository_dict, - params ) - try: - if deactivate: - # We are deactivating this repository and all of its repository dependencies. - deactivate_repository( app, repository_dict ) - else: - # We are uninstalling this repository and all of its repository dependencies. - uninstall_repository_and_repository_dependencies( app, repository_dict ) - except: - log.exception( 'Encountered error attempting to deactivate or uninstall %s.', str( repository_dict[ 'name' ] ) ) - results_dict[ 'repositories_failed_install' ].append( repository_identifier_dict ) - log.debug( 'Repository %s failed to install correctly.' % str( name ) ) - else: - # Configure and run functional tests for this repository. This is equivalent to sh run_functional_tests.sh -installed - remove_install_tests() - log.debug( 'Installation of %s succeeded, running all defined functional tests.' % str( repository.name ) ) - # Generate the shed_tools_dict that specifies the location of test data contained within this repository. If the repository - # does not have a test-data directory, this will return has_test_data = False, and we will set the do_not_test flag to True, - # and the tools_functionally_correct flag to False, as well as updating tool_test_results. - file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( {} ) ) - has_test_data, shed_tools_dict = parse_tool_panel_config( galaxy_shed_tool_conf_file, - from_json_string( file( galaxy_shed_tools_dict, 'r' ).read() ) ) - # Add an empty 'missing_test_results' entry if it is missing from the tool_test_results_dict. The - # ~/tool_shed/scripts/check_repositories_for_functional_tests.py will have entered information in the - # 'missing_test_components' entry of the tool_test_results_dict dictionary for repositories that are - # missing test components. - if 'missing_test_components' not in tool_test_results_dict: - tool_test_results_dict[ 'missing_test_components' ] = [] - missing_tool_dependencies = get_missing_tool_dependencies( repository ) - if missing_tool_dependencies or repository.missing_repository_dependencies: - results_dict = handle_missing_dependencies( app, - repository, - missing_tool_dependencies, - repository_dict, - tool_test_results_dicts, - tool_test_results_dict, - results_dict ) - else: - # If the repository has a test-data directory we write the generated shed_tools_dict to a file, so the functional - # test framework can find it. - file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( shed_tools_dict ) ) - log.debug( 'Saved generated shed_tools_dict to %s\nContents: %s' % ( str( galaxy_shed_tools_dict ), str( shed_tools_dict ) ) ) - try: - results_dict = test_repository_tools( app, - repository, - repository_dict, - tool_test_results_dicts, - tool_test_results_dict, - results_dict ) - except Exception, e: - exception_message = 'Error executing tests for repository %s: %s' % ( name, str( e ) ) - log.exception( exception_message ) - tool_test_results_dict[ 'failed_tests' ].append( exception_message ) - # Record the status of this repository in the tool shed. - params = dict( tools_functionally_correct=False, - do_not_test=False, - test_install_error=False ) - # TODO: do something useful with response_dict - response_dict = register_test_result( galaxy_tool_shed_url, - tool_test_results_dicts, - tool_test_results_dict, - repository_dict, - params ) - results_dict[ 'at_least_one_test_failed' ].append( repository_identifier_dict ) - total_repositories_tested += 1 - results_dict[ 'total_repositories_tested' ] = total_repositories_tested - return results_dict, error_message - -def is_latest_downloadable_revision( url, repository_dict ): - latest_revision = get_latest_downloadable_changeset_revision( url, name=repository_dict[ 'name' ], owner=repository_dict[ 'owner' ] ) - return str( repository_dict[ 'changeset_revision' ] ) == str( latest_revision ) - -def json_from_url( url ): - error_message = '' - url_handle = urllib.urlopen( url ) - url_contents = url_handle.read() - try: - parsed_json = from_json_string( url_contents ) - except Exception, e: - error_message = str( url_contents ) - log.exception( 'Error parsing JSON data in json_from_url(): %s.' % str( e ) ) - return None, error_message - return parsed_json, error_message - -def main(): - if tool_shed_api_key is None: - # If the tool shed URL specified in any dict is not present in the tool_sheds_conf.xml, the installation will fail. - log.debug( 'Cannot proceed without a valid tool shed API key set in the enviroment variable GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY.' ) - return 1 - if galaxy_tool_shed_url is None: - log.debug( 'Cannot proceed without a valid Tool Shed base URL set in the environment variable GALAXY_INSTALL_TEST_TOOL_SHED_URL.' ) - return 1 - # ---- Configuration ------------------------------------------------------ - galaxy_test_host = os.environ.get( 'GALAXY_INSTALL_TEST_HOST', default_galaxy_test_host ) - # Set the GALAXY_INSTALL_TEST_HOST variable so that Twill will have the Galaxy url to which to - # install repositories. - os.environ[ 'GALAXY_INSTALL_TEST_HOST' ] = galaxy_test_host - # Set the GALAXY_TEST_HOST environment variable so that the toolbox tests will have the Galaxy url - # on which to to run tool functional tests. - os.environ[ 'GALAXY_TEST_HOST' ] = galaxy_test_host - galaxy_test_port = os.environ.get( 'GALAXY_INSTALL_TEST_PORT', str( default_galaxy_test_port_max ) ) - os.environ[ 'GALAXY_TEST_PORT' ] = galaxy_test_port - tool_path = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_PATH', 'tools' ) - if 'HTTP_ACCEPT_LANGUAGE' not in os.environ: - os.environ[ 'HTTP_ACCEPT_LANGUAGE' ] = default_galaxy_locales - galaxy_test_file_dir = os.environ.get( 'GALAXY_INSTALL_TEST_FILE_DIR', default_galaxy_test_file_dir ) - if not os.path.isabs( galaxy_test_file_dir ): - galaxy_test_file_dir = os.path.abspath( galaxy_test_file_dir ) - use_distributed_object_store = os.environ.get( 'GALAXY_INSTALL_TEST_USE_DISTRIBUTED_OBJECT_STORE', False ) - if not os.path.isdir( galaxy_test_tmp_dir ): - os.mkdir( galaxy_test_tmp_dir ) - # Set up the configuration files for the Galaxy instance. - shed_tool_data_table_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_SHED_TOOL_DATA_TABLE_CONF', - os.path.join( galaxy_test_tmp_dir, 'test_shed_tool_data_table_conf.xml' ) ) - galaxy_tool_data_table_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_DATA_TABLE_CONF', - tool_data_table_conf ) - galaxy_tool_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_CONF', - os.path.join( galaxy_test_tmp_dir, 'test_tool_conf.xml' ) ) - galaxy_job_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_JOB_CONF', - os.path.join( galaxy_test_tmp_dir, 'test_job_conf.xml' ) ) - galaxy_shed_tool_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_SHED_TOOL_CONF', - os.path.join( galaxy_test_tmp_dir, 'test_shed_tool_conf.xml' ) ) - galaxy_migrated_tool_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_MIGRATED_TOOL_CONF', - os.path.join( galaxy_test_tmp_dir, 'test_migrated_tool_conf.xml' ) ) - galaxy_tool_sheds_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF', - os.path.join( galaxy_test_tmp_dir, 'test_tool_sheds_conf.xml' ) ) - galaxy_shed_tools_dict = os.environ.get( 'GALAXY_INSTALL_TEST_SHED_TOOL_DICT_FILE', - os.path.join( galaxy_test_tmp_dir, 'shed_tool_dict' ) ) - file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( {} ) ) - # Set the GALAXY_TOOL_SHED_TEST_FILE environment variable to the path of the shed_tools_dict file so that - # test.base.twilltestcase.setUp will find and parse it properly. - os.environ[ 'GALAXY_TOOL_SHED_TEST_FILE' ] = galaxy_shed_tools_dict - if 'GALAXY_INSTALL_TEST_TOOL_DATA_PATH' in os.environ: - tool_data_path = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_DATA_PATH' ) - else: - tool_data_path = tempfile.mkdtemp( dir=galaxy_test_tmp_dir ) - os.environ[ 'GALAXY_INSTALL_TEST_TOOL_DATA_PATH' ] = tool_data_path - # Configure the database connection and path. - if 'GALAXY_INSTALL_TEST_DBPATH' in os.environ: - galaxy_db_path = os.environ[ 'GALAXY_INSTALL_TEST_DBPATH' ] - else: - tempdir = tempfile.mkdtemp( dir=galaxy_test_tmp_dir ) - galaxy_db_path = os.path.join( tempdir, 'database' ) - # Configure the paths Galaxy needs to install and test tools. - galaxy_file_path = os.path.join( galaxy_db_path, 'files' ) - new_repos_path = tempfile.mkdtemp( dir=galaxy_test_tmp_dir ) - galaxy_tempfiles = tempfile.mkdtemp( dir=galaxy_test_tmp_dir ) - galaxy_shed_tool_path = tempfile.mkdtemp( dir=galaxy_test_tmp_dir, prefix='shed_tools' ) - galaxy_migrated_tool_path = tempfile.mkdtemp( dir=galaxy_test_tmp_dir ) - # Set up the tool dependency path for the Galaxy instance. - tool_dependency_dir = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR', None ) - if tool_dependency_dir is None: - tool_dependency_dir = tempfile.mkdtemp( dir=galaxy_test_tmp_dir ) - os.environ[ 'GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR' ] = tool_dependency_dir - os.environ[ 'GALAXY_TOOL_DEPENDENCY_DIR' ] = tool_dependency_dir - if 'GALAXY_INSTALL_TEST_DBURI' in os.environ: - database_connection = os.environ[ 'GALAXY_INSTALL_TEST_DBURI' ] - else: - database_connection = 'sqlite:///' + os.path.join( galaxy_db_path, 'install_and_test_repositories.sqlite' ) - kwargs = {} - for dir in [ galaxy_test_tmp_dir ]: - try: - os.makedirs( dir ) - except OSError: - pass - print "Database connection: ", database_connection - # Generate the shed_tool_data_table_conf.xml file. - file( shed_tool_data_table_conf_file, 'w' ).write( tool_data_table_conf_xml_template ) - os.environ[ 'GALAXY_INSTALL_TEST_SHED_TOOL_DATA_TABLE_CONF' ] = shed_tool_data_table_conf_file - # ---- Start up a Galaxy instance ------------------------------------------------------ - # Generate the tool_conf.xml file. - file( galaxy_tool_conf_file, 'w' ).write( tool_conf_xml ) - # Generate the job_conf.xml file. - file( galaxy_job_conf_file, 'w' ).write( job_conf_xml ) - # Generate the tool_sheds_conf.xml file, but only if a the user has not specified an existing one in the environment. - if 'GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF' not in os.environ: - file( galaxy_tool_sheds_conf_file, 'w' ).write( tool_sheds_conf_xml ) - # Generate the shed_tool_conf.xml file. - tool_conf_template_parser = string.Template( shed_tool_conf_xml_template ) - shed_tool_conf_xml = tool_conf_template_parser.safe_substitute( shed_tool_path=galaxy_shed_tool_path ) - file( galaxy_shed_tool_conf_file, 'w' ).write( shed_tool_conf_xml ) - os.environ[ 'GALAXY_INSTALL_TEST_SHED_TOOL_CONF' ] = galaxy_shed_tool_conf_file - # Generate the migrated_tool_conf.xml file. - migrated_tool_conf_xml = tool_conf_template_parser.safe_substitute( shed_tool_path=galaxy_migrated_tool_path ) - file( galaxy_migrated_tool_conf_file, 'w' ).write( migrated_tool_conf_xml ) - # Write the embedded web application's specific configuration to a temporary file. This is necessary in order for - # the external metadata script to find the right datasets. - kwargs = dict( admin_users = 'test(a)bx.psu.edu', - master_api_key = default_galaxy_master_api_key, - allow_user_creation = True, - allow_user_deletion = True, - allow_library_path_paste = True, - database_connection = database_connection, - datatype_converters_config_file = "datatype_converters_conf.xml.sample", - file_path = galaxy_file_path, - id_secret = galaxy_encode_secret, - job_config_file = galaxy_job_conf_file, - job_queue_workers = 5, - log_destination = "stdout", - migrated_tools_config = galaxy_migrated_tool_conf_file, - new_file_path = galaxy_tempfiles, - running_functional_tests = True, - shed_tool_data_table_config = shed_tool_data_table_conf_file, - shed_tool_path = galaxy_shed_tool_path, - template_path = "templates", - tool_config_file = ','.join( [ galaxy_tool_conf_file, galaxy_shed_tool_conf_file ] ), - tool_data_path = tool_data_path, - tool_data_table_config_path = galaxy_tool_data_table_conf_file, - tool_dependency_dir = tool_dependency_dir, - tool_path = tool_path, - tool_parse_help = False, - tool_sheds_config_file = galaxy_tool_sheds_conf_file, - update_integrated_tool_panel = False, - use_heartbeat = False ) - galaxy_config_file = os.environ.get( 'GALAXY_INSTALL_TEST_INI_FILE', None ) - # If the user has passed in a path for the .ini file, do not overwrite it. - if not galaxy_config_file: - galaxy_config_file = os.path.join( galaxy_test_tmp_dir, 'install_test_tool_shed_repositories_wsgi.ini' ) - config_items = [] - for label in kwargs: - config_tuple = label, kwargs[ label ] - config_items.append( config_tuple ) - # Write a temporary file, based on universe_wsgi.ini.sample, using the configuration options defined above. - generate_config_file( 'universe_wsgi.ini.sample', galaxy_config_file, config_items ) - kwargs[ 'tool_config_file' ] = [ galaxy_tool_conf_file, galaxy_shed_tool_conf_file ] - # Set the global_conf[ '__file__' ] option to the location of the temporary .ini file, which gets passed to set_metadata.sh. - kwargs[ 'global_conf' ] = get_webapp_global_conf() - kwargs[ 'global_conf' ][ '__file__' ] = galaxy_config_file - # ---- Build Galaxy Application -------------------------------------------------- - if not database_connection.startswith( 'sqlite://' ): - kwargs[ 'database_engine_option_max_overflow' ] = '20' - kwargs[ 'database_engine_option_pool_size' ] = '10' - kwargs[ 'config_file' ] = galaxy_config_file - app = UniverseApplication( **kwargs ) - database_contexts.galaxy_context = app.model.context - database_contexts.install_context = app.install_model.context - global test_toolbox - import functional.test_toolbox as imported_test_toolbox - test_toolbox = imported_test_toolbox - - log.debug( "Embedded Galaxy application started..." ) - # ---- Run galaxy webserver ------------------------------------------------------ - server = None - global_conf = get_webapp_global_conf() - global_conf[ 'database_file' ] = database_connection - webapp = buildapp.app_factory( global_conf, - use_translogger=False, - static_enabled=STATIC_ENABLED, - app=app ) - # Serve the app on a specified or random port. - if galaxy_test_port is not None: - server = httpserver.serve( webapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False ) - else: - random.seed() - for i in range( 0, 9 ): - try: - galaxy_test_port = str( random.randint( default_galaxy_test_port_min, default_galaxy_test_port_max ) ) - log.debug( "Attempting to serve app on randomly chosen port: %s", galaxy_test_port ) - server = httpserver.serve( webapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False ) - break - except socket.error, e: - if e[0] == 98: - continue - raise - else: - raise Exception( "Unable to open a port between %s and %s to start Galaxy server" % \ - ( default_galaxy_test_port_min, default_galaxy_test_port_max ) ) - os.environ[ 'GALAXY_INSTALL_TEST_PORT' ] = galaxy_test_port - # Start the server. - t = threading.Thread( target=server.serve_forever ) - t.start() - # Test if the server is up. - for i in range( 10 ): - # Directly test the app, not the proxy. - conn = httplib.HTTPConnection( galaxy_test_host, galaxy_test_port ) - conn.request( "GET", "/" ) - if conn.getresponse().status == 200: - break - time.sleep( 0.1 ) - else: - raise Exception( "Test HTTP server did not return '200 OK' after 10 tries" ) - log.debug( "Embedded galaxy web server started..." ) - log.debug( "The embedded Galaxy application is running on %s:%s" % ( str( galaxy_test_host ), str( galaxy_test_port ) ) ) - log.debug( "Repositories will be installed from the tool shed at %s" % str( galaxy_tool_shed_url ) ) - success = False - # If a tool_data_table_conf.test.xml file was found, add the entries from it into the app's tool data tables. - if additional_tool_data_tables: - app.tool_data_tables.add_new_entries_from_config_file( config_filename=additional_tool_data_tables, - tool_data_path=additional_tool_data_path, - shed_tool_data_table_config=None, - persist=False ) - now = time.strftime( "%Y-%m-%d %H:%M:%S" ) - print "####################################################################################" - print "# %s - running repository installation and testing script." % now - print "####################################################################################" - results_dict, error_message = install_and_test_repositories( app, galaxy_shed_tools_dict, galaxy_shed_tool_conf_file ) - if error_message: - log.debug( error_message ) - else: - total_repositories_tested = results_dict[ 'total_repositories_tested' ] - all_tests_passed = results_dict[ 'all_tests_passed' ] - at_least_one_test_failed = results_dict[ 'at_least_one_test_failed' ] - repositories_failed_install = results_dict[ 'repositories_failed_install' ] - now = time.strftime( "%Y-%m-%d %H:%M:%S" ) - print "####################################################################################" - print "# %s - repository installation and testing script completed." % now - print "# Repository revisions tested: %s" % str( total_repositories_tested ) - if not can_update_tool_shed: - print "# This run will not update the Tool Shed database." - if total_repositories_tested > 0: - if all_tests_passed: - print '# ----------------------------------------------------------------------------------' - print "# %d repositories successfully passed all functional tests:" % len( all_tests_passed ) - show_summary_output( all_tests_passed ) - if at_least_one_test_failed: - print '# ----------------------------------------------------------------------------------' - print "# %d repositories failed at least 1 functional test:" % len( at_least_one_test_failed ) - show_summary_output( at_least_one_test_failed ) - if repositories_failed_install: - # Set success to False so that the return code will not be 0. - success = False - print '# ----------------------------------------------------------------------------------' - print "# %d repositories have installation errors:" % len( repositories_failed_install ) - show_summary_output( repositories_failed_install ) - else: - success = True - else: - success = True - print "####################################################################################" - log.debug( "Shutting down..." ) - # Gracefully shut down the embedded web server and UniverseApplication. - if server: - log.debug( "Shutting down embedded galaxy web server..." ) - server.server_close() - server = None - log.debug( "Embedded galaxy server stopped..." ) - if app: - log.debug( "Shutting down galaxy application..." ) - app.shutdown() - app = None - log.debug( "Embedded galaxy application stopped..." ) - # Clean up test files unless otherwise specified. - if 'GALAXY_INSTALL_TEST_NO_CLEANUP' not in os.environ: - for dir in [ galaxy_test_tmp_dir ]: - if os.path.exists( dir ): - try: - shutil.rmtree( dir ) - log.debug( "Cleaned up temporary files in %s", str( dir ) ) - except: - pass - else: - log.debug( 'GALAXY_INSTALL_TEST_NO_CLEANUP set, not cleaning up.' ) - # Normally the value of 'success' would determine whether this test suite is marked as passed or failed - # in the automated buildbot framework. However, due to the procedure used here we only want to report - # failure if a repository fails to install correctly, so we have overriden the value of 'success' here - # based on what actions the script has executed. - if success: - return 0 - else: - return 1 - -def parse_exclude_list( xml_filename ): - """Return a list of repositories to exclude from testing.""" - # This method should return a list with the following structure: - # [{ 'reason': The default reason or the reason specified in this section, - # 'repositories': [( name, owner, changeset revision if changeset revision else None ), - # ( name, owner, changeset revision if changeset revision else None )]}] - exclude_list = [] - exclude_verbose = [] - xml_tree, error_message = parse_xml( xml_filename ) - if error_message: - log.debug( 'The xml document %s defining the exclude list is invalid, so no repositories will be excluded from testing: %s' % \ - ( str( xml_filename ), str( error_message ) ) ) - return exclude_list - tool_sheds = xml_tree.findall( 'repositories' ) - xml_element = [] - exclude_count = 0 - for tool_shed in tool_sheds: - if galaxy_tool_shed_url != tool_shed.attrib[ 'tool_shed' ]: - continue - else: - xml_element = tool_shed - for reason_section in xml_element: - reason_text = reason_section.find( 'text' ).text - repositories = reason_section.findall( 'repository' ) - exclude_dict = dict( reason=reason_text, repositories=[] ) - for repository in repositories: - repository_tuple = get_repository_tuple_from_elem( repository ) - if repository_tuple not in exclude_dict[ 'repositories' ]: - exclude_verbose.append( repository_tuple ) - exclude_count += 1 - exclude_dict[ 'repositories' ].append( repository_tuple ) - exclude_list.append( exclude_dict ) - log.debug( 'The xml document %s containing the exclude list defines the following %s repositories to be excluded from testing...' % \ - ( str( xml_filename ), str( exclude_count ) ) ) - #if '-list_repositories' in sys.argv: - for name, owner, changeset_revision in exclude_verbose: - if changeset_revision: - log.debug( 'Repository %s owned by %s, changeset revision %s.' % ( str( name ), str( owner ), str( changeset_revision ) ) ) - else: - log.debug( 'Repository %s owned by %s, all revisions.' % ( str( name ), str( owner ) ) ) - return exclude_list - -def register_test_result( url, tool_test_results_dicts, tool_test_results_dict, repository_dict, params ): - """ - Update the repository metadata tool_test_results and appropriate flags using the Tool SHed API. This method - updates tool_test_results with the relevant data, sets the do_not_test and tools_functionally correct flags - to the appropriate values and updates the time_last_tested field to the value of the received time_tested. - """ - if can_update_tool_shed: - metadata_revision_id = repository_dict.get( 'id', None ) - if metadata_revision_id is not None: - tool_test_results_dicts.insert( 0, tool_test_results_dict ) - params[ 'tool_test_results' ] = tool_test_results_dicts - # Set the time_last_tested entry so that the repository_metadata.time_last_tested will be set in the tool shed. - params[ 'time_last_tested' ] = 'This entry will result in this value being set via the Tool Shed API.' - url = '%s' % ( suc.url_join( galaxy_tool_shed_url,'api', 'repository_revisions', str( metadata_revision_id ) ) ) - try: - return update( tool_shed_api_key, url, params, return_formatted=False ) - except Exception, e: - log.exception( 'Error attempting to register test results: %s' % str( e ) ) - return {} - else: - return {} - -def remove_generated_tests( app ): - """ - Delete any configured tool functional tests from the test_toolbox.__dict__, otherwise nose will find them - and try to re-run the tests after uninstalling the repository, which will cause false failure reports, - since the test data has been deleted from disk by now. - """ - tests_to_delete = [] - tools_to_delete = [] - global test_toolbox - for key in test_toolbox.__dict__: - if key.startswith( 'TestForTool_' ): - log.debug( 'Tool test found in test_toolbox, deleting: %s' % str( key ) ) - # We can't delete this test just yet, we're still iterating over __dict__. - tests_to_delete.append( key ) - tool_id = key.replace( 'TestForTool_', '' ) - for tool in app.toolbox.tools_by_id: - if tool.replace( '_', ' ' ) == tool_id.replace( '_', ' ' ): - tools_to_delete.append( tool ) - for key in tests_to_delete: - # Now delete the tests found in the previous loop. - del test_toolbox.__dict__[ key ] - for tool in tools_to_delete: - del app.toolbox.tools_by_id[ tool ] - -def remove_install_tests(): - """ - Delete any configured repository installation tests from the test_toolbox.__dict__, otherwise nose will find them - and try to install the repository again while running tool functional tests. - """ - tests_to_delete = [] - global test_toolbox - # Push all the toolbox tests to module level - for key in test_install_repositories.__dict__: - if key.startswith( 'TestInstallRepository_' ): - log.debug( 'Repository installation process found, deleting: %s' % str( key ) ) - # We can't delete this test just yet, we're still iterating over __dict__. - tests_to_delete.append( key ) - for key in tests_to_delete: - # Now delete the tests found in the previous loop. - del test_install_repositories.__dict__[ key ] - -def run_tests( test_config ): - loader = nose.loader.TestLoader( config=test_config ) - test_config.plugins.addPlugin( ReportResults() ) - plug_loader = test_config.plugins.prepareTestLoader( loader ) - if plug_loader is not None: - loader = plug_loader - tests = loader.loadTestsFromNames( test_config.testNames ) - test_runner = nose.core.TextTestRunner( stream=test_config.stream, - verbosity=test_config.verbosity, - config=test_config ) - plug_runner = test_config.plugins.prepareTestRunner( test_runner ) - if plug_runner is not None: - test_runner = plug_runner - result = test_runner.run( tests ) - return result, test_config.plugins._plugins - -def show_summary_output( repository_dicts ): - # Group summary display by repository owner. - repository_dicts_by_owner = {} - for repository_dict in repository_dicts: - name = str( repository_dict[ 'name' ] ) - owner = str( repository_dict[ 'owner' ] ) - changeset_revision = str( repository_dict[ 'changeset_revision' ] ) - if owner in repository_dicts_by_owner: - repository_dicts_by_owner[ owner ].append( repository_dict ) - else: - repository_dicts_by_owner[ owner ] = [ repository_dict ] - # Display grouped summary. - for owner, grouped_repository_dicts in repository_dicts_by_owner.items(): - print "# " - for repository_dict in grouped_repository_dicts: - name = str( repository_dict[ 'name' ] ) - owner = str( repository_dict[ 'owner' ] ) - changeset_revision = str( repository_dict[ 'changeset_revision' ] ) - print "# Revision %s of repository %s owned by %s" % ( changeset_revision, name, owner ) - -def test_repository_tools( app, repository, repository_dict, tool_test_results_dicts, tool_test_results_dict, results_dict ): - """Test tools contained in the received repository.""" - name = str( repository.name ) - owner = str( repository.owner ) - changeset_revision = str( repository.changeset_revision ) - repository_identifier_dict = dict( name=name, owner=owner, changeset_revision=changeset_revision ) - # Set the module-level variable 'toolbox', so that test.functional.test_toolbox will generate the appropriate test methods. - test_toolbox.toolbox = app.toolbox - # Generate the test methods for this installed repository. We need to pass in True here, or it will look - # in $GALAXY_HOME/test-data for test data, which may result in missing or invalid test files. - test_toolbox.build_tests( testing_shed_tools=True, master_api_key=default_galaxy_master_api_key ) - # Set up nose to run the generated functional tests. - test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) - test_config.configure( sys.argv ) - # Run the configured tests. - result, test_plugins = run_tests( test_config ) - if result.wasSuccessful(): - # This repository's tools passed all functional tests. Use the ReportResults nose plugin to get a list - # of tests that passed. - for plugin in test_plugins: - if hasattr( plugin, 'getTestStatus' ): - test_identifier = '%s/%s' % ( owner, name ) - passed_tests = plugin.getTestStatus( test_identifier ) - break - tool_test_results_dict[ 'passed_tests' ] = [] - for test_id in passed_tests: - # Normalize the tool ID and version display. - tool_id, tool_version = get_tool_info_from_test_id( test_id ) - test_result = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version ) - tool_test_results_dict[ 'passed_tests' ].append( test_result ) - # Update the repository_metadata table in the tool shed's database to include the passed tests. - passed_repository_dict = repository_identifier_dict - results_dict[ 'all_tests_passed' ].append( passed_repository_dict ) - params = dict( tools_functionally_correct=True, - do_not_test=False, - test_install_error=False ) - # Call the register_test_result() method to execute a PUT request to the repository_revisions API - # controller with the status of the test. This also sets the do_not_test and tools_functionally - # correct flags and updates the time_last_tested field to today's date. - # TODO: do something useful with response_dict - response_dict = register_test_result( galaxy_tool_shed_url, - tool_test_results_dicts, - tool_test_results_dict, - repository_dict, - params ) - log.debug( 'Revision %s of repository %s owned by %s installed and passed functional tests.' % \ - ( changeset_revision, name, owner ) ) - else: - # The get_failed_test_dicts() method returns a list. - failed_test_dicts = get_failed_test_dicts( result, from_tool_test=True ) - tool_test_results_dict[ 'failed_tests' ] = failed_test_dicts - failed_repository_dict = repository_identifier_dict - results_dict[ 'at_least_one_test_failed' ].append( failed_repository_dict ) - set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_dict ) - params = dict( tools_functionally_correct=False, - test_install_error=False, - do_not_test=str( set_do_not_test ) ) - # TODO: do something useful with response_dict - response_dict = register_test_result( galaxy_tool_shed_url, - tool_test_results_dicts, - tool_test_results_dict, - repository_dict, - params ) - log.debug( 'Revision %s of repository %s owned by %s installed successfully but did not pass functional tests.' % \ - ( changeset_revision, name, owner ) ) - # Run the uninstall method. This removes tool functional test methods from the test_toolbox module and uninstalls the - # repository using Twill. - deactivate = asbool( os.environ.get( 'GALAXY_INSTALL_TEST_KEEP_TOOL_DEPENDENCIES', False ) ) - if deactivate: - log.debug( 'Deactivating changeset revision %s of repository %s' % ( str( changeset_revision ), str( name ) ) ) - # We are deactivating this repository and all of its repository dependencies. - deactivate_repository( app, repository_dict ) - else: - log.debug( 'Uninstalling changeset revision %s of repository %s' % ( str( changeset_revision ), str( name ) ) ) - # We are uninstalling this repository and all of its repository dependencies. - uninstall_repository_and_repository_dependencies( app, repository_dict ) - # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. - test_toolbox.toolbox = app.toolbox - return results_dict - -def uninstall_repository_and_repository_dependencies( app, repository_dict ): - """Uninstall a repository and all of its repository dependencies.""" - # This method assumes that the repositor defined by the received repository_dict is not a repository - # dependency of another repository. - sa_session = app.install_model.context - # Clean out any generated tests. This is necessary for Twill. - remove_generated_tests( app ) - # The dict contains the only repository the app should have installed at this point. - name = str( repository_dict[ 'name' ] ) - owner = str( repository_dict[ 'owner' ] ) - changeset_revision = str( repository_dict[ 'changeset_revision' ] ) - # Since this install and test framework uninstalls repositories immediately after installing and testing - # them, the values of repository.installed_changeset_revision and repository.changeset_revision should be - # the same. - repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) - if repository.can_uninstall( app ): - # A repository can be uninstalled only if no dependent repositories are installed. So uninstallation order - # id critical. A repository is always uninstalled first, and the each of its dependencies is checked to see - # if it can be uninstalled. - uninstall_repository_dict = dict( name=name, - owner=owner, - changeset_revision=changeset_revision ) - log.debug( 'Revision %s of repository %s owned by %s selected for uninstallation.' % ( changeset_revision, name, owner ) ) - try: - test_install_repositories.generate_uninstall_method( uninstall_repository_dict ) - # Set up nose to run the generated uninstall method as a functional test. - test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) - test_config.configure( sys.argv ) - # Run the uninstall method. This method uses the Galaxy web interface to uninstall the previously installed - # repository and all of its repository dependencies, deleting each of them from disk. - result, _ = run_tests( test_config ) - repository_uninstall_successful = result.wasSuccessful() - except Exception, e: - repository_uninstall_successful = False - log.exception( 'Uninstallation of revision %s of repository %s owned by %s failed: %s.' % \ - ( rd_changeset_revision, rd_name, rd_owner, str( e ) ) ) - if repository_uninstall_successful: - # Now that the repository is uninstalled we can attempt to uninstall each of its repository dependencies. - # We have to do this through Twill in order to maintain app.toolbox and shed_tool_conf.xml in a state that - # is valid for future tests. Since some of the repository's repository dependencies may require other of - # the repository's repository dependencies, we'll keep track of the repositories we've been able to unistall. - processed_repository_dependency_ids = [] - while len( processed_repository_dependency_ids ) < len( repository.repository_dependencies ): - for repository_dependency in repository.repository_dependencies: - if repository_dependency.id not in processed_repository_dependency_ids and repository_dependency.can_uninstall( app ): - processed_repository_dependency_ids.append( repository_dependency.id ) - rd_name = str( repository_dependency.name ) - rd_owner = str( repository_dependency.owner ) - rd_changeset_revision = str( repository_dependency.changeset_revision ) - uninstall_repository_dict = dict( name=rd_name, - owner=rd_owner, - changeset_revision=rd_changeset_revision ) - log.debug( 'Revision %s of repository dependency %s owned by %s selected for uninstallation.' % \ - ( rd_changeset_revision, rd_name, rd_owner ) ) - # Generate a test method to uninstall the repository dependency through the embedded Galaxy application's - # web interface. - try: - test_install_repositories.generate_uninstall_method( uninstall_repository_dict ) - # Set up nose to run the generated uninstall method as a functional test. - test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) - test_config.configure( sys.argv ) - # Run the uninstall method. - result, _ = run_tests( test_config ) - if not result.wasSuccessful(): - # We won't set ok here because we'll continue to uninstall whatever we can. - log.debug( 'Uninstallation of revision %s of repository %s owned by %s failed.' % \ - ( rd_changeset_revision, rd_name, rd_owner ) ) - except Exception, e: - log.exception( 'Uninstallation of revision %s of repository %s owned by %s failed: %s.' % \ - ( rd_changeset_revision, rd_name, rd_owner, str( e ) ) ) - else: - log.debug( 'Uninstallation of revision %s of repository %s owned by %s failed.' % ( changeset_revision, name, owner ) ) - else: - log_reason_repository_cannot_be_uninstalled( app, repository ) - -def uninstall_tool_dependency( app, tool_dependency ): - """Attempt to uninstall a tool dependency.""" - sa_session = app.install_model.context - # Clean out any generated tests. This is necessary for Twill. - tool_dependency_install_path = tool_dependency.installation_directory( app ) - uninstalled, error_message = tool_dependency_util.remove_tool_dependency( app, tool_dependency ) - if error_message: - log.debug( 'There was an error attempting to remove directory: %s' % str( tool_dependency_install_path ) ) - log.debug( error_message ) - else: - log.debug( 'Successfully removed tool dependency installation directory: %s' % str( tool_dependency_install_path ) ) - if not uninstalled or tool_dependency.status != app.model.ToolDependency.installation_status.UNINSTALLED: - tool_dependency.status = app.model.ToolDependency.installation_status.UNINSTALLED - sa_session.add( tool_dependency ) - sa_session.flush() - if os.path.exists( tool_dependency_install_path ): - log.debug( 'Uninstallation of tool dependency succeeded, but the installation path still exists on the filesystem. It is now being explicitly deleted.') - suc.remove_dir( tool_dependency_install_path ) - -if __name__ == "__main__": - # The tool_test_results_dict should always have the following structure: - # { - # "test_environment": - # { - # "galaxy_revision": "9001:abcd1234", - # "galaxy_database_version": "114", - # "tool_shed_revision": "9001:abcd1234", - # "tool_shed_mercurial_version": "2.3.1", - # "tool_shed_database_version": "17", - # "python_version": "2.7.2", - # "architecture": "x86_64", - # "system": "Darwin 12.2.0" - # }, - # "passed_tests": - # [ - # { - # "test_id": "The test ID, generated by twill", - # "tool_id": "The tool ID that was tested", - # "tool_version": "The tool version that was tested", - # }, - # ] - # "failed_tests": - # [ - # { - # "test_id": "The test ID, generated by twill", - # "tool_id": "The tool ID that was tested", - # "tool_version": "The tool version that was tested", - # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." - # "traceback": "The captured traceback." - # }, - # ] - # "installation_errors": - # { - # 'tool_dependencies': - # [ - # { - # 'type': 'Type of tool dependency, e.g. package, set_environment, etc.', - # 'name': 'Name of the tool dependency.', - # 'version': 'Version if this is a package, otherwise blank.', - # 'error_message': 'The error message returned when installation was attempted.', - # }, - # ], - # 'repository_dependencies': - # [ - # { - # 'tool_shed': 'The tool shed that this repository was installed from.', - # 'name': 'The name of the repository that failed to install.', - # 'owner': 'Owner of the failed repository.', - # 'changeset_revision': 'Changeset revision of the failed repository.', - # 'error_message': 'The error message that was returned when the repository failed to install.', - # }, - # ], - # 'current_repository': - # [ - # { - # 'tool_shed': 'The tool shed that this repository was installed from.', - # 'name': 'The name of the repository that failed to install.', - # 'owner': 'Owner of the failed repository.', - # 'changeset_revision': 'Changeset revision of the failed repository.', - # 'error_message': 'The error message that was returned when the repository failed to install.', - # }, - # ], - # { - # "name": "The name of the repository.", - # "owner": "The owner of the repository.", - # "changeset_revision": "The changeset revision of the repository.", - # "error_message": "The message stored in tool_dependency.error_message." - # }, - # } - # "missing_test_components": - # [ - # { - # "tool_id": "The tool ID that missing components.", - # "tool_version": "The version of the tool." - # "tool_guid": "The guid of the tool." - # "missing_components": "Which components are missing, e.g. the test data filename, or the test-data directory." - # }, - # ] - # "not_tested": - # { - # "reason": "The Galaxy development team has determined that this repository should not be installed and tested by the automated framework." - # } - # } - sys.exit( main() ) diff -r b1e82182414d359f6f43e117e92ec6d6638878fe -r 43b23f5503997727debf8232239434974491e55e test/install_and_test_tool_shed_repositories/repositories_with_tools/__init__.py --- /dev/null +++ b/test/install_and_test_tool_shed_repositories/repositories_with_tools/__init__.py @@ -0,0 +1,1 @@ +"""Install and test tool shed repositories that contain tools and tool test components.""" \ No newline at end of file This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 17 Dec '13

17 Dec '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8eaabf513c44/ Changeset: 8eaabf513c44 User: jmchilton Date: 2013-12-17 18:19:41 Summary: Reuse code between scripts/manage_tools.py and lib/galaxy/model/orm/scripts.py Affected #: 2 files diff -r a2bad69f03f4786309ba80ff305ebdd7b68f93fa -r 8eaabf513c4494a3990ab93bebc7d16ad9276e3d lib/galaxy/model/orm/scripts.py --- a/lib/galaxy/model/orm/scripts.py +++ b/lib/galaxy/model/orm/scripts.py @@ -43,6 +43,31 @@ } +def require_dialect_egg( db_url ): + dialect = ( db_url.split( ':', 1 ) )[0] + try: + egg = dialect_to_egg[dialect] + try: + pkg_resources.require( egg ) + log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) + except: + # If the module is in the path elsewhere (i.e. non-egg), it'll still load. + log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) + except KeyError: + # Let this go, it could possibly work with db's we don't support + log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) + + +def read_config_file_arg( argv, default ): + if '-c' in argv: + pos = argv.index( '-c' ) + argv.pop(pos) + config_file = argv.pop( pos ) + else: + config_file = default + return config_file + + def get_config( argv, cwd=None ): """ Read sys.argv and parse out repository of migrations and database url. @@ -73,12 +98,7 @@ database = 'galaxy' database_defaults = DATABASE[ database ] - if '-c' in argv: - pos = argv.index( '-c' ) - argv.pop(pos) - config_file = argv.pop( pos ) - else: - config_file = database_defaults.get( 'config_file', DEFAULT_CONFIG_FILE ) + config_file = read_config_file_arg( argv, database_defaults.get( 'config_file', DEFAULT_CONFIG_FILE ) ) repo = database_defaults[ 'repo' ] config_prefix = database_defaults.get( 'config_prefix', DEFAULT_CONFIG_PREFIX ) default_sqlite_file = database_defaults[ 'default_sqlite_file' ] @@ -95,16 +115,5 @@ else: db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % default_sqlite_file - dialect = ( db_url.split( ':', 1 ) )[0] - try: - egg = dialect_to_egg[dialect] - try: - pkg_resources.require( egg ) - log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) - except: - # If the module is in the path elsewhere (i.e. non-egg), it'll still load. - log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) - except KeyError: - # Let this go, it could possibly work with db's we don't support - log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) + require_dialect_egg( db_url ) return dict(db_url=db_url, repo=repo, config_file=config_file, database=database) diff -r a2bad69f03f4786309ba80ff305ebdd7b68f93fa -r 8eaabf513c4494a3990ab93bebc7d16ad9276e3d scripts/manage_tools.py --- a/scripts/manage_tools.py +++ b/scripts/manage_tools.py @@ -1,7 +1,9 @@ -import sys, os.path, logging +import sys +import os.path +import logging new_path = [ os.path.join( os.getcwd(), "lib" ) ] -new_path.extend( sys.path[1:] ) # remove scripts/ from the path +new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path from galaxy import eggs @@ -15,15 +17,12 @@ from migrate.versioning.shell import main from ConfigParser import SafeConfigParser -from galaxy.model.orm import dialect_to_egg +from galaxy.model.orm.scripts import require_dialect_egg +from galaxy.model.orm.scripts import read_config_file_arg log = logging.getLogger( __name__ ) -config_file = 'universe_wsgi.ini' -if '-c' in sys.argv: - pos = sys.argv.index( '-c' ) - sys.argv.pop( pos ) - config_file = sys.argv.pop( pos ) +config_file = read_config_file_arg( sys.argv, 'universe_wsgi.ini' ) if not os.path.exists( config_file ): print "Galaxy config file does not exist (hint: use '-c config.ini' for non-standard locations): %s" % config_file sys.exit( 1 ) @@ -42,17 +41,6 @@ else: db_url = "sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE" -dialect = ( db_url.split( ':', 1 ) )[0] -try: - egg = dialect_to_egg[dialect] - try: - pkg_resources.require( egg ) - log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) - except: - # If the module is in the path elsewhere (i.e. non-egg), it'll still load. - log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) -except KeyError: - # Let this go, it could possibly work with db's we don't support - log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) +require_dialect_egg( db_url ) main( repository=repo, url=db_url ) https://bitbucket.org/galaxy/galaxy-central/commits/723431aa61fb/ Changeset: 723431aa61fb User: jmchilton Date: 2013-12-17 18:19:41 Summary: Tools now properly migrated into install_database_connection database... ... this second database is specified. Affected #: 1 file diff -r 8eaabf513c4494a3990ab93bebc7d16ad9276e3d -r 723431aa61fb4c7d5cd1f73fe34e16af29368ef4 scripts/manage_tools.py --- a/scripts/manage_tools.py +++ b/scripts/manage_tools.py @@ -34,6 +34,8 @@ if config_file == 'universe_wsgi.ini.sample' and 'GALAXY_TEST_DBURI' in os.environ: # Running functional tests. db_url = os.environ[ 'GALAXY_TEST_DBURI' ] +elif cp.has_option( "app:main", "install_database_connection" ): + db_url = cp.get( "app:main", "install_database_connection" ) elif cp.has_option( "app:main", "database_connection" ): db_url = cp.get( "app:main", "database_connection" ) elif cp.has_option( "app:main", "database_file" ): https://bitbucket.org/galaxy/galaxy-central/commits/b1e82182414d/ Changeset: b1e82182414d User: jmchilton Date: 2013-12-17 18:47:48 Summary: Merge latest galaxy-central. Affected #: 7 files diff -r 723431aa61fb4c7d5cd1f73fe34e16af29368ef4 -r b1e82182414d359f6f43e117e92ec6d6638878fe lib/galaxy/webapps/galaxy/controllers/history.py --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -69,7 +69,7 @@ default_sort_key = "-update_time" columns = [ HistoryListNameColumn( "Name", key="name", attach_popup=True, filterable="advanced" ), - DatasetsByStateColumn( "Datasets", key="datasets_by_state", sortable=False ), + DatasetsByStateColumn( "Datasets", key="datasets_by_state", sortable=False, nowrap=True), grids.IndividualTagsColumn( "Tags", key="tags", model_tag_association_class=model.HistoryTagAssociation, \ filterable="advanced", grid_name="HistoryListGrid" ), grids.SharingStatusColumn( "Sharing", key="sharing", filterable="advanced", sortable=False ), diff -r 723431aa61fb4c7d5cd1f73fe34e16af29368ef4 -r b1e82182414d359f6f43e117e92ec6d6638878fe lib/tool_shed/scripts/clean_up_tool_dependency_directory.py --- /dev/null +++ b/lib/tool_shed/scripts/clean_up_tool_dependency_directory.py @@ -0,0 +1,58 @@ +import argparse +import os +import sys +import shutil + +def main( args ): + if not os.path.exists( args.basepath ): + print 'Tool dependency path %s does not exist.' % str( args.basepath ) + return 1 + if args.delete: + print 'Deleting contents of tool dependency path %s.' % args.basepath + for node in os.listdir( args.basepath ): + path = os.path.join( args.basepath, node ) + if os.path.isdir( path ): + try: + shutil.rmtree( path ) + print 'Deleted directory %s and all its contents.' % path + except Exception, e: + print 'Error deleting directory %s: %s' % ( path, str( e ) ) + pass + elif os.path.isfile( path ): + try: + os.remove( path ) + print 'Deleted file %s.' % path + except Exception, e: + print 'Error deleting file %s: %s' % ( path, str( e ) ) + pass + elif os.path.islink( path ): + print 'Deleting symlink %s with target %s.' % ( path, os.path.realpath( path ) ) + try: + os.remove( path ) + except Exception, e: + print 'Error deleting symlink %s: %s' % ( path, str( e ) ) + pass + else: + print 'Tool dependency path %s contains the following files and directories:' % args.basepath + for element in os.listdir( args.basepath ): + print element + return 0 + +if __name__ == '__main__': + description = 'Clean out or list the contents of the provided tool dependency path. Remove if ' + description += 'the --delete command line argument is provided.' + parser = argparse.ArgumentParser( description=description ) + parser.add_argument( '--delete', + dest='delete', + required=False, + action='store_true', + default=False, + help='Whether to delete all folders and files or list them on exit.' ) + parser.add_argument( '--basepath', + dest='basepath', + required=True, + action='store', + metavar='name', + help='The base path where tool dependencies are installed.' ) + args = parser.parse_args() + sys.exit( main( args ) ) diff -r 723431aa61fb4c7d5cd1f73fe34e16af29368ef4 -r b1e82182414d359f6f43e117e92ec6d6638878fe test/base/tool_shed_util.py --- a/test/base/tool_shed_util.py +++ b/test/base/tool_shed_util.py @@ -15,6 +15,9 @@ log = logging.getLogger(__name__) +# Set a 10 minute timeout for repository installation. +repository_installation_timeout = 600 + def get_installed_repository_info( elem, last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision, tool_path ): """ Return the GALAXY_TEST_FILE_DIR, the containing repository name and the change set revision for the tool elem. diff -r 723431aa61fb4c7d5cd1f73fe34e16af29368ef4 -r b1e82182414d359f6f43e117e92ec6d6638878fe test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -1,39 +1,38 @@ -import pkg_resources -pkg_resources.require( "twill==0.9" ) - +import difflib +import filecmp +import logging +import os +import re +import pprint +import shutil import StringIO -import os -import filecmp +import subprocess +import tarfile +import tempfile import time import unittest import urllib -import logging -import difflib -import tarfile import zipfile -import tempfile -import re -import shutil -import subprocess -import pprint -import twill -import twill.commands as tc -from twill.other_packages._mechanize_dist import ClientForm -pkg_resources.require( "elementtree" ) -pkg_resources.require( "MarkupSafe" ) -from markupsafe import escape -from elementtree import ElementTree from galaxy.web import security from galaxy.web.framework.helpers import iff from galaxy.util.json import from_json_string from base.asserts import verify_assertions -buffer = StringIO.StringIO() +from galaxy import eggs +eggs.require( "elementtree" ) +eggs.require( 'twill' ) + +from elementtree import ElementTree + +import twill +import twill.commands as tc +from twill.other_packages._mechanize_dist import ClientForm #Force twill to log to a buffer -- FIXME: Should this go to stdout and be captured by nose? -twill.set_output(buffer) -tc.config('use_tidy', 0) +buffer = StringIO.StringIO() +twill.set_output( buffer ) +tc.config( 'use_tidy', 0 ) # Dial ClientCookie logging down (very noisy) logging.getLogger( "ClientCookie.cookies" ).setLevel( logging.WARNING ) diff -r 723431aa61fb4c7d5cd1f73fe34e16af29368ef4 -r b1e82182414d359f6f43e117e92ec6d6638878fe test/install_and_test_tool_shed_repositories/base/twilltestcase.py --- a/test/install_and_test_tool_shed_repositories/base/twilltestcase.py +++ b/test/install_and_test_tool_shed_repositories/base/twilltestcase.py @@ -1,9 +1,22 @@ +import logging +import os +import re +import test_db_util +import time +import urllib + import galaxy.model as model import galaxy.model.tool_shed_install as install_model -import common, string, os, re, test_db_util, simplejson, logging, time, sys -import galaxy.util as util -from base.twilltestcase import tc, from_json_string, TwillTestCase, security, urllib -from tool_shed.util.encoding_util import tool_shed_encode, tool_shed_decode +import galaxy.util + +from galaxy.web import security +from base.twilltestcase import TwillTestCase +from base.tool_shed_util import repository_installation_timeout + +from galaxy import eggs +eggs.require( 'twill' ) + +import twill.commands as tc log = logging.getLogger( __name__ ) @@ -52,7 +65,7 @@ if install_parameters: iri_ids = install_parameters.group(1) # In some cases, the returned iri_ids are of the form: "[u'<encoded id>', u'<encoded id>']" - # This regex ensures that non-hex characters are stripped out of the list, so that util.listify/decode_id + # This regex ensures that non-hex characters are stripped out of the list, so that galaxy.util.listify/decode_id # will handle them correctly. It's safe to pass the cleaned list to manage_repositories, because it can parse # comma-separated values. repository_ids = str( iri_ids ) @@ -60,9 +73,9 @@ encoded_kwd = install_parameters.group(2) reinstalling = install_parameters.group(3) url = '/admin_toolshed/manage_repositories?operation=install&tool_shed_repository_ids=%s&encoded_kwd=%s&reinstalling=%s' % \ - ( ','.join( util.listify( repository_ids ) ), encoded_kwd, reinstalling ) + ( ','.join( galaxy.util.listify( repository_ids ) ), encoded_kwd, reinstalling ) self.visit_url( url ) - return util.listify( repository_ids ) + return galaxy.util.listify( repository_ids ) def install_repository( self, repository_info_dict, install_tool_dependencies=True, install_repository_dependencies=True, strings_displayed=[], strings_not_displayed=[], preview_strings_displayed=[], @@ -161,9 +174,9 @@ timeout_counter = timeout_counter + 1 if timeout_counter % 10 == 0: log.debug( 'Waited %d seconds for repository %s.' % ( timeout_counter, str( galaxy_repository.name ) ) ) - # This timeout currently defaults to 180 seconds, or 3 minutes. - if timeout_counter > common.repository_installation_timeout: - raise AssertionError( 'Repository installation timed out, %d seconds elapsed, repository state is %s.' % \ - ( timeout_counter, repository.status ) ) + # This timeout currently defaults to 10 minutes. + if timeout_counter > repository_installation_timeout: + raise AssertionError( 'Repository installation timed out after %d seconds, repository state is %s.' % \ + ( timeout_counter, repository.status ) ) break time.sleep( 1 ) diff -r 723431aa61fb4c7d5cd1f73fe34e16af29368ef4 -r b1e82182414d359f6f43e117e92ec6d6638878fe test/tool_shed/base/common.py --- a/test/tool_shed/base/common.py +++ b/test/tool_shed/base/common.py @@ -33,6 +33,3 @@ ''' new_repository_dependencies_line = ''' <repository toolshed="${toolshed_url}" name="${repository_name}" owner="${owner}" changeset_revision="${changeset_revision}"${prior_installation_required} />''' - -# Set a 3 minute timeout for repository installation. This should be sufficient, since we're not installing tool dependencies. -repository_installation_timeout = 180 \ No newline at end of file diff -r 723431aa61fb4c7d5cd1f73fe34e16af29368ef4 -r b1e82182414d359f6f43e117e92ec6d6638878fe test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -12,16 +12,27 @@ import urllib import galaxy.webapps.tool_shed.util.hgweb_config import galaxy.model.tool_shed_install as galaxy_model -import galaxy.util as util +import galaxy.util + +from base.tool_shed_util import repository_installation_timeout +from base.twilltestcase import TwillTestCase +from galaxy.util.json import from_json_string +from galaxy.web import security +from tool_shed.util.encoding_util import tool_shed_encode from tool_shed.util import shed_util_common as suc from tool_shed.util import xml_util -from base.twilltestcase import tc, from_json_string, TwillTestCase, security, urllib -from tool_shed.util.encoding_util import tool_shed_encode, tool_shed_decode from galaxy import eggs -eggs.require('mercurial') -from mercurial import hg, ui, commands +eggs.require( 'mercurial' ) +eggs.require( 'twill' ) + from mercurial.util import Abort +from mercurial import commands +from mercurial import hg +from mercurial import ui + +import twill.commands as tc + log = logging.getLogger( __name__ ) @@ -848,7 +859,7 @@ if install_parameters: iri_ids = install_parameters.group(1) # In some cases, the returned iri_ids are of the form: "[u'<encoded id>', u'<encoded id>']" - # This regex ensures that non-hex characters are stripped out of the list, so that util.listify/decode_id + # This regex ensures that non-hex characters are stripped out of the list, so that galaxy.util.listify/decode_id # will handle them correctly. It's safe to pass the cleaned list to manage_repositories, because it can parse # comma-separated values. repository_ids = str( iri_ids ) @@ -856,9 +867,9 @@ encoded_kwd = install_parameters.group(2) reinstalling = install_parameters.group(3) url = '/admin_toolshed/manage_repositories?operation=install&tool_shed_repository_ids=%s&encoded_kwd=%s&reinstalling=%s' % \ - ( ','.join( util.listify( repository_ids ) ), encoded_kwd, reinstalling ) + ( ','.join( galaxy.util.listify( repository_ids ) ), encoded_kwd, reinstalling ) self.visit_galaxy_url( url ) - return util.listify( repository_ids ) + return galaxy.util.listify( repository_ids ) def install_repositories_from_search_results( self, repositories, install_tool_dependencies=False, strings_displayed=[], strings_not_displayed=[], **kwd ): @@ -1327,7 +1338,7 @@ def verify_installed_repository_data_table_entries( self, required_data_table_entries ): # The value of the received required_data_table_entries will be something like: [ 'sam_fa_indexes' ] - data_tables = util.parse_xml( self.shed_tool_data_table_conf ) + data_tables = xml_util.parse_xml( self.shed_tool_data_table_conf ) found = False # With the tool shed, the "path" attribute that is hard-coded into the tool_data_tble_conf.xml # file is ignored. This is because the tool shed requires the directory location to which this @@ -1445,8 +1456,8 @@ while galaxy_repository.status not in final_states: test_db_util.ga_refresh( galaxy_repository ) timeout_counter = timeout_counter + 1 - # This timeout currently defaults to 180 seconds, or 3 minutes. - if timeout_counter > common.repository_installation_timeout: + # This timeout currently defaults to 10 minutes. + if timeout_counter > repository_installation_timeout: raise AssertionError( 'Repository installation timed out, %d seconds elapsed, repository state is %s.' % \ ( timeout_counter, galaxy_repository.status ) ) break Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: guerler: Grids: Fix dataset state viewer in saved histories
by commits-noreply＠bitbucket.org 17 Dec '13

17 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/62ed22bb4360/ Changeset: 62ed22bb4360 User: guerler Date: 2013-12-17 18:44:33 Summary: Grids: Fix dataset state viewer in saved histories Affected #: 1 file diff -r 270931cae37f2efaac24f6c22236e5fa0b87ce23 -r 62ed22bb4360d11fcfedcce09c961741186a3b88 lib/galaxy/webapps/galaxy/controllers/history.py --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -69,7 +69,7 @@ default_sort_key = "-update_time" columns = [ HistoryListNameColumn( "Name", key="name", attach_popup=True, filterable="advanced" ), - DatasetsByStateColumn( "Datasets", key="datasets_by_state", sortable=False ), + DatasetsByStateColumn( "Datasets", key="datasets_by_state", sortable=False, nowrap=True), grids.IndividualTagsColumn( "Tags", key="tags", model_tag_association_class=model.HistoryTagAssociation, \ filterable="advanced", grid_name="HistoryListGrid" ), grids.SharingStatusColumn( "Sharing", key="sharing", filterable="advanced", sortable=False ), Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Add script to clean up or display the contents of a tool dependency path.
by commits-noreply＠bitbucket.org 17 Dec '13

17 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/270931cae37f/ Changeset: 270931cae37f User: inithello Date: 2013-12-17 18:20:08 Summary: Add script to clean up or display the contents of a tool dependency path. Affected #: 1 file diff -r 786c58661b6429a7bcdb0978df1da74dddbd656d -r 270931cae37f2efaac24f6c22236e5fa0b87ce23 lib/tool_shed/scripts/clean_up_tool_dependency_directory.py --- /dev/null +++ b/lib/tool_shed/scripts/clean_up_tool_dependency_directory.py @@ -0,0 +1,58 @@ +import argparse +import os +import sys +import shutil + +def main( args ): + if not os.path.exists( args.basepath ): + print 'Tool dependency path %s does not exist.' % str( args.basepath ) + return 1 + if args.delete: + print 'Deleting contents of tool dependency path %s.' % args.basepath + for node in os.listdir( args.basepath ): + path = os.path.join( args.basepath, node ) + if os.path.isdir( path ): + try: + shutil.rmtree( path ) + print 'Deleted directory %s and all its contents.' % path + except Exception, e: + print 'Error deleting directory %s: %s' % ( path, str( e ) ) + pass + elif os.path.isfile( path ): + try: + os.remove( path ) + print 'Deleted file %s.' % path + except Exception, e: + print 'Error deleting file %s: %s' % ( path, str( e ) ) + pass + elif os.path.islink( path ): + print 'Deleting symlink %s with target %s.' % ( path, os.path.realpath( path ) ) + try: + os.remove( path ) + except Exception, e: + print 'Error deleting symlink %s: %s' % ( path, str( e ) ) + pass + else: + print 'Tool dependency path %s contains the following files and directories:' % args.basepath + for element in os.listdir( args.basepath ): + print element + return 0 + +if __name__ == '__main__': + description = 'Clean out or list the contents of the provided tool dependency path. Remove if ' + description += 'the --delete command line argument is provided.' + parser = argparse.ArgumentParser( description=description ) + parser.add_argument( '--delete', + dest='delete', + required=False, + action='store_true', + default=False, + help='Whether to delete all folders and files or list them on exit.' ) + parser.add_argument( '--basepath', + dest='basepath', + required=True, + action='store', + metavar='name', + help='The base path where tool dependencies are installed.' ) + args = parser.parse_args() + sys.exit( main( args ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix imports in various functional test scripts.
by commits-noreply＠bitbucket.org 17 Dec '13

17 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/786c58661b64/ Changeset: 786c58661b64 User: greg Date: 2013-12-17 17:20:23 Summary: Fix imports in various functional test scripts. Affected #: 5 files diff -r a2bad69f03f4786309ba80ff305ebdd7b68f93fa -r 786c58661b6429a7bcdb0978df1da74dddbd656d test/base/tool_shed_util.py --- a/test/base/tool_shed_util.py +++ b/test/base/tool_shed_util.py @@ -15,6 +15,9 @@ log = logging.getLogger(__name__) +# Set a 10 minute timeout for repository installation. +repository_installation_timeout = 600 + def get_installed_repository_info( elem, last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision, tool_path ): """ Return the GALAXY_TEST_FILE_DIR, the containing repository name and the change set revision for the tool elem. diff -r a2bad69f03f4786309ba80ff305ebdd7b68f93fa -r 786c58661b6429a7bcdb0978df1da74dddbd656d test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -1,39 +1,38 @@ -import pkg_resources -pkg_resources.require( "twill==0.9" ) - +import difflib +import filecmp +import logging +import os +import re +import pprint +import shutil import StringIO -import os -import filecmp +import subprocess +import tarfile +import tempfile import time import unittest import urllib -import logging -import difflib -import tarfile import zipfile -import tempfile -import re -import shutil -import subprocess -import pprint -import twill -import twill.commands as tc -from twill.other_packages._mechanize_dist import ClientForm -pkg_resources.require( "elementtree" ) -pkg_resources.require( "MarkupSafe" ) -from markupsafe import escape -from elementtree import ElementTree from galaxy.web import security from galaxy.web.framework.helpers import iff from galaxy.util.json import from_json_string from base.asserts import verify_assertions -buffer = StringIO.StringIO() +from galaxy import eggs +eggs.require( "elementtree" ) +eggs.require( 'twill' ) + +from elementtree import ElementTree + +import twill +import twill.commands as tc +from twill.other_packages._mechanize_dist import ClientForm #Force twill to log to a buffer -- FIXME: Should this go to stdout and be captured by nose? -twill.set_output(buffer) -tc.config('use_tidy', 0) +buffer = StringIO.StringIO() +twill.set_output( buffer ) +tc.config( 'use_tidy', 0 ) # Dial ClientCookie logging down (very noisy) logging.getLogger( "ClientCookie.cookies" ).setLevel( logging.WARNING ) diff -r a2bad69f03f4786309ba80ff305ebdd7b68f93fa -r 786c58661b6429a7bcdb0978df1da74dddbd656d test/install_and_test_tool_shed_repositories/base/twilltestcase.py --- a/test/install_and_test_tool_shed_repositories/base/twilltestcase.py +++ b/test/install_and_test_tool_shed_repositories/base/twilltestcase.py @@ -1,9 +1,22 @@ +import logging +import os +import re +import test_db_util +import time +import urllib + import galaxy.model as model import galaxy.model.tool_shed_install as install_model -import common, string, os, re, test_db_util, simplejson, logging, time, sys -import galaxy.util as util -from base.twilltestcase import tc, from_json_string, TwillTestCase, security, urllib -from tool_shed.util.encoding_util import tool_shed_encode, tool_shed_decode +import galaxy.util + +from galaxy.web import security +from base.twilltestcase import TwillTestCase +from base.tool_shed_util import repository_installation_timeout + +from galaxy import eggs +eggs.require( 'twill' ) + +import twill.commands as tc log = logging.getLogger( __name__ ) @@ -52,7 +65,7 @@ if install_parameters: iri_ids = install_parameters.group(1) # In some cases, the returned iri_ids are of the form: "[u'<encoded id>', u'<encoded id>']" - # This regex ensures that non-hex characters are stripped out of the list, so that util.listify/decode_id + # This regex ensures that non-hex characters are stripped out of the list, so that galaxy.util.listify/decode_id # will handle them correctly. It's safe to pass the cleaned list to manage_repositories, because it can parse # comma-separated values. repository_ids = str( iri_ids ) @@ -60,9 +73,9 @@ encoded_kwd = install_parameters.group(2) reinstalling = install_parameters.group(3) url = '/admin_toolshed/manage_repositories?operation=install&tool_shed_repository_ids=%s&encoded_kwd=%s&reinstalling=%s' % \ - ( ','.join( util.listify( repository_ids ) ), encoded_kwd, reinstalling ) + ( ','.join( galaxy.util.listify( repository_ids ) ), encoded_kwd, reinstalling ) self.visit_url( url ) - return util.listify( repository_ids ) + return galaxy.util.listify( repository_ids ) def install_repository( self, repository_info_dict, install_tool_dependencies=True, install_repository_dependencies=True, strings_displayed=[], strings_not_displayed=[], preview_strings_displayed=[], @@ -161,9 +174,9 @@ timeout_counter = timeout_counter + 1 if timeout_counter % 10 == 0: log.debug( 'Waited %d seconds for repository %s.' % ( timeout_counter, str( galaxy_repository.name ) ) ) - # This timeout currently defaults to 180 seconds, or 3 minutes. - if timeout_counter > common.repository_installation_timeout: - raise AssertionError( 'Repository installation timed out, %d seconds elapsed, repository state is %s.' % \ - ( timeout_counter, repository.status ) ) + # This timeout currently defaults to 10 minutes. + if timeout_counter > repository_installation_timeout: + raise AssertionError( 'Repository installation timed out after %d seconds, repository state is %s.' % \ + ( timeout_counter, repository.status ) ) break time.sleep( 1 ) diff -r a2bad69f03f4786309ba80ff305ebdd7b68f93fa -r 786c58661b6429a7bcdb0978df1da74dddbd656d test/tool_shed/base/common.py --- a/test/tool_shed/base/common.py +++ b/test/tool_shed/base/common.py @@ -33,6 +33,3 @@ ''' new_repository_dependencies_line = ''' <repository toolshed="${toolshed_url}" name="${repository_name}" owner="${owner}" changeset_revision="${changeset_revision}"${prior_installation_required} />''' - -# Set a 3 minute timeout for repository installation. This should be sufficient, since we're not installing tool dependencies. -repository_installation_timeout = 180 \ No newline at end of file diff -r a2bad69f03f4786309ba80ff305ebdd7b68f93fa -r 786c58661b6429a7bcdb0978df1da74dddbd656d test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -12,16 +12,27 @@ import urllib import galaxy.webapps.tool_shed.util.hgweb_config import galaxy.model.tool_shed_install as galaxy_model -import galaxy.util as util +import galaxy.util + +from base.tool_shed_util import repository_installation_timeout +from base.twilltestcase import TwillTestCase +from galaxy.util.json import from_json_string +from galaxy.web import security +from tool_shed.util.encoding_util import tool_shed_encode from tool_shed.util import shed_util_common as suc from tool_shed.util import xml_util -from base.twilltestcase import tc, from_json_string, TwillTestCase, security, urllib -from tool_shed.util.encoding_util import tool_shed_encode, tool_shed_decode from galaxy import eggs -eggs.require('mercurial') -from mercurial import hg, ui, commands +eggs.require( 'mercurial' ) +eggs.require( 'twill' ) + from mercurial.util import Abort +from mercurial import commands +from mercurial import hg +from mercurial import ui + +import twill.commands as tc + log = logging.getLogger( __name__ ) @@ -848,7 +859,7 @@ if install_parameters: iri_ids = install_parameters.group(1) # In some cases, the returned iri_ids are of the form: "[u'<encoded id>', u'<encoded id>']" - # This regex ensures that non-hex characters are stripped out of the list, so that util.listify/decode_id + # This regex ensures that non-hex characters are stripped out of the list, so that galaxy.util.listify/decode_id # will handle them correctly. It's safe to pass the cleaned list to manage_repositories, because it can parse # comma-separated values. repository_ids = str( iri_ids ) @@ -856,9 +867,9 @@ encoded_kwd = install_parameters.group(2) reinstalling = install_parameters.group(3) url = '/admin_toolshed/manage_repositories?operation=install&tool_shed_repository_ids=%s&encoded_kwd=%s&reinstalling=%s' % \ - ( ','.join( util.listify( repository_ids ) ), encoded_kwd, reinstalling ) + ( ','.join( galaxy.util.listify( repository_ids ) ), encoded_kwd, reinstalling ) self.visit_galaxy_url( url ) - return util.listify( repository_ids ) + return galaxy.util.listify( repository_ids ) def install_repositories_from_search_results( self, repositories, install_tool_dependencies=False, strings_displayed=[], strings_not_displayed=[], **kwd ): @@ -1327,7 +1338,7 @@ def verify_installed_repository_data_table_entries( self, required_data_table_entries ): # The value of the received required_data_table_entries will be something like: [ 'sam_fa_indexes' ] - data_tables = util.parse_xml( self.shed_tool_data_table_conf ) + data_tables = xml_util.parse_xml( self.shed_tool_data_table_conf ) found = False # With the tool shed, the "path" attribute that is hard-coded into the tool_data_tble_conf.xml # file is ignored. This is because the tool shed requires the directory location to which this @@ -1445,8 +1456,8 @@ while galaxy_repository.status not in final_states: test_db_util.ga_refresh( galaxy_repository ) timeout_counter = timeout_counter + 1 - # This timeout currently defaults to 180 seconds, or 3 minutes. - if timeout_counter > common.repository_installation_timeout: + # This timeout currently defaults to 10 minutes. + if timeout_counter > repository_installation_timeout: raise AssertionError( 'Repository installation timed out, %d seconds elapsed, repository state is %s.' % \ ( timeout_counter, galaxy_repository.status ) ) break Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Re-tweak config.py imports for legacy installs.
by commits-noreply＠bitbucket.org 17 Dec '13

17 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a2bad69f03f4/ Changeset: a2bad69f03f4 User: jmchilton Date: 2013-12-17 17:10:41 Summary: Re-tweak config.py imports for legacy installs. Affected #: 1 file diff -r 374783ca6a480fbb571bfe9eb46052667b9d0479 -r a2bad69f03f4786309ba80ff305ebdd7b68f93fa lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -1,6 +1,8 @@ """ Universe configuration builder. """ +# absolute_import needed for tool_shed package. +from __future__ import absolute_import import sys, os, tempfile, re import logging, logging.config Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 9 new changesets
by commits-noreply＠bitbucket.org 17 Dec '13

17 Dec '13

9 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0084ac72aa7b/ Changeset: 0084ac72aa7b User: jmchilton Date: 2013-11-13 05:02:58 Summary: FASTQ Opt: In convert_read_to_format, adjust when new_encoding logic is calculated. Doesn't change the behavior or optimize anything, this is just done to simplify subsequent commits. Baseline: I took the first 100 megabytes of a 26 gigabyte example of a FASTQ file filtered with the FASTQ filter tool that I found in an "Important Galaxy User"'s history on main. I ran with the same command-line on my dev box with the start of this file and using Python's -m to profile function times and total amount of time to serve as a baseline as I optimized the FASTQ filter code. Here is the start of the output: Kept 135848 of 250000 reads (54.34%). 200015991 function calls (199765991 primitive calls) in 136.934 seconds Extrapolating this out, that 26 gigabyte file would take roughly 10 hours to process on my laptop - this is slightly longer than what it took on main - indicating to me this is likely not disk bound since my SSD would probably outperform main? Affected #: 1 file diff -r d61de8f362929483c2a6a6cb8e7b7be77fa5cfa4 -r 0084ac72aa7bdd253454027edf1bcbb3cdeb496d lib/galaxy_utils/sequence/fastq.py --- a/lib/galaxy_utils/sequence/fastq.py +++ b/lib/galaxy_utils/sequence/fastq.py @@ -89,7 +89,6 @@ score_list = self.convert_score_solexa_to_phred( self.get_decimal_quality_scores() ) else: score_list = self.get_decimal_quality_scores() - new_read.quality = "%s " % " ".join( map( str, new_class.restrict_scores_to_valid_range( score_list ) ) ) #need trailing space to be valid decimal fastq if force_quality_encoding is None: if self.is_ascii_encoded(): new_encoding = 'ascii' @@ -97,6 +96,7 @@ new_encoding = 'decimal' else: new_encoding = force_quality_encoding + new_read.quality = "%s " % " ".join( map( str, new_class.restrict_scores_to_valid_range( score_list ) ) ) #need trailing space to be valid decimal fastq if new_encoding == 'ascii': new_read.quality = "".join( new_read.get_ascii_quality_scores() ) return new_read https://bitbucket.org/galaxy/galaxy-central/commits/046c608abfa3/ Changeset: 046c608abfa3 User: jmchilton Date: 2013-11-13 05:02:58 Summary: FASTQ Opt: Restructure fastq_filter to only call execfile once and reuse the same function. After this optimization: Kept 135848 of 250000 reads (54.34%). 200015991 function calls (199765991 primitive calls) in 85.375 seconds Down from 136.934 seconds on previous changeset. Main Difference: (ncalls|tottime|percall|cumtime|percall|filename:lineno(function)) 250000 45.130 0.000 67.811 0.000 {execfile} - became - 1 0.000 0.000 0.000 0.000 {execfile} Affected #: 2 files diff -r 0084ac72aa7bdd253454027edf1bcbb3cdeb496d -r 046c608abfa31091f3e030db3979f848666fa602 tools/fastq/fastq_filter.py --- a/tools/fastq/fastq_filter.py +++ b/tools/fastq/fastq_filter.py @@ -18,10 +18,10 @@ i = None reads_kept = 0 + execfile(script_filename, globals()) for i, fastq_read in enumerate( fastqReader( open( input_filename ), format = input_type ) ): - local = {'fastq_read':fastq_read, 'ret_val':False} - execfile( script_filename, {}, local ) - if local['ret_val']: + ret_val = fastq_read_pass_filter( fastq_read ) ## fastq_read_pass_filter defined in script_filename + if ret_val: out.write( fastq_read ) reads_kept += 1 out.close() diff -r 0084ac72aa7bdd253454027edf1bcbb3cdeb496d -r 046c608abfa31091f3e030db3979f848666fa602 tools/fastq/fastq_filter.xml --- a/tools/fastq/fastq_filter.xml +++ b/tools/fastq/fastq_filter.xml @@ -96,7 +96,6 @@ return False #end for return True -ret_val = fastq_read_pass_filter( fastq_read ) </configfile></configfiles><outputs> https://bitbucket.org/galaxy/galaxy-central/commits/04c47872b79e/ Changeset: 04c47872b79e User: jmchilton Date: 2013-11-13 05:02:58 Summary: FASTQ Opt: Utilize optimized in place alternatives to restrict_scores_to_valid_range. Separate out ascii vs. decimal encoding branches of convert_read_to_format and use these new transform_ alternatives operate "in place" (don't produce new lists/allocate memory). The ascii version transforms to ascii in place instead of requiring another call and creating another array. Runtime Result: Kept 135848 of 250000 reads (54.34%). 117148711 function calls (116898711 primitive calls) in 48.272 seconds Down from 85.375 seconds on previous changeset. Main Difference: (ncalls|tottime|percall|cumtime|percall|filename:lineno(function)) 135848 1.185 0.000 59.856 0.000 fastq.py:71(convert_read_to_format) -became- 135848 0.502 0.000 22.979 0.000 fastq.py:99(convert_read_to_format) About a third the time is spent convert reads to the correct format. This is a varaint of the core optimization I made when optimizing the FASTQ groomer for MSI - it has likewise a substantial impact on the performance of that tool. Affected #: 1 file diff -r 046c608abfa31091f3e030db3979f848666fa602 -r 04c47872b79eea8dd0b745aed3e7324513653d78 lib/galaxy_utils/sequence/fastq.py --- a/lib/galaxy_utils/sequence/fastq.py +++ b/lib/galaxy_utils/sequence/fastq.py @@ -35,6 +35,34 @@ return max( min( score, cls.quality_max ), cls.quality_min ) return map( restrict_score, decimal_score_list ) @classmethod + def transform_scores_to_valid_range( cls, decimal_score_list): + cls_quality_max = cls.quality_max + cls_quality_min = cls.quality_min + for i in range( len( decimal_score_list ) ): + score = decimal_score_list[i] + if(score > cls_quality_max): + transformed_score = cls_quality_max + elif( score < cls_quality_min ): + transformed_score = cls_quality_min + else: + transformed_score = score + decimal_score_list[i] = str(transformed_score) + @classmethod + def transform_scores_to_valid_range_ascii( cls, decimal_score_list ): + cls_quality_max = cls.quality_max + cls_quality_min = cls.quality_min + to_quality = cls.ascii_min - cls.quality_min + for i in range( len( decimal_score_list ) ): + score = decimal_score_list[i] + if(score > cls_quality_max): + transformed_score = cls_quality_max + elif( score < cls_quality_min ): + transformed_score = cls_quality_min + else: + transformed_score = score + transformed_score = chr(transformed_score + to_quality) + decimal_score_list[i] = transformed_score + @classmethod def convert_base_to_color_space( cls, sequence ): return cls.color_space_converter.to_color_space( sequence ) @classmethod @@ -96,9 +124,14 @@ new_encoding = 'decimal' else: new_encoding = force_quality_encoding - new_read.quality = "%s " % " ".join( map( str, new_class.restrict_scores_to_valid_range( score_list ) ) ) #need trailing space to be valid decimal fastq if new_encoding == 'ascii': - new_read.quality = "".join( new_read.get_ascii_quality_scores() ) + new_class.transform_scores_to_valid_range_ascii( score_list ) + restricted_scores = map( str, score_list ) + new_read.quality = "".join( restricted_scores ) + else: # decimal + new_class.transform_scores_to_valid_range( score_list ) + restricted_scores = map( str, score_list ) + new_read.quality = "%s " % " ".join( restricted_scores ) #need trailing space to be valid decimal fastq return new_read def get_sequence( self ): return self.sequence https://bitbucket.org/galaxy/galaxy-central/commits/f54589589c1b/ Changeset: f54589589c1b User: jmchilton Date: 2013-11-13 05:02:58 Summary: FASTQ Opt: No need to map(str) over chr and join, can just join chr's. Prevent an extra array creation and chr-> str map per base. Shaves 10% off remaining run time of filter on SSD. Kept 135848 of 250000 reads (54.34%). 117012863 function calls (116762863 primitive calls) in 46.897 seconds Down from 48.272 seconds on previous changset. Main Difference: (ncalls|tottime|percall|cumtime|percall|filename:lineno(function)) 135848 0.502 0.000 22.979 0.000 fastq.py:99(convert_read_to_format) -became- 135848 0.448 0.000 21.159 0.000 fastq.py:99(convert_read_to_format) Not huge, but a consistent improvement. Affected #: 1 file diff -r 04c47872b79eea8dd0b745aed3e7324513653d78 -r f54589589c1b3882127aa96cc1d62b7eba9bef61 lib/galaxy_utils/sequence/fastq.py --- a/lib/galaxy_utils/sequence/fastq.py +++ b/lib/galaxy_utils/sequence/fastq.py @@ -1,4 +1,4 @@ -#Dan Blankenberg +##Dan Blankenberg import math import string import transform @@ -126,12 +126,10 @@ new_encoding = force_quality_encoding if new_encoding == 'ascii': new_class.transform_scores_to_valid_range_ascii( score_list ) - restricted_scores = map( str, score_list ) - new_read.quality = "".join( restricted_scores ) + new_read.quality = "".join( score_list ) else: # decimal new_class.transform_scores_to_valid_range( score_list ) - restricted_scores = map( str, score_list ) - new_read.quality = "%s " % " ".join( restricted_scores ) #need trailing space to be valid decimal fastq + new_read.quality = "%s " % " ".join( score_list ) #need trailing space to be valid decimal fastq return new_read def get_sequence( self ): return self.sequence https://bitbucket.org/galaxy/galaxy-central/commits/6d723e3bf2eb/ Changeset: 6d723e3bf2eb User: jmchilton Date: 2013-11-13 05:02:58 Summary: FASTQ Opt: Precompute difference ascii -> decimal difference. Kept 135848 of 250000 reads (54.34%). 117012863 function calls (116762863 primitive calls) in 39.416 seconds Down from 46.897 seconds on previous changeset. Main Difference WRT: (ncalls|tottime|percall|cumtime|percall|filename:lineno(function)) 385848 18.215 0.000 22.866 0.000 fastq.py:90(get_decimal_quality_scores) -became- 385848 11.415 0.000 15.546 0.000 fastq.py:91(get_decimal_quality_scores) Looks like a math optimization but actual optimization is coming from reading local variable instead of dereferencing object variable twice per base per. Affected #: 1 file diff -r f54589589c1b3882127aa96cc1d62b7eba9bef61 -r 6d723e3bf2ebddf0730e7b38761ca2b511d129c3 lib/galaxy_utils/sequence/fastq.py --- a/lib/galaxy_utils/sequence/fastq.py +++ b/lib/galaxy_utils/sequence/fastq.py @@ -82,14 +82,16 @@ quality = self.quality.rstrip() #decimal scores should have a trailing space if quality: try: - return [ chr( int( val ) + self.ascii_min - self.quality_min ) for val in quality.split() ] + to_quality = self.ascii_min - self.quality_min + return [ chr( int( val ) + to_quality ) for val in quality.split() ] except ValueError, e: raise ValueError( 'Error Parsing quality String. ASCII quality strings cannot contain spaces (%s): %s' % ( self.quality, e ) ) else: return [] def get_decimal_quality_scores( self ): if self.is_ascii_encoded(): - return [ ord( val ) - self.ascii_min + self.quality_min for val in self.quality ] + to_quality = self.quality_min - self.ascii_min + return [ ord( val ) + to_quality for val in self.quality ] else: quality = self.quality.rstrip() #decimal scores should have a trailing space if quality: https://bitbucket.org/galaxy/galaxy-central/commits/6265e769718e/ Changeset: 6265e769718e User: jmchilton Date: 2013-11-13 05:02:58 Summary: FASTQ Opt: Do not generate arrays just to check length. ... just compute what length would be. Kept 135848 of 250000 reads (54.34%). 117012863 function calls (116762863 primitive calls) in 38.412 seconds Down from 39.416 seconds on previous changeset. Main Difference: (ncalls|tottime|percall|cumtime|percall|filename:lineno(function)) 500000 0.518 0.000 1.889 0.000 fastq.py:171(insufficient_quality_length) 250000 0.322 0.000 1.103 0.000 fastq.py:173(assert_sequence_quality_lengths) - became - 500000 0.306 0.000 0.932 0.000 fastq.py:187(insufficient_quality_length) 250000 0.164 0.000 0.478 0.000 fastq.py:189(assert_sequence_quality_lengths) Affected #: 1 file diff -r 6d723e3bf2ebddf0730e7b38761ca2b511d129c3 -r 6265e769718e2487b4089c181948769cf67869ce lib/galaxy_utils/sequence/fastq.py --- a/lib/galaxy_utils/sequence/fastq.py +++ b/lib/galaxy_utils/sequence/fastq.py @@ -88,6 +88,22 @@ raise ValueError( 'Error Parsing quality String. ASCII quality strings cannot contain spaces (%s): %s' % ( self.quality, e ) ) else: return [] + def get_ascii_quality_scores_len( self ): + """ + Compute ascii quality score length, without generating relatively + expensive qualty score array. + """ + if self.is_ascii_encoded(): + return len( self.quality ) + else: + quality = self.quality.rstrip() + if quality: + try: + return len( quality.split() ) + except ValueError, e: + raise ValueError( 'Error Parsing quality String. ASCII quality strings cannot contain spaces (%s): %s' % ( self.quality, e ) ) + else: + return 0 def get_decimal_quality_scores( self ): if self.is_ascii_encoded(): to_quality = self.quality_min - self.ascii_min @@ -168,9 +184,9 @@ return False return True def insufficient_quality_length( self ): - return len( self.get_ascii_quality_scores() ) < len( self.sequence ) + return self.get_ascii_quality_scores_len() < len( self.sequence ) def assert_sequence_quality_lengths( self ): - qual_len = len( self.get_ascii_quality_scores() ) + qual_len = self.get_ascii_quality_scores_len() seq_len = len( self.sequence ) assert qual_len == seq_len, "Invalid FASTQ file: quality score length (%i) does not match sequence length (%i)" % ( qual_len, seq_len ) def reverse( self, clone = True ): @@ -235,11 +251,11 @@ return False def insufficient_quality_length( self ): if self.has_adapter_base(): - return len( self.get_ascii_quality_scores() ) + 1 < len( self.sequence ) + return self.get_ascii_quality_scores_len() + 1 < len( self.sequence ) return fastqSequencingRead.insufficient_quality_length( self ) def assert_sequence_quality_lengths( self ): if self.has_adapter_base(): - qual_len = len( self.get_ascii_quality_scores() ) + qual_len = self.get_ascii_quality_scores_len() seq_len = len( self.sequence ) assert ( qual_len + 1 == seq_len ) or ( qual_len == seq_len ), "Invalid FASTQ file: quality score length (%i) does not match sequence length (%i with adapter base)" % ( qual_len, seq_len ) #SRA adds FAKE/DUMMY quality scores to the adapter base, we'll allow the reading of the Improper score here, but remove it in the Reader when "apply_galaxy_conventions" is set to True else: https://bitbucket.org/galaxy/galaxy-central/commits/b5970d24670f/ Changeset: b5970d24670f User: jmchilton Date: 2013-11-13 05:02:58 Summary: FASTQ Opt: Eliminate a few extra calls to is_ascii_encoded. Kept 135848 of 250000 reads (54.34%). 117127015 function calls (116877015 primitive calls) in 38.632 seconds Down from 39.416 seconds on previous changeset. Main Difference (ncalls|tottime|percall|cumtime|percall|filename:lineno(function)) 385848 11.289 0.000 15.499 0.000 fastq.py:91(get_decimal_quality_scores) 1271696 0.716 0.000 0.716 0.000 fastq.py:71(is_ascii_encoded) 250000 0.185 0.000 10.434 0.000 fastq.py:107(get_decimal_quality_scores) 385848 11.380 0.000 15.368 0.000 fastq.py:109(__get_decimal_quality_scores) 1135848 0.649 0.000 0.649 0.000 fastq.py:71(is_ascii_encoded) Affected #: 1 file diff -r 6265e769718e2487b4089c181948769cf67869ce -r b5970d24670fefc0bf028bcab681f81ef2dd62e9 lib/galaxy_utils/sequence/fastq.py --- a/lib/galaxy_utils/sequence/fastq.py +++ b/lib/galaxy_utils/sequence/fastq.py @@ -105,7 +105,9 @@ else: return 0 def get_decimal_quality_scores( self ): - if self.is_ascii_encoded(): + return self.__get_decimal_quality_scores(self.is_ascii_encoded()) + def __get_decimal_quality_scores( self, ascii ): + if ascii: to_quality = self.quality_min - self.ascii_min return [ ord( val ) + to_quality for val in self.quality ] else: @@ -128,15 +130,16 @@ else: new_read.sequence = self.convert_color_to_base_space( self.sequence ) new_read.description = self.description + is_ascii = self.is_ascii_encoded() if self.score_system != new_read.score_system: if self.score_system == 'phred': - score_list = self.convert_score_phred_to_solexa( self.get_decimal_quality_scores() ) + score_list = self.convert_score_phred_to_solexa( self.__get_decimal_quality_scores(is_ascii) ) else: - score_list = self.convert_score_solexa_to_phred( self.get_decimal_quality_scores() ) + score_list = self.convert_score_solexa_to_phred( self.__get_decimal_quality_scores(is_ascii) ) else: - score_list = self.get_decimal_quality_scores() + score_list = self.__get_decimal_quality_scores(is_ascii) if force_quality_encoding is None: - if self.is_ascii_encoded(): + if is_ascii: new_encoding = 'ascii' else: new_encoding = 'decimal' https://bitbucket.org/galaxy/galaxy-central/commits/bb6c21a42274/ Changeset: bb6c21a42274 User: jmchilton Date: 2013-11-13 05:02:58 Summary: FASTQ Opt: Disable formatting output, reading is already enforcing it? This is would be huge optimization for filter at this point - it cuts the runtime of my filter test to half of what is remaining. This changeset if just leaving a comment because I am not certain the results are the same - though I strongly suspect they would be - I just want some confirmation before pulling the trigger. None of the test cases fail as a result of this or any of these changesets. Kept 135848 of 250000 reads (54.34%). 60407639 function calls in 18.837 seconds If change made, this would be down from 38.632 seconds on previous call. Main Difference: (ncalls|tottime|percall|cumtime|percall|filename:lineno(function)) 135848 0.357 0.000 19.710 0.000 fastq.py:631(write) - to - 135848 0.219 0.000 0.609 0.000 fastq.py:631(write) Affected #: 1 file diff -r b5970d24670fefc0bf028bcab681f81ef2dd62e9 -r bb6c21a422741b32ab78f7a4fe6dae571019d532 tools/fastq/fastq_filter.py --- a/tools/fastq/fastq_filter.py +++ b/tools/fastq/fastq_filter.py @@ -14,6 +14,8 @@ os.mkdir( additional_files_path ) shutil.copy( script_filename, os.path.join( additional_files_path, 'debug.txt' ) ) + ## Dan, Others: Can we simply drop the "format=input_type" here since it is specified in reader. + ## This optimization would cut runtime roughly in half (for my test case anyway). -John out = fastqWriter( open( output_filename, 'wb' ), format = input_type ) i = None https://bitbucket.org/galaxy/galaxy-central/commits/374783ca6a48/ Changeset: 374783ca6a48 User: dannon Date: 2013-12-17 16:42:49 Summary: Merged in jmchilton/galaxy-central-fork-1 (pull request #259) FASTQ Optimizations (Round 1 - Filtering) Affected #: 3 files diff -r 31179a01a1ec4176884926907b212259e4148319 -r 374783ca6a480fbb571bfe9eb46052667b9d0479 lib/galaxy_utils/sequence/fastq.py --- a/lib/galaxy_utils/sequence/fastq.py +++ b/lib/galaxy_utils/sequence/fastq.py @@ -1,4 +1,4 @@ -#Dan Blankenberg +##Dan Blankenberg import math import string import transform @@ -35,6 +35,34 @@ return max( min( score, cls.quality_max ), cls.quality_min ) return map( restrict_score, decimal_score_list ) @classmethod + def transform_scores_to_valid_range( cls, decimal_score_list): + cls_quality_max = cls.quality_max + cls_quality_min = cls.quality_min + for i in range( len( decimal_score_list ) ): + score = decimal_score_list[i] + if(score > cls_quality_max): + transformed_score = cls_quality_max + elif( score < cls_quality_min ): + transformed_score = cls_quality_min + else: + transformed_score = score + decimal_score_list[i] = str(transformed_score) + @classmethod + def transform_scores_to_valid_range_ascii( cls, decimal_score_list ): + cls_quality_max = cls.quality_max + cls_quality_min = cls.quality_min + to_quality = cls.ascii_min - cls.quality_min + for i in range( len( decimal_score_list ) ): + score = decimal_score_list[i] + if(score > cls_quality_max): + transformed_score = cls_quality_max + elif( score < cls_quality_min ): + transformed_score = cls_quality_min + else: + transformed_score = score + transformed_score = chr(transformed_score + to_quality) + decimal_score_list[i] = transformed_score + @classmethod def convert_base_to_color_space( cls, sequence ): return cls.color_space_converter.to_color_space( sequence ) @classmethod @@ -54,14 +82,34 @@ quality = self.quality.rstrip() #decimal scores should have a trailing space if quality: try: - return [ chr( int( val ) + self.ascii_min - self.quality_min ) for val in quality.split() ] + to_quality = self.ascii_min - self.quality_min + return [ chr( int( val ) + to_quality ) for val in quality.split() ] except ValueError, e: raise ValueError( 'Error Parsing quality String. ASCII quality strings cannot contain spaces (%s): %s' % ( self.quality, e ) ) else: return [] + def get_ascii_quality_scores_len( self ): + """ + Compute ascii quality score length, without generating relatively + expensive qualty score array. + """ + if self.is_ascii_encoded(): + return len( self.quality ) + else: + quality = self.quality.rstrip() + if quality: + try: + return len( quality.split() ) + except ValueError, e: + raise ValueError( 'Error Parsing quality String. ASCII quality strings cannot contain spaces (%s): %s' % ( self.quality, e ) ) + else: + return 0 def get_decimal_quality_scores( self ): - if self.is_ascii_encoded(): - return [ ord( val ) - self.ascii_min + self.quality_min for val in self.quality ] + return self.__get_decimal_quality_scores(self.is_ascii_encoded()) + def __get_decimal_quality_scores( self, ascii ): + if ascii: + to_quality = self.quality_min - self.ascii_min + return [ ord( val ) + to_quality for val in self.quality ] else: quality = self.quality.rstrip() #decimal scores should have a trailing space if quality: @@ -82,23 +130,27 @@ else: new_read.sequence = self.convert_color_to_base_space( self.sequence ) new_read.description = self.description + is_ascii = self.is_ascii_encoded() if self.score_system != new_read.score_system: if self.score_system == 'phred': - score_list = self.convert_score_phred_to_solexa( self.get_decimal_quality_scores() ) + score_list = self.convert_score_phred_to_solexa( self.__get_decimal_quality_scores(is_ascii) ) else: - score_list = self.convert_score_solexa_to_phred( self.get_decimal_quality_scores() ) + score_list = self.convert_score_solexa_to_phred( self.__get_decimal_quality_scores(is_ascii) ) else: - score_list = self.get_decimal_quality_scores() - new_read.quality = "%s " % " ".join( map( str, new_class.restrict_scores_to_valid_range( score_list ) ) ) #need trailing space to be valid decimal fastq + score_list = self.__get_decimal_quality_scores(is_ascii) if force_quality_encoding is None: - if self.is_ascii_encoded(): + if is_ascii: new_encoding = 'ascii' else: new_encoding = 'decimal' else: new_encoding = force_quality_encoding if new_encoding == 'ascii': - new_read.quality = "".join( new_read.get_ascii_quality_scores() ) + new_class.transform_scores_to_valid_range_ascii( score_list ) + new_read.quality = "".join( score_list ) + else: # decimal + new_class.transform_scores_to_valid_range( score_list ) + new_read.quality = "%s " % " ".join( score_list ) #need trailing space to be valid decimal fastq return new_read def get_sequence( self ): return self.sequence @@ -135,9 +187,9 @@ return False return True def insufficient_quality_length( self ): - return len( self.get_ascii_quality_scores() ) < len( self.sequence ) + return self.get_ascii_quality_scores_len() < len( self.sequence ) def assert_sequence_quality_lengths( self ): - qual_len = len( self.get_ascii_quality_scores() ) + qual_len = self.get_ascii_quality_scores_len() seq_len = len( self.sequence ) assert qual_len == seq_len, "Invalid FASTQ file: quality score length (%i) does not match sequence length (%i)" % ( qual_len, seq_len ) def reverse( self, clone = True ): @@ -202,11 +254,11 @@ return False def insufficient_quality_length( self ): if self.has_adapter_base(): - return len( self.get_ascii_quality_scores() ) + 1 < len( self.sequence ) + return self.get_ascii_quality_scores_len() + 1 < len( self.sequence ) return fastqSequencingRead.insufficient_quality_length( self ) def assert_sequence_quality_lengths( self ): if self.has_adapter_base(): - qual_len = len( self.get_ascii_quality_scores() ) + qual_len = self.get_ascii_quality_scores_len() seq_len = len( self.sequence ) assert ( qual_len + 1 == seq_len ) or ( qual_len == seq_len ), "Invalid FASTQ file: quality score length (%i) does not match sequence length (%i with adapter base)" % ( qual_len, seq_len ) #SRA adds FAKE/DUMMY quality scores to the adapter base, we'll allow the reading of the Improper score here, but remove it in the Reader when "apply_galaxy_conventions" is set to True else: diff -r 31179a01a1ec4176884926907b212259e4148319 -r 374783ca6a480fbb571bfe9eb46052667b9d0479 tools/fastq/fastq_filter.py --- a/tools/fastq/fastq_filter.py +++ b/tools/fastq/fastq_filter.py @@ -14,14 +14,16 @@ os.mkdir( additional_files_path ) shutil.copy( script_filename, os.path.join( additional_files_path, 'debug.txt' ) ) + ## Dan, Others: Can we simply drop the "format=input_type" here since it is specified in reader. + ## This optimization would cut runtime roughly in half (for my test case anyway). -John out = fastqWriter( open( output_filename, 'wb' ), format = input_type ) i = None reads_kept = 0 + execfile(script_filename, globals()) for i, fastq_read in enumerate( fastqReader( open( input_filename ), format = input_type ) ): - local = {'fastq_read':fastq_read, 'ret_val':False} - execfile( script_filename, {}, local ) - if local['ret_val']: + ret_val = fastq_read_pass_filter( fastq_read ) ## fastq_read_pass_filter defined in script_filename + if ret_val: out.write( fastq_read ) reads_kept += 1 out.close() diff -r 31179a01a1ec4176884926907b212259e4148319 -r 374783ca6a480fbb571bfe9eb46052667b9d0479 tools/fastq/fastq_filter.xml --- a/tools/fastq/fastq_filter.xml +++ b/tools/fastq/fastq_filter.xml @@ -96,7 +96,6 @@ return False #end for return True -ret_val = fastq_read_pass_filter( fastq_read ) </configfile></configfiles><outputs> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 17 Dec '13

17 Dec '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f6723d1ddaf5/ Changeset: f6723d1ddaf5 Branch: yhoogstrate/some-minor-select-field-changes-includin-1387270048132 User: yhoogstrate Date: 2013-12-17 09:49:21 Summary: Some minor select field changes including Illumina 1.8+ Affected #: 1 file diff -r af87cdcf36084831fce7d99e621b1efc943d2aff -r f6723d1ddaf5c15e194aa5bde2fc78937b9c5423 tools/fastq/fastq_groomer.xml --- a/tools/fastq/fastq_groomer.xml +++ b/tools/fastq/fastq_groomer.xml @@ -17,7 +17,7 @@ <param name="input_type" type="select" label="Input FASTQ quality scores type"><option value="solexa">Solexa</option><option value="illumina">Illumina 1.3-1.7</option> - <option value="sanger" selected="True">Sanger</option> + <option value="sanger" selected="True">Sanger & Illumina 1.8+</option><option value="cssanger">Color Space Sanger</option></param><conditional name="options_type"> @@ -31,7 +31,7 @@ <when value="advanced"><param name="output_type" type="select" label="Output FASTQ quality scores type" help="Galaxy tools are designed to work with the Sanger Quality score format."><option value="solexa">Solexa</option> - <option value="illumina">Illumina 1.3+</option> + <option value="illumina">Illumina 1.3-1.7</option><option value="sanger" selected="True">Sanger (recommended)</option><option value="cssanger">Color Space Sanger</option></param> https://bitbucket.org/galaxy/galaxy-central/commits/31179a01a1ec/ Changeset: 31179a01a1ec User: dannon Date: 2013-12-17 16:18:22 Summary: Merged in yhoogstrate/galaxy-central/yhoogstrate/some-minor-select-field-changes-includin-1387270048132 (pull request #276) Some minor select field changes including Illumina 1.8+ Affected #: 1 file diff -r 029cc5c14f1a8fb81e4bbdc3872757b297e1efa0 -r 31179a01a1ec4176884926907b212259e4148319 tools/fastq/fastq_groomer.xml --- a/tools/fastq/fastq_groomer.xml +++ b/tools/fastq/fastq_groomer.xml @@ -17,7 +17,7 @@ <param name="input_type" type="select" label="Input FASTQ quality scores type"><option value="solexa">Solexa</option><option value="illumina">Illumina 1.3-1.7</option> - <option value="sanger" selected="True">Sanger</option> + <option value="sanger" selected="True">Sanger & Illumina 1.8+</option><option value="cssanger">Color Space Sanger</option></param><conditional name="options_type"> @@ -31,7 +31,7 @@ <when value="advanced"><param name="output_type" type="select" label="Output FASTQ quality scores type" help="Galaxy tools are designed to work with the Sanger Quality score format."><option value="solexa">Solexa</option> - <option value="illumina">Illumina 1.3+</option> + <option value="illumina">Illumina 1.3-1.7</option><option value="sanger" selected="True">Sanger (recommended)</option><option value="cssanger">Color Space Sanger</option></param> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Handle strangely defined tool tests when setting metadata in tool shed repositories.
by commits-noreply＠bitbucket.org 17 Dec '13

17 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/029cc5c14f1a/ Changeset: 029cc5c14f1a User: greg Date: 2013-12-17 16:08:03 Summary: Handle strangely defined tool tests when setting metadata in tool shed repositories. Affected #: 2 files diff -r 97c69fcfeaac1756a7ad5d92c8ade74b2d611561 -r 029cc5c14f1a8fb81e4bbdc3872757b297e1efa0 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -927,10 +927,13 @@ guid = suc.generate_tool_guid( repository_clone_url, tool ) # Handle tool.requirements. tool_requirements = [] - for tr in tool.requirements: - requirement_dict = dict( name=tr.name, - type=tr.type, - version=tr.version ) + for tool_requirement in tool.requirements: + name = str( tool_requirement.name ) + type = str( tool_requirement.type ) + version = str( tool_requirement.version ) if tool_requirement.version else None + requirement_dict = dict( name=name, + type=type, + version=version ) tool_requirements.append( requirement_dict ) # Handle tool.tests. tool_tests = [] @@ -941,18 +944,31 @@ value, extra = required_file required_files.append( ( value ) ) inputs = [] - for input_field, values in ttb.inputs.iteritems(): - if len( values ) == 1: - inputs.append( ( input_field, values[0] ) ) - else: - inputs.append( ( input_field, values ) ) + for param_name, values in ttb.inputs.iteritems(): + # Handle improperly defined or strange test parameters and values. + if param_name is not None: + if values is None: + # An example is the 3rd test in http://testtoolshed.g2.bx.psu.edu/view/devteam/samtools_rmdup + # which is defined as: + # <test> + # <param name="input1" value="1.bam" ftype="bam" /> + # <param name="bam_paired_end_type_selector" value="PE" /> + # <param name="force_se" /> + # <output name="output1" file="1.bam" ftype="bam" sort="True" /> + # </test> + inputs.append( ( param_name, values ) ) + else: + if len( values ) == 1: + inputs.append( ( param_name, values[ 0 ] ) ) + else: + inputs.append( ( param_name, values ) ) outputs = [] for output in ttb.outputs: name, file_name, extra = output outputs.append( ( name, suc.strip_path( file_name ) if file_name else None ) ) if file_name not in required_files and file_name is not None: required_files.append( file_name ) - test_dict = dict( name=ttb.name, + test_dict = dict( name=str( ttb.name ), required_files=required_files, inputs=inputs, outputs=outputs ) diff -r 97c69fcfeaac1756a7ad5d92c8ade74b2d611561 -r 029cc5c14f1a8fb81e4bbdc3872757b297e1efa0 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -254,11 +254,8 @@ else: deleted = False uninstalled = False - tool_shed_repository = get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( app, - tool_shed, - name, - owner, - installed_changeset_revision ) + tool_shed_repository = \ + get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( app, tool_shed, name, owner, installed_changeset_revision ) if tool_shed_repository: log.debug( "Updating an existing row for repository '%s' in the tool_shed_repository table, status set to '%s'." % ( str( name ), str( status ) ) ) tool_shed_repository.description = description @@ -271,19 +268,20 @@ tool_shed_repository.status = status else: log.debug( "Adding new row for repository '%s' in the tool_shed_repository table, status set to '%s'." % ( str( name ), str( status ) ) ) - tool_shed_repository = app.install_model.ToolShedRepository( tool_shed=tool_shed, - name=name, - description=description, - owner=owner, - installed_changeset_revision=installed_changeset_revision, - changeset_revision=current_changeset_revision, - ctx_rev=ctx_rev, - metadata=metadata_dict, - includes_datatypes=includes_datatypes, - dist_to_shed=dist_to_shed, - deleted=deleted, - uninstalled=uninstalled, - status=status ) + tool_shed_repository = \ + app.install_model.ToolShedRepository( tool_shed=tool_shed, + name=name, + description=description, + owner=owner, + installed_changeset_revision=installed_changeset_revision, + changeset_revision=current_changeset_revision, + ctx_rev=ctx_rev, + metadata=metadata_dict, + includes_datatypes=includes_datatypes, + dist_to_shed=dist_to_shed, + deleted=deleted, + uninstalled=uninstalled, + status=status ) context.add( tool_shed_repository ) context.flush() return tool_shed_repository Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 17 Dec '13

17 Dec '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/08320798463e/ Changeset: 08320798463e User: nsoranzo Date: 2013-12-06 11:17:25 Summary: Specify mosaik v. 1.1.0021 (last compatible with the wrapper). Add requirement for samtools. Affected #: 1 file diff -r a29d54619813d5da992b897557162a360b8d610c -r 08320798463e173f61f0b53fbd2db43419d373e2 tools/sr_mapping/mosaik.xml --- a/tools/sr_mapping/mosaik.xml +++ b/tools/sr_mapping/mosaik.xml @@ -1,7 +1,10 @@ <?xml version="1.0"?><tool id="mosaik_wrapper" name="Map with Mosaik" version="1.1.1"><description/> - <requirements><requirement type="package">mosaik</requirement></requirements> + <requirements> + <requirement type="package" version="1.1.0021">mosaik</requirement> + <requirement type="package" version="0.1.18">samtools</requirement> + </requirements><command> #set $processors = '-p ${GALAXY_SLOTS:-4}' #set $lm = '' https://bitbucket.org/galaxy/galaxy-central/commits/5fbc02eff708/ Changeset: 5fbc02eff708 User: nsoranzo Date: 2013-12-06 11:20:56 Summary: Fix for failure preparing job: "NotFound: cannot find 'mfl'" with paired-end library. Add mate FASTQ file to inputs. 'ls' param was not used. Add missing '-mfl ' to MosaikBuild command. Remove always empty $lm variable. Add <version_command>. Affected #: 1 file diff -r 08320798463e173f61f0b53fbd2db43419d373e2 -r 5fbc02eff708cc86afd60a4fa7588c9f72e877b5 tools/sr_mapping/mosaik.xml --- a/tools/sr_mapping/mosaik.xml +++ b/tools/sr_mapping/mosaik.xml @@ -1,28 +1,30 @@ <?xml version="1.0"?> -<tool id="mosaik_wrapper" name="Map with Mosaik" version="1.1.1"> +<tool id="mosaik_wrapper" name="Map with Mosaik" version="1.1.2"><description/><requirements><requirement type="package" version="1.1.0021">mosaik</requirement><requirement type="package" version="0.1.18">samtools</requirement></requirements> + <version_command>MosaikAligner | sed -e 's/\x1b\[[[:digit:]]\{1,2\}$;[[:digit:]]\{1,2\}$\{0,1\}m//g' | grep -o 'MosaikAligner [[:digit:].]\{1,\}'</version_command><command> #set $processors = '-p ${GALAXY_SLOTS:-4}' - #set $lm = '' - #if $paired.kind == 'single': - #set $mfl = '' - #set $ls = '' - #else: - #set $ls = '-ls $mfl' - #end if MosaikBuild -fr #if $genomeSource.refGenomeSource == 'indexed': ${genomeSource.indexReference.fields.path} #else: - $genomeSource.historyReference + ${genomeSource.historyReference} #end if - -oa mosaik_ref_file; - MosaikBuild -q $reads $mfl -st $st -out mosaik_reads_file; - MosaikAligner -ia mosaik_ref_file -in mosaik_reads_file -out mosaik_aligned_file $ls -mm $mm -mhp $mhp -act $act -bw $bw $processors $lm -hs 15; + -oa mosaik_ref_file; + MosaikBuild -q $reads + #if $paired.kind == 'single' + #set $ls_string = '' + #else + -q2 ${paired.reads2} + -mfl ${paired.mfl} + #set $ls_string = '-ls %s' % $paired.ls + #end if + -st $st -out mosaik_reads_file; + MosaikAligner -ia mosaik_ref_file -in mosaik_reads_file -out mosaik_aligned_file $ls_string -mm $mm -mhp $mhp -act $act -bw $bw $processors -hs 15; MosaikText -in mosaik_aligned_file -$outFormat sam_bam_file; #if str($outFormat) == 'bam': samtools sort sam_bam_file sorted_bam; @@ -50,12 +52,12 @@ <param format="fasta" name="historyReference" type="data" metadata_name="dbkey" label="Select a reference from history"/></when></conditional> - <param format="fastq" name="reads" type="data" label="Fastq Reads File"/> - <param name="outFormat" type="select" label="Output Format"> - <option value="sam">Sam</option> - <option value="bam">Bam</option> + <param format="fastq" name="reads" type="data" label="FASTQ reads file" /> + <param name="outFormat" type="select" label="Output format"> + <option value="sam">SAM</option> + <option value="bam">BAM</option></param> - <param name="st" type="select" label="Sequencing Technology Used"> + <param name="st" type="select" label="Sequencing technology used"><option value="454">454</option><option value="illumina">Illumina</option><option value="solid">Solid</option> @@ -69,14 +71,15 @@ </param><when value="single"/><when value="paired"> - <param name="mfl" type="integer" value="200" label="Insert Size" help="the length between the paired reads"/> - <param name="ls" type="integer" value="50" label="Realignment Window" help="Window size to realign mate pairs that are out of position. Large values slow down performance"/> + <param format="fastq" name="reads2" type="data" label="FASTQ 2nd mate" /> + <param name="mfl" type="integer" value="200" label="Median fragment length" /> + <param name="ls" type="integer" min="0" value="50" label="Local alignment search radius to rescue mates" help="A large value slows down performances" /></when></conditional> - <param name="mm" size="5" type="integer" value="6" label="Mismatches allowed" help="mismatches allowed per sequence"/> - <param name="act" size="5" type="integer" value="35" label="Alignment Candidate Threshold" help="determines which hash regions will be aligned with Smith Waterman"/> - <param name="bw" size="5" type="integer" value="19" label="Smith-Waterman band width"/> - <param name="mhp" size="5" type="integer" value="100" label="Maximum # Of Positions Stored Per Seed" help="number of places in the reference the aligner will try to place a particular hash"/> + <param name="mm" type="integer" value="6" label="Number of mismatches allowed per sequence" /> + <param name="act" type="integer" value="35" label="Alignment candidate threshold" help="Determines which hash regions will be aligned with Smith-Waterman" /> + <param name="bw" type="integer" value="9" label="Smith-Waterman band width" /> + <param name="mhp" type="integer" value="100" label="Maximum number of positions stored per seed" help="Number of places in the reference the aligner will try to place a particular hash" /></inputs><outputs><data format="sam" name="output"> https://bitbucket.org/galaxy/galaxy-central/commits/97c69fcfeaac/ Changeset: 97c69fcfeaac User: dannon Date: 2013-12-17 15:00:44 Summary: Merged in nsoranzo/galaxy-central (pull request #271) Fixes for MOSAIK tool Affected #: 1 file diff -r 3eb13c85bf4140448865328b9c6feaed96d8682f -r 97c69fcfeaac1756a7ad5d92c8ade74b2d611561 tools/sr_mapping/mosaik.xml --- a/tools/sr_mapping/mosaik.xml +++ b/tools/sr_mapping/mosaik.xml @@ -1,25 +1,30 @@ <?xml version="1.0"?> -<tool id="mosaik_wrapper" name="Map with Mosaik" version="1.1.1"> +<tool id="mosaik_wrapper" name="Map with Mosaik" version="1.1.2"><description/> - <requirements><requirement type="package">mosaik</requirement></requirements> + <requirements> + <requirement type="package" version="1.1.0021">mosaik</requirement> + <requirement type="package" version="0.1.18">samtools</requirement> + </requirements> + <version_command>MosaikAligner | sed -e 's/\x1b\[[[:digit:]]\{1,2\}$;[[:digit:]]\{1,2\}$\{0,1\}m//g' | grep -o 'MosaikAligner [[:digit:].]\{1,\}'</version_command><command> #set $processors = '-p ${GALAXY_SLOTS:-4}' - #set $lm = '' - #if $paired.kind == 'single': - #set $mfl = '' - #set $ls = '' - #else: - #set $ls = '-ls $mfl' - #end if MosaikBuild -fr #if $genomeSource.refGenomeSource == 'indexed': ${genomeSource.indexReference.fields.path} #else: - $genomeSource.historyReference + ${genomeSource.historyReference} #end if - -oa mosaik_ref_file; - MosaikBuild -q $reads $mfl -st $st -out mosaik_reads_file; - MosaikAligner -ia mosaik_ref_file -in mosaik_reads_file -out mosaik_aligned_file $ls -mm $mm -mhp $mhp -act $act -bw $bw $processors $lm -hs 15; + -oa mosaik_ref_file; + MosaikBuild -q $reads + #if $paired.kind == 'single' + #set $ls_string = '' + #else + -q2 ${paired.reads2} + -mfl ${paired.mfl} + #set $ls_string = '-ls %s' % $paired.ls + #end if + -st $st -out mosaik_reads_file; + MosaikAligner -ia mosaik_ref_file -in mosaik_reads_file -out mosaik_aligned_file $ls_string -mm $mm -mhp $mhp -act $act -bw $bw $processors -hs 15; MosaikText -in mosaik_aligned_file -$outFormat sam_bam_file; #if str($outFormat) == 'bam': samtools sort sam_bam_file sorted_bam; @@ -47,12 +52,12 @@ <param format="fasta" name="historyReference" type="data" metadata_name="dbkey" label="Select a reference from history"/></when></conditional> - <param format="fastq" name="reads" type="data" label="Fastq Reads File"/> - <param name="outFormat" type="select" label="Output Format"> - <option value="sam">Sam</option> - <option value="bam">Bam</option> + <param format="fastq" name="reads" type="data" label="FASTQ reads file" /> + <param name="outFormat" type="select" label="Output format"> + <option value="sam">SAM</option> + <option value="bam">BAM</option></param> - <param name="st" type="select" label="Sequencing Technology Used"> + <param name="st" type="select" label="Sequencing technology used"><option value="454">454</option><option value="illumina">Illumina</option><option value="solid">Solid</option> @@ -66,14 +71,15 @@ </param><when value="single"/><when value="paired"> - <param name="mfl" type="integer" value="200" label="Insert Size" help="the length between the paired reads"/> - <param name="ls" type="integer" value="50" label="Realignment Window" help="Window size to realign mate pairs that are out of position. Large values slow down performance"/> + <param format="fastq" name="reads2" type="data" label="FASTQ 2nd mate" /> + <param name="mfl" type="integer" value="200" label="Median fragment length" /> + <param name="ls" type="integer" min="0" value="50" label="Local alignment search radius to rescue mates" help="A large value slows down performances" /></when></conditional> - <param name="mm" size="5" type="integer" value="6" label="Mismatches allowed" help="mismatches allowed per sequence"/> - <param name="act" size="5" type="integer" value="35" label="Alignment Candidate Threshold" help="determines which hash regions will be aligned with Smith Waterman"/> - <param name="bw" size="5" type="integer" value="19" label="Smith-Waterman band width"/> - <param name="mhp" size="5" type="integer" value="100" label="Maximum # Of Positions Stored Per Seed" help="number of places in the reference the aligner will try to place a particular hash"/> + <param name="mm" type="integer" value="6" label="Number of mismatches allowed per sequence" /> + <param name="act" type="integer" value="35" label="Alignment candidate threshold" help="Determines which hash regions will be aligned with Smith-Waterman" /> + <param name="bw" type="integer" value="9" label="Smith-Waterman band width" /> + <param name="mhp" type="integer" value="100" label="Maximum number of positions stored per seed" help="Number of places in the reference the aligner will try to place a particular hash" /></inputs><outputs><data format="sam" name="output"> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 17 Dec '13

17 Dec '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b92f61b482d9/ Changeset: b92f61b482d9 User: BjoernGruening Date: 2013-11-11 10:09:54 Summary: add dependency of type package to the ccat wrapper Affected #: 1 file diff -r 8e448f2ac955e6d434476b18c59b2cffcfc55404 -r b92f61b482d9b5897cb78ac260ae1dcec3260045 tools/peak_calling/ccat_wrapper.xml --- a/tools/peak_calling/ccat_wrapper.xml +++ b/tools/peak_calling/ccat_wrapper.xml @@ -10,6 +10,7 @@ '$output_peak_file' '$output_region_file' '$output_top_file' '$output_log_file'</command><requirements><requirement type="binary" version="3.0">CCAT</requirement> + <requirement type="package" version="3.0">CCAT</requirement></requirements><inputs><param name="input_tag_file" type="data" format="bed" label="ChIP-Seq Tag File" > https://bitbucket.org/galaxy/galaxy-central/commits/3eb13c85bf41/ Changeset: 3eb13c85bf41 User: BjoernGruening Date: 2013-11-11 15:35:45 Summary: remove binary dependency Affected #: 1 file diff -r b92f61b482d9b5897cb78ac260ae1dcec3260045 -r 3eb13c85bf4140448865328b9c6feaed96d8682f tools/peak_calling/ccat_wrapper.xml --- a/tools/peak_calling/ccat_wrapper.xml +++ b/tools/peak_calling/ccat_wrapper.xml @@ -9,7 +9,6 @@ 'CCAT in Galaxy' '$output_peak_file' '$output_region_file' '$output_top_file' '$output_log_file'</command><requirements> - <requirement type="binary" version="3.0">CCAT</requirement><requirement type="package" version="3.0">CCAT</requirement></requirements><inputs> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 17 Dec '13

17 Dec '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/714f5b1ac790/ Changeset: 714f5b1ac790 User: jmchilton Date: 2013-12-15 23:16:34 Summary: Move history contents filtering logic into model. Filtering for deleted, visible, and ids with ORM should prevent loading unneeded objects into memory (a history with 1000 items and 5 visible will now only cause 5 items to be loaded instead of 1000 just to filter out 5). Simplifies history_contents API logic somewhat, allows easier 'unit' testing (included), and provides a clearer entry point for additional 'showing' additional contents types (read dataset collections). Affected #: 3 files diff -r 4fe46fd3181bfb63e8093200d71801cb507b586d -r 714f5b1ac7902cc5b79d5fdf6390ea70b7e17afb lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -19,6 +19,7 @@ import socket import time from string import Template +from itertools import ifilter import galaxy.datatypes import galaxy.datatypes.registry @@ -43,6 +44,11 @@ # Default Value Required for unit tests datatypes_registry.load_datatypes() +# When constructing filters with in for a fixed set of ids, maximum +# number of items to place in the IN statement. Different databases +# are going to have different limits so it is likely best to not let +# this be unlimited - filter in Python if over this limit. +MAX_IN_FILTER_LENGTH = 100 class NoConverterException(Exception): def __init__(self, value): @@ -893,6 +899,32 @@ rval = galaxy.datatypes.data.nice_size( rval ) return rval + def contents_iter( self, **kwds ): + """ + Fetch filtered list of contents of history. + """ + python_filter = None + db_session = object_session( self ) + assert db_session != None + query = db_session.query( HistoryDatasetAssociation ).filter( HistoryDatasetAssociation.table.c.history_id == self.id ) + deleted = galaxy.util.string_as_bool_or_none( kwds.get( 'deleted', None ) ) + if deleted is not None: + query = query.filter( HistoryDatasetAssociation.deleted == bool( kwds['deleted'] ) ) + visible = galaxy.util.string_as_bool_or_none( kwds.get( 'visible', None ) ) + if visible is not None: + query = query.filter( HistoryDatasetAssociation.visible == bool( kwds['visible'] ) ) + if 'ids' in kwds: + ids = kwds['ids'] + max_in_filter_length = kwds.get('max_in_filter_length', MAX_IN_FILTER_LENGTH) + if len(ids) < max_in_filter_length: + query = query.filter( HistoryDatasetAssociation.id.in_(ids) ) + else: + python_filter = lambda hda: hda.id in ids + if python_filter: + return ifilter(python_filter, query) + else: + return query + def copy_tags_from(self,target_user,source_history): for src_shta in source_history.tags: new_shta = src_shta.copy() diff -r 4fe46fd3181bfb63e8093200d71801cb507b586d -r 714f5b1ac7902cc5b79d5fdf6390ea70b7e17afb lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -51,47 +51,28 @@ else: history = self.get_history( trans, history_id, check_ownership=True, check_accessible=True ) - # if ids, return _FULL_ data (as show) for each id passed + contents_kwds = {} if ids: - ids = ids.split( ',' ) - for index, hda in enumerate( history.datasets ): - encoded_hda_id = trans.security.encode_id( hda.id ) - if encoded_hda_id in ids: - #TODO: share code with show - rval.append( self._detailed_hda_dict( trans, hda ) ) - - # if no ids passed, return a _SUMMARY_ of _all_ datasets in the history + ids = map( lambda id: trans.security.decode_id( id ), ids.split( ',' ) ) + contents_kwds[ 'ids' ] = ids + # If explicit ids given, always used detailed result. + details = 'all' else: + contents_kwds[ 'deleted' ] = kwd.get( 'deleted', None ) + contents_kwds[ 'visible' ] = kwd.get( 'visible', None ) # details param allows a mixed set of summary and detailed hdas #TODO: this is getting convoluted due to backwards compat details = kwd.get( 'details', None ) or [] if details and details != 'all': details = util.listify( details ) - # by default return all datasets - even if deleted or hidden (defaulting the next switches to None) - # if specified return those datasets that match the setting - # backwards compat - return_deleted = util.string_as_bool_or_none( kwd.get( 'deleted', None ) ) - return_visible = util.string_as_bool_or_none( kwd.get( 'visible', None ) ) - - for hda in history.datasets: - # if either return_ setting has been requested (!= None), skip hdas that don't match the request - if return_deleted is not None: - if( ( return_deleted and not hda.deleted ) - or ( not return_deleted and hda.deleted ) ): - continue - if return_visible is not None: - if( ( return_visible and not hda.visible ) - or ( not return_visible and hda.visible ) ): - continue - - encoded_hda_id = trans.security.encode_id( hda.id ) - if( ( encoded_hda_id in details ) - or ( details == 'all' ) ): - rval.append( self._detailed_hda_dict( trans, hda ) ) - else: - rval.append( self._summary_hda_dict( trans, history_id, hda ) ) - + for hda in history.contents_iter( **contents_kwds ): + encoded_hda_id = trans.security.encode_id( hda.id ) + detailed = details == 'all' or ( encoded_hda_id in details ) + if detailed: + rval.append( self._detailed_hda_dict( trans, hda ) ) + else: + rval.append( self._summary_hda_dict( trans, history_id, hda ) ) except Exception, e: # for errors that are not specific to one hda (history lookup or summary list) rval = "Error in history API at listing contents: " + str( e ) diff -r 4fe46fd3181bfb63e8093200d71801cb507b586d -r 714f5b1ac7902cc5b79d5fdf6390ea70b7e17afb test/unit/test_galaxy_mapping.py --- a/test/unit/test_galaxy_mapping.py +++ b/test/unit/test_galaxy_mapping.py @@ -180,6 +180,39 @@ assert hist1.name == "History 2b" # gvk TODO need to ad test for GalaxySessions, but not yet sure what they should look like. + def test_history_contents( self ): + model = self.model + u = model.User( email="contents(a)foo.bar.baz", password="password" ) + # gs = model.GalaxySession() + h1 = model.History( name="HistoryContentsHistory1", user=u) + + self.persist( u, h1, expunge=True ) + + d1 = self.new_hda( h1, name="1" ) + d2 = self.new_hda( h1, name="2", visible=False ) + d3 = self.new_hda( h1, name="3", deleted=True ) + d4 = self.new_hda( h1, name="4", visible=False, deleted=True ) + + def contents_iter_names(**kwds): + history = model.context.query( model.History ).filter( + model.History.name == "HistoryContentsHistory1" + ).first() + return set( map( lambda hda: hda.name, history.contents_iter( **kwds ) ) ) + + assert contents_iter_names() == set( [ "1", "2", "3", "4" ] ) + assert contents_iter_names( deleted=False ) == set( [ "1", "2" ] ) + assert contents_iter_names( visible=True ) == set( [ "1", "3" ] ) + assert contents_iter_names( visible=False ) == set( [ "2", "4" ] ) + assert contents_iter_names( deleted=True, visible=False ) == set( [ "4" ] ) + + assert contents_iter_names( ids=[ d1.id, d2.id, d3.id, d4.id ] ) == set( [ "1", "2", "3", "4" ] ) + assert contents_iter_names( ids=[ d1.id, d2.id, d3.id, d4.id ], max_in_filter_length=1 ) == set( [ "1", "2", "3", "4" ] ) + + assert contents_iter_names( ids=[ d1.id, d3.id ] ) == set( [ "1", "3" ] ) + + def new_hda( self, history, **kwds ): + return self.persist( self.model.HistoryDatasetAssociation( history=history, create_dataset=True, sa_session=self.model.session, **kwds ), flush=True ) + @classmethod def setUpClass(cls): # Start the database and connect the mapping @@ -191,10 +224,16 @@ return cls.model.session.query( type ) @classmethod - def persist(cls, *args): + def persist(cls, *args, **kwargs): + session = cls.model.session + flush = kwargs.get('flush', True) for arg in args: - cls.model.session.add( arg ) - cls.expunge() + session.add( arg ) + if flush: + session.flush() + if kwargs.get('expunge', not flush): + cls.expunge() + return arg # Return last or only arg. @classmethod def expunge(cls): https://bitbucket.org/galaxy/galaxy-central/commits/8e448f2ac955/ Changeset: 8e448f2ac955 User: jmchilton Date: 2013-12-17 14:31:52 Summary: Merged in jmchilton/galaxy-central-fork-1 (pull request #275) Move history contents filtering logic into model. Affected #: 3 files diff -r 90a639389de4a4348a32e21d87a13de4b1a72f56 -r 8e448f2ac955e6d434476b18c59b2cffcfc55404 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -19,6 +19,7 @@ import socket import time from string import Template +from itertools import ifilter import galaxy.datatypes import galaxy.datatypes.registry @@ -43,6 +44,11 @@ # Default Value Required for unit tests datatypes_registry.load_datatypes() +# When constructing filters with in for a fixed set of ids, maximum +# number of items to place in the IN statement. Different databases +# are going to have different limits so it is likely best to not let +# this be unlimited - filter in Python if over this limit. +MAX_IN_FILTER_LENGTH = 100 class NoConverterException(Exception): def __init__(self, value): @@ -893,6 +899,32 @@ rval = galaxy.datatypes.data.nice_size( rval ) return rval + def contents_iter( self, **kwds ): + """ + Fetch filtered list of contents of history. + """ + python_filter = None + db_session = object_session( self ) + assert db_session != None + query = db_session.query( HistoryDatasetAssociation ).filter( HistoryDatasetAssociation.table.c.history_id == self.id ) + deleted = galaxy.util.string_as_bool_or_none( kwds.get( 'deleted', None ) ) + if deleted is not None: + query = query.filter( HistoryDatasetAssociation.deleted == bool( kwds['deleted'] ) ) + visible = galaxy.util.string_as_bool_or_none( kwds.get( 'visible', None ) ) + if visible is not None: + query = query.filter( HistoryDatasetAssociation.visible == bool( kwds['visible'] ) ) + if 'ids' in kwds: + ids = kwds['ids'] + max_in_filter_length = kwds.get('max_in_filter_length', MAX_IN_FILTER_LENGTH) + if len(ids) < max_in_filter_length: + query = query.filter( HistoryDatasetAssociation.id.in_(ids) ) + else: + python_filter = lambda hda: hda.id in ids + if python_filter: + return ifilter(python_filter, query) + else: + return query + def copy_tags_from(self,target_user,source_history): for src_shta in source_history.tags: new_shta = src_shta.copy() diff -r 90a639389de4a4348a32e21d87a13de4b1a72f56 -r 8e448f2ac955e6d434476b18c59b2cffcfc55404 lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -51,47 +51,28 @@ else: history = self.get_history( trans, history_id, check_ownership=True, check_accessible=True ) - # if ids, return _FULL_ data (as show) for each id passed + contents_kwds = {} if ids: - ids = ids.split( ',' ) - for index, hda in enumerate( history.datasets ): - encoded_hda_id = trans.security.encode_id( hda.id ) - if encoded_hda_id in ids: - #TODO: share code with show - rval.append( self._detailed_hda_dict( trans, hda ) ) - - # if no ids passed, return a _SUMMARY_ of _all_ datasets in the history + ids = map( lambda id: trans.security.decode_id( id ), ids.split( ',' ) ) + contents_kwds[ 'ids' ] = ids + # If explicit ids given, always used detailed result. + details = 'all' else: + contents_kwds[ 'deleted' ] = kwd.get( 'deleted', None ) + contents_kwds[ 'visible' ] = kwd.get( 'visible', None ) # details param allows a mixed set of summary and detailed hdas #TODO: this is getting convoluted due to backwards compat details = kwd.get( 'details', None ) or [] if details and details != 'all': details = util.listify( details ) - # by default return all datasets - even if deleted or hidden (defaulting the next switches to None) - # if specified return those datasets that match the setting - # backwards compat - return_deleted = util.string_as_bool_or_none( kwd.get( 'deleted', None ) ) - return_visible = util.string_as_bool_or_none( kwd.get( 'visible', None ) ) - - for hda in history.datasets: - # if either return_ setting has been requested (!= None), skip hdas that don't match the request - if return_deleted is not None: - if( ( return_deleted and not hda.deleted ) - or ( not return_deleted and hda.deleted ) ): - continue - if return_visible is not None: - if( ( return_visible and not hda.visible ) - or ( not return_visible and hda.visible ) ): - continue - - encoded_hda_id = trans.security.encode_id( hda.id ) - if( ( encoded_hda_id in details ) - or ( details == 'all' ) ): - rval.append( self._detailed_hda_dict( trans, hda ) ) - else: - rval.append( self._summary_hda_dict( trans, history_id, hda ) ) - + for hda in history.contents_iter( **contents_kwds ): + encoded_hda_id = trans.security.encode_id( hda.id ) + detailed = details == 'all' or ( encoded_hda_id in details ) + if detailed: + rval.append( self._detailed_hda_dict( trans, hda ) ) + else: + rval.append( self._summary_hda_dict( trans, history_id, hda ) ) except Exception, e: # for errors that are not specific to one hda (history lookup or summary list) rval = "Error in history API at listing contents: " + str( e ) diff -r 90a639389de4a4348a32e21d87a13de4b1a72f56 -r 8e448f2ac955e6d434476b18c59b2cffcfc55404 test/unit/test_galaxy_mapping.py --- a/test/unit/test_galaxy_mapping.py +++ b/test/unit/test_galaxy_mapping.py @@ -180,6 +180,39 @@ assert hist1.name == "History 2b" # gvk TODO need to ad test for GalaxySessions, but not yet sure what they should look like. + def test_history_contents( self ): + model = self.model + u = model.User( email="contents(a)foo.bar.baz", password="password" ) + # gs = model.GalaxySession() + h1 = model.History( name="HistoryContentsHistory1", user=u) + + self.persist( u, h1, expunge=True ) + + d1 = self.new_hda( h1, name="1" ) + d2 = self.new_hda( h1, name="2", visible=False ) + d3 = self.new_hda( h1, name="3", deleted=True ) + d4 = self.new_hda( h1, name="4", visible=False, deleted=True ) + + def contents_iter_names(**kwds): + history = model.context.query( model.History ).filter( + model.History.name == "HistoryContentsHistory1" + ).first() + return set( map( lambda hda: hda.name, history.contents_iter( **kwds ) ) ) + + assert contents_iter_names() == set( [ "1", "2", "3", "4" ] ) + assert contents_iter_names( deleted=False ) == set( [ "1", "2" ] ) + assert contents_iter_names( visible=True ) == set( [ "1", "3" ] ) + assert contents_iter_names( visible=False ) == set( [ "2", "4" ] ) + assert contents_iter_names( deleted=True, visible=False ) == set( [ "4" ] ) + + assert contents_iter_names( ids=[ d1.id, d2.id, d3.id, d4.id ] ) == set( [ "1", "2", "3", "4" ] ) + assert contents_iter_names( ids=[ d1.id, d2.id, d3.id, d4.id ], max_in_filter_length=1 ) == set( [ "1", "2", "3", "4" ] ) + + assert contents_iter_names( ids=[ d1.id, d3.id ] ) == set( [ "1", "3" ] ) + + def new_hda( self, history, **kwds ): + return self.persist( self.model.HistoryDatasetAssociation( history=history, create_dataset=True, sa_session=self.model.session, **kwds ), flush=True ) + @classmethod def setUpClass(cls): # Start the database and connect the mapping @@ -191,10 +224,16 @@ return cls.model.session.query( type ) @classmethod - def persist(cls, *args): + def persist(cls, *args, **kwargs): + session = cls.model.session + flush = kwargs.get('flush', True) for arg in args: - cls.model.session.add( arg ) - cls.expunge() + session.add( arg ) + if flush: + session.flush() + if kwargs.get('expunge', not flush): + cls.expunge() + return arg # Return last or only arg. @classmethod def expunge(cls): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 17 Dec '13

17 Dec '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ee365d4406e6/ Changeset: ee365d4406e6 User: jmchilton Date: 2013-12-13 06:25:25 Summary: casperjs: Fix user login, registration, logout for recent masthead changes... They are generated in javascript instead of HTML now so I don't think they are available when page loads. My fix is to wait for the User menu to appear before attempting these operations. Affected #: 4 files diff -r b3eeedb81a1e0db8d643af0cbba68f55eb49cdb0 -r ee365d4406e604b573c3011e5b642dd7075f561d test/casperjs/api-configuration-tests.js --- a/test/casperjs/api-configuration-tests.js +++ b/test/casperjs/api-configuration-tests.js @@ -102,18 +102,19 @@ //}, function failedLoginRegister(){ // this.info( 'Admin level configuration API tests not run: no admin account available' ); spaceghost.thenOpen( spaceghost.baseUrl ).then( function(){ + spaceghost.waitForMasthead( function() { + if( spaceghost.user.userIsAdmin() ){ + this.test.comment( 'index should get a (full) list of configuration settings ' + + 'when requested by an admin user' ); + configIndex = this.api.configuration.index(); + this.debug( this.jsonStr( configIndex ) ); + this.test.assert( utils.isObject( configIndex ), "index returned an object" ); + this.test.assert( hasKeys( configIndex, adminKeys ), 'Has the proper keys' ); - if( spaceghost.user.userIsAdmin() ){ - this.test.comment( 'index should get a (full) list of configuration settings ' - + 'when requested by an admin user' ); - configIndex = this.api.configuration.index(); - this.debug( this.jsonStr( configIndex ) ); - this.test.assert( utils.isObject( configIndex ), "index returned an object" ); - this.test.assert( hasKeys( configIndex, adminKeys ), 'Has the proper keys' ); - - } else { - this.info( 'Admin level configuration API tests not run: no admin account available' ); - } + } else { + this.info( 'Admin level configuration API tests not run: no admin account available' ); + } + }); }); // =================================================================== diff -r b3eeedb81a1e0db8d643af0cbba68f55eb49cdb0 -r ee365d4406e604b573c3011e5b642dd7075f561d test/casperjs/modules/user.js --- a/test/casperjs/modules/user.js +++ b/test/casperjs/modules/user.js @@ -44,17 +44,18 @@ }; spaceghost.thenOpen( spaceghost.baseUrl, function(){ - this.clickLabel( spaceghost.data.labels.masthead.menus.user ); - this.clickLabel( spaceghost.data.labels.masthead.userMenu.register ); + this.waitForMasthead( function() { + this.clickLabel( spaceghost.data.labels.masthead.menus.user ); + this.clickLabel( spaceghost.data.labels.masthead.userMenu.register ); - this.withMainPanel( function mainBeforeRegister(){ - spaceghost.debug( '(' + spaceghost.getCurrentUrl() + ') registering user:\n' - + spaceghost.jsonStr( userInfo ) ); - this.fill( spaceghost.data.selectors.registrationPage.form, userInfo, false ); - // need manual submit (not a normal html form) - this.click( xpath( spaceghost.data.selectors.registrationPage.submit_xpath ) ); + this.withMainPanel( function mainBeforeRegister(){ + spaceghost.debug( '(' + spaceghost.getCurrentUrl() + ') registering user:\n' + + spaceghost.jsonStr( userInfo ) ); + this.fill( spaceghost.data.selectors.registrationPage.form, userInfo, false ); + // need manual submit (not a normal html form) + this.click( xpath( spaceghost.data.selectors.registrationPage.submit_xpath ) ); + }); }); - //// debugging //spaceghost.withFrame( spaceghost.data.selectors.frames.main, function mainAfterRegister(){ // var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); @@ -79,24 +80,26 @@ }; spaceghost.thenOpen( spaceghost.baseUrl, function(){ - spaceghost.clickLabel( spaceghost.data.labels.masthead.menus.user ); - spaceghost.clickLabel( spaceghost.data.labels.masthead.userMenu.login ); + spaceghost.waitForMasthead( function() { + spaceghost.clickLabel( spaceghost.data.labels.masthead.menus.user ); + spaceghost.clickLabel( spaceghost.data.labels.masthead.userMenu.login ); - spaceghost.withMainPanel( function mainBeforeLogin(){ - spaceghost.debug( '(' + spaceghost.getCurrentUrl() + ') logging in user:\n' - + spaceghost.jsonStr( loginInfo ) ); - spaceghost.fill( spaceghost.data.selectors.loginPage.form, loginInfo, false ); - spaceghost.click( xpath( spaceghost.data.selectors.loginPage.submit_xpath ) ); + spaceghost.withMainPanel( function mainBeforeLogin(){ + spaceghost.debug( '(' + spaceghost.getCurrentUrl() + ') logging in user:\n' + + spaceghost.jsonStr( loginInfo ) ); + spaceghost.fill( spaceghost.data.selectors.loginPage.form, loginInfo, false ); + spaceghost.click( xpath( spaceghost.data.selectors.loginPage.submit_xpath ) ); + }); + + //// debugging + //spaceghost.withFrame( spaceghost.data.selectors.frames.main, function mainAfterLogin(){ + // //TODO: prob. could use a more generalized form of this for url breakdown/checking + // if( spaceghost.getCurrentUrl().search( spaceghost.data.selectors.loginPage.url_regex ) != -1 ){ + // var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); + // spaceghost.debug( 'post login message:\n' + spaceghost.jsonStr( messageInfo ) ); + // } + //}); }); - - //// debugging - //spaceghost.withFrame( spaceghost.data.selectors.frames.main, function mainAfterLogin(){ - // //TODO: prob. could use a more generalized form of this for url breakdown/checking - // if( spaceghost.getCurrentUrl().search( spaceghost.data.selectors.loginPage.url_regex ) != -1 ){ - // var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); - // spaceghost.debug( 'post login message:\n' + spaceghost.jsonStr( messageInfo ) ); - // } - //}); }); }; @@ -133,17 +136,19 @@ var spaceghost = this.spaceghost; this._submitLogin( email, password ); - spaceghost.withMainPanel( function mainAfterLogin(){ - if( spaceghost.getCurrentUrl().search( spaceghost.data.selectors.loginPage.url_regex ) !== -1 ){ - var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); - if( messageInfo && messageInfo.attributes[ 'class' ] === 'errormessage' ){ - this.warning( 'Login failed: ' + messageInfo.text ); - throw new spaceghost.GalaxyError( 'LoginError: ' + messageInfo.text ); + spaceghost.waitForMasthead( function() { + spaceghost.withMainPanel( function mainAfterLogin(){ + if( spaceghost.getCurrentUrl().search( spaceghost.data.selectors.loginPage.url_regex ) !== -1 ){ + var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); + if( messageInfo && messageInfo.attributes[ 'class' ] === 'errormessage' ){ + this.warning( 'Login failed: ' + messageInfo.text ); + throw new spaceghost.GalaxyError( 'LoginError: ' + messageInfo.text ); + } } - } - if( spaceghost.user.loggedInAs() === email ){ - spaceghost.info( 'logged in as ' + email ); - } + if( spaceghost.user.loggedInAs() === email ){ + spaceghost.info( 'logged in as ' + email ); + } + }); }); return spaceghost; }; @@ -173,8 +178,10 @@ spaceghost.thenOpen( spaceghost.baseUrl, function(){ this.info( 'user logging out' ); //TODO: handle already logged out - spaceghost.clickLabel( spaceghost.data.labels.masthead.menus.user ); - spaceghost.clickLabel( spaceghost.data.labels.masthead.userMenu.logout ); + spaceghost.waitForMasthead( function _logout() { + spaceghost.clickLabel( spaceghost.data.labels.masthead.menus.user ); + spaceghost.clickLabel( spaceghost.data.labels.masthead.userMenu.logout ); + }); }); return spaceghost; }; diff -r b3eeedb81a1e0db8d643af0cbba68f55eb49cdb0 -r ee365d4406e604b573c3011e5b642dd7075f561d test/casperjs/registration-tests.js --- a/test/casperjs/registration-tests.js +++ b/test/casperjs/registration-tests.js @@ -62,18 +62,24 @@ this.user._submitRegistration( email, password, username, confirm ); }); spaceghost.thenOpen( spaceghost.baseUrl, function(){ - this.clickLabel( 'User' ); - this.test.assertSelectorHasText( 'a #user-email', email, '#user-email === ' + email ); + this.waitForMasthead( function() { + this.clickLabel( 'User' ); + this.test.assertSelectorHasText( 'a #user-email', email, '#user-email === ' + email ); + }); }); // ------------------------------------------------------------------- log out that user spaceghost.then( function(){ - this.test.comment( 'logging out user: ' + email ); - this.user.logout(); + this.waitForMasthead( function() { + this.test.comment( 'logging out user: ' + email ); + this.user.logout(); + }); }); spaceghost.then( function(){ - this.debug( 'email:' + this.getElementInfo( 'a #user-email' ).html ); - this.test.assert( !this.getElementInfo( 'a #user-email' ).html, '#user-email is empty' ); + this.waitForMasthead( function() { + this.debug( 'email:' + this.getElementInfo( 'a #user-email' ).html ); + this.test.assert( !this.getElementInfo( 'a #user-email' ).html, '#user-email is empty' ); + }); }); // ------------------------------------------------------------------- bad user registrations diff -r b3eeedb81a1e0db8d643af0cbba68f55eb49cdb0 -r ee365d4406e604b573c3011e5b642dd7075f561d test/casperjs/spaceghost.js --- a/test/casperjs/spaceghost.js +++ b/test/casperjs/spaceghost.js @@ -1103,6 +1103,10 @@ return match[2]; }; +SpaceGhost.prototype.waitForMasthead = function wait( then ) { + return this.waitForText( this.data.labels.masthead.menus.user, then ); +} + // =================================================================== TEST DATA /** General use selectors, labels, and text. Kept here to allow a centralized location. https://bitbucket.org/galaxy/galaxy-central/commits/90a639389de4/ Changeset: 90a639389de4 User: jmchilton Date: 2013-12-17 14:30:24 Summary: Merged in jmchilton/galaxy-central-fork-1 (pull request #274) casperjs: Fix user login, registration, logout for recent masthead changes... Affected #: 4 files diff -r af87cdcf36084831fce7d99e621b1efc943d2aff -r 90a639389de4a4348a32e21d87a13de4b1a72f56 test/casperjs/api-configuration-tests.js --- a/test/casperjs/api-configuration-tests.js +++ b/test/casperjs/api-configuration-tests.js @@ -102,18 +102,19 @@ //}, function failedLoginRegister(){ // this.info( 'Admin level configuration API tests not run: no admin account available' ); spaceghost.thenOpen( spaceghost.baseUrl ).then( function(){ + spaceghost.waitForMasthead( function() { + if( spaceghost.user.userIsAdmin() ){ + this.test.comment( 'index should get a (full) list of configuration settings ' + + 'when requested by an admin user' ); + configIndex = this.api.configuration.index(); + this.debug( this.jsonStr( configIndex ) ); + this.test.assert( utils.isObject( configIndex ), "index returned an object" ); + this.test.assert( hasKeys( configIndex, adminKeys ), 'Has the proper keys' ); - if( spaceghost.user.userIsAdmin() ){ - this.test.comment( 'index should get a (full) list of configuration settings ' - + 'when requested by an admin user' ); - configIndex = this.api.configuration.index(); - this.debug( this.jsonStr( configIndex ) ); - this.test.assert( utils.isObject( configIndex ), "index returned an object" ); - this.test.assert( hasKeys( configIndex, adminKeys ), 'Has the proper keys' ); - - } else { - this.info( 'Admin level configuration API tests not run: no admin account available' ); - } + } else { + this.info( 'Admin level configuration API tests not run: no admin account available' ); + } + }); }); // =================================================================== diff -r af87cdcf36084831fce7d99e621b1efc943d2aff -r 90a639389de4a4348a32e21d87a13de4b1a72f56 test/casperjs/modules/user.js --- a/test/casperjs/modules/user.js +++ b/test/casperjs/modules/user.js @@ -44,17 +44,18 @@ }; spaceghost.thenOpen( spaceghost.baseUrl, function(){ - this.clickLabel( spaceghost.data.labels.masthead.menus.user ); - this.clickLabel( spaceghost.data.labels.masthead.userMenu.register ); + this.waitForMasthead( function() { + this.clickLabel( spaceghost.data.labels.masthead.menus.user ); + this.clickLabel( spaceghost.data.labels.masthead.userMenu.register ); - this.withMainPanel( function mainBeforeRegister(){ - spaceghost.debug( '(' + spaceghost.getCurrentUrl() + ') registering user:\n' - + spaceghost.jsonStr( userInfo ) ); - this.fill( spaceghost.data.selectors.registrationPage.form, userInfo, false ); - // need manual submit (not a normal html form) - this.click( xpath( spaceghost.data.selectors.registrationPage.submit_xpath ) ); + this.withMainPanel( function mainBeforeRegister(){ + spaceghost.debug( '(' + spaceghost.getCurrentUrl() + ') registering user:\n' + + spaceghost.jsonStr( userInfo ) ); + this.fill( spaceghost.data.selectors.registrationPage.form, userInfo, false ); + // need manual submit (not a normal html form) + this.click( xpath( spaceghost.data.selectors.registrationPage.submit_xpath ) ); + }); }); - //// debugging //spaceghost.withFrame( spaceghost.data.selectors.frames.main, function mainAfterRegister(){ // var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); @@ -79,24 +80,26 @@ }; spaceghost.thenOpen( spaceghost.baseUrl, function(){ - spaceghost.clickLabel( spaceghost.data.labels.masthead.menus.user ); - spaceghost.clickLabel( spaceghost.data.labels.masthead.userMenu.login ); + spaceghost.waitForMasthead( function() { + spaceghost.clickLabel( spaceghost.data.labels.masthead.menus.user ); + spaceghost.clickLabel( spaceghost.data.labels.masthead.userMenu.login ); - spaceghost.withMainPanel( function mainBeforeLogin(){ - spaceghost.debug( '(' + spaceghost.getCurrentUrl() + ') logging in user:\n' - + spaceghost.jsonStr( loginInfo ) ); - spaceghost.fill( spaceghost.data.selectors.loginPage.form, loginInfo, false ); - spaceghost.click( xpath( spaceghost.data.selectors.loginPage.submit_xpath ) ); + spaceghost.withMainPanel( function mainBeforeLogin(){ + spaceghost.debug( '(' + spaceghost.getCurrentUrl() + ') logging in user:\n' + + spaceghost.jsonStr( loginInfo ) ); + spaceghost.fill( spaceghost.data.selectors.loginPage.form, loginInfo, false ); + spaceghost.click( xpath( spaceghost.data.selectors.loginPage.submit_xpath ) ); + }); + + //// debugging + //spaceghost.withFrame( spaceghost.data.selectors.frames.main, function mainAfterLogin(){ + // //TODO: prob. could use a more generalized form of this for url breakdown/checking + // if( spaceghost.getCurrentUrl().search( spaceghost.data.selectors.loginPage.url_regex ) != -1 ){ + // var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); + // spaceghost.debug( 'post login message:\n' + spaceghost.jsonStr( messageInfo ) ); + // } + //}); }); - - //// debugging - //spaceghost.withFrame( spaceghost.data.selectors.frames.main, function mainAfterLogin(){ - // //TODO: prob. could use a more generalized form of this for url breakdown/checking - // if( spaceghost.getCurrentUrl().search( spaceghost.data.selectors.loginPage.url_regex ) != -1 ){ - // var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); - // spaceghost.debug( 'post login message:\n' + spaceghost.jsonStr( messageInfo ) ); - // } - //}); }); }; @@ -133,17 +136,19 @@ var spaceghost = this.spaceghost; this._submitLogin( email, password ); - spaceghost.withMainPanel( function mainAfterLogin(){ - if( spaceghost.getCurrentUrl().search( spaceghost.data.selectors.loginPage.url_regex ) !== -1 ){ - var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); - if( messageInfo && messageInfo.attributes[ 'class' ] === 'errormessage' ){ - this.warning( 'Login failed: ' + messageInfo.text ); - throw new spaceghost.GalaxyError( 'LoginError: ' + messageInfo.text ); + spaceghost.waitForMasthead( function() { + spaceghost.withMainPanel( function mainAfterLogin(){ + if( spaceghost.getCurrentUrl().search( spaceghost.data.selectors.loginPage.url_regex ) !== -1 ){ + var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); + if( messageInfo && messageInfo.attributes[ 'class' ] === 'errormessage' ){ + this.warning( 'Login failed: ' + messageInfo.text ); + throw new spaceghost.GalaxyError( 'LoginError: ' + messageInfo.text ); + } } - } - if( spaceghost.user.loggedInAs() === email ){ - spaceghost.info( 'logged in as ' + email ); - } + if( spaceghost.user.loggedInAs() === email ){ + spaceghost.info( 'logged in as ' + email ); + } + }); }); return spaceghost; }; @@ -173,8 +178,10 @@ spaceghost.thenOpen( spaceghost.baseUrl, function(){ this.info( 'user logging out' ); //TODO: handle already logged out - spaceghost.clickLabel( spaceghost.data.labels.masthead.menus.user ); - spaceghost.clickLabel( spaceghost.data.labels.masthead.userMenu.logout ); + spaceghost.waitForMasthead( function _logout() { + spaceghost.clickLabel( spaceghost.data.labels.masthead.menus.user ); + spaceghost.clickLabel( spaceghost.data.labels.masthead.userMenu.logout ); + }); }); return spaceghost; }; diff -r af87cdcf36084831fce7d99e621b1efc943d2aff -r 90a639389de4a4348a32e21d87a13de4b1a72f56 test/casperjs/registration-tests.js --- a/test/casperjs/registration-tests.js +++ b/test/casperjs/registration-tests.js @@ -62,18 +62,24 @@ this.user._submitRegistration( email, password, username, confirm ); }); spaceghost.thenOpen( spaceghost.baseUrl, function(){ - this.clickLabel( 'User' ); - this.test.assertSelectorHasText( 'a #user-email', email, '#user-email === ' + email ); + this.waitForMasthead( function() { + this.clickLabel( 'User' ); + this.test.assertSelectorHasText( 'a #user-email', email, '#user-email === ' + email ); + }); }); // ------------------------------------------------------------------- log out that user spaceghost.then( function(){ - this.test.comment( 'logging out user: ' + email ); - this.user.logout(); + this.waitForMasthead( function() { + this.test.comment( 'logging out user: ' + email ); + this.user.logout(); + }); }); spaceghost.then( function(){ - this.debug( 'email:' + this.getElementInfo( 'a #user-email' ).html ); - this.test.assert( !this.getElementInfo( 'a #user-email' ).html, '#user-email is empty' ); + this.waitForMasthead( function() { + this.debug( 'email:' + this.getElementInfo( 'a #user-email' ).html ); + this.test.assert( !this.getElementInfo( 'a #user-email' ).html, '#user-email is empty' ); + }); }); // ------------------------------------------------------------------- bad user registrations diff -r af87cdcf36084831fce7d99e621b1efc943d2aff -r 90a639389de4a4348a32e21d87a13de4b1a72f56 test/casperjs/spaceghost.js --- a/test/casperjs/spaceghost.js +++ b/test/casperjs/spaceghost.js @@ -1103,6 +1103,10 @@ return match[2]; }; +SpaceGhost.prototype.waitForMasthead = function wait( then ) { + return this.waitForText( this.data.labels.masthead.menus.user, then ); +} + // =================================================================== TEST DATA /** General use selectors, labels, and text. Kept here to allow a centralized location. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Eliminate deprecated repositories from the tool shed's install and test framework.
by commits-noreply＠bitbucket.org 17 Dec '13

17 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/af87cdcf3608/ Changeset: af87cdcf3608 User: greg Date: 2013-12-17 04:06:16 Summary: Eliminate deprecated repositories from the tool shed's install and test framework. Affected #: 1 file diff -r 422f9675cb5025e20b402ad92aa591a87ca3a883 -r af87cdcf36084831fce7d99e621b1efc943d2aff test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -176,14 +176,14 @@ else: galaxy_encode_secret = os.environ[ 'GALAXY_INSTALL_TEST_SECRET' ] -testing_single_repository = {} +testing_single_repository_dict = {} if 'repository_name' in os.environ and 'repository_owner' in os.environ: - testing_single_repository[ 'name' ] = os.environ[ 'repository_name' ] - testing_single_repository[ 'owner' ] = os.environ[ 'repository_owner' ] + testing_single_repository_dict[ 'name' ] = str( os.environ[ 'repository_name' ] ) + testing_single_repository_dict[ 'owner' ] = str( os.environ[ 'repository_owner' ] ) if 'repository_revision' in os.environ: - testing_single_repository[ 'changeset_revision' ] = os.environ[ 'repository_revision' ] + testing_single_repository_dict[ 'changeset_revision' ] = str( os.environ[ 'repository_revision' ] ) else: - testing_single_repository[ 'changeset_revision' ] = None + testing_single_repository_dict[ 'changeset_revision' ] = None class ReportResults( Plugin ): @@ -364,6 +364,8 @@ log.debug( 'Testing is restricted to the latest downloadable revision in this test run.' ) repository_dicts = [] parts = [ 'repository_revisions' ] + # We'll filter out deprecated repositories from testing since testing them is necessary only if reproducibility + # is guaranteed and we currently do not guarantee reproducibility. params = dict( do_not_test='false', downloadable='true', includes_tools='true', @@ -381,21 +383,25 @@ if error_message: log.debug( 'Error getting additional details from the API: %s' % str( error_message ) ) else: - # Don't test empty repositories. - changeset_revision = baseline_repository_dict[ 'changeset_revision' ] - if changeset_revision != suc.INITIAL_CHANGELOG_HASH: - # Merge the dictionary returned from /api/repository_revisions with the detailed repository_dict and - # append it to the list of repository_dicts to install and test. - if latest_revision_only: - latest_revision = repository_dict[ 'latest_revision' ] - if changeset_revision == latest_revision: + # Don't test deprecated repositories since testing them is necessary only if reproducibility is guaranteed + # and we are not currently guaranteeing reproducibility. + deprecated = asbool( repository_dict.get( 'deprecated', False ) ) + if not deprecated: + # Don't test empty repositories. + changeset_revision = baseline_repository_dict[ 'changeset_revision' ] + if changeset_revision != suc.INITIAL_CHANGELOG_HASH: + # Merge the dictionary returned from /api/repository_revisions with the detailed repository_dict and + # append it to the list of repository_dicts to install and test. + if latest_revision_only: + latest_revision = repository_dict[ 'latest_revision' ] + if changeset_revision == latest_revision: + repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) ) + else: repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) ) - else: - repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) ) - if testing_single_repository: - tsr_name = testing_single_repository[ 'name' ] - tsr_owner = testing_single_repository[ 'owner' ] - tsr_changeset_revision = testing_single_repository[ 'changeset_revision' ] + if testing_single_repository_dict: + tsr_name = testing_single_repository_dict[ 'name' ] + tsr_owner = testing_single_repository_dict[ 'owner' ] + tsr_changeset_revision = testing_single_repository_dict[ 'changeset_revision' ] log.debug( 'Testing single repository with name %s and owner %s.' % ( str( tsr_name ), str( tsr_owner ) ) ) for repository_to_install in repository_dicts: rti_name = repository_to_install[ 'name' ] Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Handle exceptions when uninstalling repositories in the tool shed's install nad test framework.
by commits-noreply＠bitbucket.org 17 Dec '13

17 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/422f9675cb50/ Changeset: 422f9675cb50 User: greg Date: 2013-12-17 01:27:28 Summary: Handle exceptions when uninstalling repositories in the tool shed's install nad test framework. Affected #: 1 file diff -r 46521f5722c621a8b84d2a8101af87d82cc5eba8 -r 422f9675cb5025e20b402ad92aa591a87ca3a883 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -1220,9 +1220,9 @@ for owner, grouped_repository_dicts in repository_dicts_by_owner.items(): print "# " for repository_dict in grouped_repository_dicts: - name = repository_dict[ 'name' ] - owner = repository_dict[ 'owner' ] - changeset_revision = repository_dict[ 'changeset_revision' ] + name = str( repository_dict[ 'name' ] ) + owner = str( repository_dict[ 'owner' ] ) + changeset_revision = str( repository_dict[ 'changeset_revision' ] ) print "# Revision %s of repository %s owned by %s" % ( changeset_revision, name, owner ) def test_repository_tools( app, repository, repository_dict, tool_test_results_dicts, tool_test_results_dict, results_dict ): @@ -1328,15 +1328,20 @@ owner=owner, changeset_revision=changeset_revision ) log.debug( 'Revision %s of repository %s owned by %s selected for uninstallation.' % ( changeset_revision, name, owner ) ) - test_install_repositories.generate_uninstall_method( uninstall_repository_dict ) - # Set up nose to run the generated uninstall method as a functional test. - test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) - test_config.configure( sys.argv ) - # Run the uninstall method. This method uses the Galaxy web interface to uninstall the previously installed - # repository and all of its repository dependencies, deleting each of them from disk. - result, _ = run_tests( test_config ) - success = result.wasSuccessful() - if success: + try: + test_install_repositories.generate_uninstall_method( uninstall_repository_dict ) + # Set up nose to run the generated uninstall method as a functional test. + test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) + test_config.configure( sys.argv ) + # Run the uninstall method. This method uses the Galaxy web interface to uninstall the previously installed + # repository and all of its repository dependencies, deleting each of them from disk. + result, _ = run_tests( test_config ) + repository_uninstall_successful = result.wasSuccessful() + except Exception, e: + repository_uninstall_successful = False + log.exception( 'Uninstallation of revision %s of repository %s owned by %s failed: %s.' % \ + ( rd_changeset_revision, rd_name, rd_owner, str( e ) ) ) + if repository_uninstall_successful: # Now that the repository is uninstalled we can attempt to uninstall each of its repository dependencies. # We have to do this through Twill in order to maintain app.toolbox and shed_tool_conf.xml in a state that # is valid for future tests. Since some of the repository's repository dependencies may require other of @@ -1356,15 +1361,19 @@ ( rd_changeset_revision, rd_name, rd_owner ) ) # Generate a test method to uninstall the repository dependency through the embedded Galaxy application's # web interface. - test_install_repositories.generate_uninstall_method( uninstall_repository_dict ) - # Set up nose to run the generated uninstall method as a functional test. - test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) - test_config.configure( sys.argv ) - # Run the uninstall method. - result, _ = run_tests( test_config ) - success = result.wasSuccessful() - if not success: - log.debug( 'Uninstallation of revision %s of repository %s owned by %s failed: %s' % \ + try: + test_install_repositories.generate_uninstall_method( uninstall_repository_dict ) + # Set up nose to run the generated uninstall method as a functional test. + test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) + test_config.configure( sys.argv ) + # Run the uninstall method. + result, _ = run_tests( test_config ) + if not result.wasSuccessful(): + # We won't set ok here because we'll continue to uninstall whatever we can. + log.debug( 'Uninstallation of revision %s of repository %s owned by %s failed.' % \ + ( rd_changeset_revision, rd_name, rd_owner ) ) + except Exception, e: + log.exception( 'Uninstallation of revision %s of repository %s owned by %s failed: %s.' % \ ( rd_changeset_revision, rd_name, rd_owner, str( e ) ) ) else: log.debug( 'Uninstallation of revision %s of repository %s owned by %s failed.' % ( changeset_revision, name, owner ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix in the tool shed's install and test framework.
by commits-noreply＠bitbucket.org 16 Dec '13

16 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/46521f5722c6/ Changeset: 46521f5722c6 User: greg Date: 2013-12-16 23:25:23 Summary: Fix in the tool shed's install and test framework. Affected #: 1 file diff -r 32d4520018961092d767e350c9a2ad49031347d1 -r 46521f5722c621a8b84d2a8101af87d82cc5eba8 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -525,7 +525,7 @@ deactivate_repository( app, repository_dict ) else: # We are uninstalling this repository and all of its repository dependencies. - uninstall_repository( app, repository_dict ) + uninstall_repository_and_repository_dependencies( app, repository_dict ) results_dict[ 'repositories_failed_install' ].append( dict( name=str( repository.name ), owner=str( repository.owner ), changeset_revision=str( repository.changeset_revision ) ) ) @@ -730,7 +730,7 @@ deactivate_repository( app, repository_dict ) else: # We are uninstalling this repository and all of its repository dependencies. - uninstall_repository( app, repository_dict ) + uninstall_repository_and_repository_dependencies( app, repository_dict ) except: log.exception( 'Encountered error attempting to deactivate or uninstall %s.', str( repository_dict[ 'name' ] ) ) results_dict[ 'repositories_failed_install' ].append( repository_identifier_dict ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: RFC: API design guidelines and considerations.
by commits-noreply＠bitbucket.org 16 Dec '13

16 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/32d452001896/ Changeset: 32d452001896 User: jmchilton Date: 2013-12-16 22:57:31 Summary: RFC: API design guidelines and considerations. Affected #: 1 file diff -r 2025eca47bca55eb69d3ea4fe95c54b3163e61ac -r 32d4520018961092d767e350c9a2ad49031347d1 doc/source/lib/galaxy.webapps.galaxy.api.rst --- a/doc/source/lib/galaxy.webapps.galaxy.api.rst +++ b/doc/source/lib/galaxy.webapps.galaxy.api.rst @@ -208,7 +208,8 @@ API Return Codes and Formats ================== -TODO (here, or add a new page and link it) +A set of error codes for API requests is being established and will be +documented here. This is a long-term project however so stayed tuned. API Controllers =============== @@ -397,3 +398,67 @@ :undoc-members: :show-inheritance: + +API Design Guidelines +===================== + +The following section outlines guidelines related to extending and/or modifing +the Galaxy API. The Galaxy API has grown in an ad-hoc fashion over time by +many contributors and so clients SHOULD NOT expect the API will conform to +these guidelines - but developers contributing to the Galaxy API SHOULD follow +these guidelines. + + - API functionality should include docstring documentation for consumption + by readthedocs.org. + - Developers should familarize themselves with the HTTP status code definitions + http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html. The API responses + should properly set the status code according to the result - in particular + 2XX responses should be used for successful requests, 4XX for various + kinds of client errors, and 5XX for errors on the server side. + - If there is an error processing some part of request (one item in a list + for instance), the status code should be set to reflect the error and the + partial result may or may not be returned depending on the controller - + this behavior should be documented. + - (TODO) API methods should throw a finite number of exceptions (defined in) + `galaxy.exceptions` and these should subclass `MessageException` and not + paste/wsgi HTTP exceptions. When possible, the framework itself should be + responsible catching these exceptions, setting the status code, and + building an error response. + - Error responses should not consist of plain text strings - they should be + dictionaries describing the error and containing the following keys (TODO: + spell out nature of this.) Various error conditions (once a format has + been chosen and framework to enforce it in place) should be spelled out + in this document. + - Backward compatibility is important and should maintained when possible. + If changing behavior in a non-backward compatibile way please ensure one + of the following holds - there is a strong reason to believe no consumers + depend on a behavior, the behavior is effectively broken, or the API + method being modified has not been part of a tagged dist release. + +The following bullet points represent good practices more than guidelines, please +consider them when modifying the API. + + - Functionality should not be copied and pasted between controllers - + consider refactoring functionality into associated classes or short of + that into Mixins (http://en.wikipedia.org/wiki/Composition_over_inheritance) + - API additions are more permanent changes to Galaxy than many other potential + changes and so a second opinion on API changes should be sought. (Consider a + pull request!) + - New API functionality should include functional tests. These functional + tests should be implemented in Python and placed in + `test/functional/api`. (Once such a framework is in place - it is not + right now). + - Changes to reflect modifications to the API should be pushed upstream to + the BioBlend project possible. + +Longer term goals/notes. + + - It would be advantageous to have a clearer separation of anonymous and + admin handling functionality. + - If at some point in the future, functionality needs to be added that + breaks backward compatibility in a significant way to a compontent used by + the community should be alerted - a "dev" variant of the API will be + established and the community should be alerted and given a timeframe + for when the old behavior will be replaced with the new behavior. + - Consistent standards for range-based requests, batch requests, filtered + requests, etc... should be established and documented here. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix for uninstalling repositories in an ordered fashion.
by commits-noreply＠bitbucket.org 16 Dec '13

16 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/2025eca47bca/ Changeset: 2025eca47bca User: greg Date: 2013-12-16 22:01:25 Summary: Fix for uninstalling repositories in an ordered fashion. Affected #: 1 file diff -r 10f283c957e5f8dbca6c93aae7655926600a3d12 -r 2025eca47bca55eb69d3ea4fe95c54b3163e61ac test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -1363,10 +1363,9 @@ # Run the uninstall method. result, _ = run_tests( test_config ) success = result.wasSuccessful() - if not success: - log.debug( 'Uninstallation of revision %s of repository %s owned by %s failed: %s' % \ - ( rd_changeset_revision, rd_name, rd_owner, str( e ) ) ) - break + if not success: + log.debug( 'Uninstallation of revision %s of repository %s owned by %s failed: %s' % \ + ( rd_changeset_revision, rd_name, rd_owner, str( e ) ) ) else: log.debug( 'Uninstallation of revision %s of repository %s owned by %s failed.' % ( changeset_revision, name, owner ) ) else: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Enhance the tool shed's install and test framework to uninstall repositories in an ordered fashion so that repositories are not uninstalled when they are dependencies of other installed repositories.
by commits-noreply＠bitbucket.org 16 Dec '13

16 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/10f283c957e5/ Changeset: 10f283c957e5 User: greg Date: 2013-12-16 21:52:29 Summary: Enhance the tool shed's install and test framework to uninstall repositories in an ordered fashion so that repositories are not uninstalled when they are dependencies of other installed repositories. Affected #: 6 files diff -r f2ce61f7d1ad2ea38faedf952aa0abf60ca06e2c -r 10f283c957e5f8dbca6c93aae7655926600a3d12 lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py --- a/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py +++ b/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py @@ -30,6 +30,7 @@ from galaxy.model.orm import and_ from galaxy.model.orm import not_ from galaxy.model.orm import select +from galaxy.util import listify from galaxy.web import url_for from tool_shed.repository_types.util import TOOL_DEPENDENCY_DEFINITION diff -r f2ce61f7d1ad2ea38faedf952aa0abf60ca06e2c -r 10f283c957e5f8dbca6c93aae7655926600a3d12 test/base/tool_shed_util.py --- a/test/base/tool_shed_util.py +++ b/test/base/tool_shed_util.py @@ -38,6 +38,47 @@ return None, repository_name, changeset_revision return last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision +def log_reason_repository_cannot_be_uninstalled( app, repository ): + # This method should be altered if / when the app.install_model.ToolShedRepository.can_uninstall() + # method is altered. Any block returning a False value from that method should be handled here. + name = str( repository.name ) + owner = str( repository.owner ) + installed_changeset_revision = str( repository.installed_changeset_revision ) + log.debug( "Revision %s of repository %s owned by %s cannot be uninstalled because:" % \ + ( installed_changeset_revision, name, owner ) ) + if repository.status == app.install_model.ToolShedRepository.installation_status.UNINSTALLED: + log.debug( 'it is already uninstalled.' ) + else: + irm = app.installed_repository_manager + repository_tup = ( str( repository.tool_shed ), name, owner, installed_changeset_revision ) + # Find other installed repositories that require this repository. + installed_dependent_repository_tups = \ + irm.installed_dependent_repositories_of_installed_repositories.get( repository_tup, [] ) + if installed_dependent_repository_tups: + for installed_dependent_repository_tup in installed_dependent_repository_tups: + idr_tool_shed, idr_name, idr_owner, idr_installed_changeset_revision = installed_dependent_repository_tup + log.debug( "it is required by revision %s of repository %s owned by %s" % \ + ( idr_installed_changeset_revision, idr_name, idr_owner ) ) + else: + # Find installed tool dependencies that require this repository's installed tool dependencies. + installed_dependent_td_tups = None + installed_tool_dependency_tups = irm.installed_tool_dependencies_of_installed_repositories.get( repository_tup, [] ) + for td_tup in installed_tool_dependency_tups: + installed_dependent_td_tups = \ + irm.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies.get( td_tup, [] ) + if installed_dependent_td_tups is not None: + # This repository cannot be uninstalled because it contains installed tool dependencies that + # are required at run time by other installed tool dependencies. + log.debug( "it contains installed tool dependencies that are required at run time by these installed tool dependencies:" ) + for installed_dependent_td_tup in installed_dependent_td_tups: + repository_id, td_name, td_version, td_type = installed_dependent_td_tup + dependent_repository = test_db_util.get_repository( repository_id ) + dr_name = str( dependent_repository.name ) + dr_owner = str( dependent_repository.owner ) + dr_installed_changeset_revison = str( dependent_repository.installed_changeset_revision ) + log.debug( "- version %s of %s %s contained in revision %s of repository %s owned by %s" % \ + ( td_version, td_type, td_name, dr_installed_changeset_revison, dr_name, dr_owner ) ) + def parse_tool_panel_config( config, shed_tools_dict ): """ Parse a shed-related tool panel config to generate the shed_tools_dict. This only happens when testing tools installed from the tool shed. diff -r f2ce61f7d1ad2ea38faedf952aa0abf60ca06e2c -r 10f283c957e5f8dbca6c93aae7655926600a3d12 test/install_and_test_tool_shed_repositories/base/test_db_util.py --- a/test/install_and_test_tool_shed_repositories/base/test_db_util.py +++ b/test/install_and_test_tool_shed_repositories/base/test_db_util.py @@ -26,16 +26,15 @@ def get_repository( repository_id ): return database_contexts.install_context.query( install_model.ToolShedRepository ) \ - .filter( install_model.ToolShedRepository.table.c.id == repository_id ) \ - .first() + .filter( install_model.ToolShedRepository.table.c.id == repository_id ) \ + .first() def get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ): return database_contexts.install_context.query( install_model.ToolShedRepository ) \ - .filter( and_( install_model.ToolShedRepository.table.c.name == name, - install_model.ToolShedRepository.table.c.owner == owner, - install_model.ToolShedRepository.table.c.installed_changeset_revision == changeset_revision ) ) \ - .one() - + .filter( and_( install_model.ToolShedRepository.table.c.name == name, + install_model.ToolShedRepository.table.c.owner == owner, + install_model.ToolShedRepository.table.c.installed_changeset_revision == changeset_revision ) ) \ + .one() def get_private_role( user ): for role in user.all_roles(): @@ -46,18 +45,18 @@ def get_tool_dependencies_for_installed_repository( repository_id, status=None, exclude_status=None ): if status is not None: return database_contexts.install_context.query( install_model.ToolDependency ) \ - .filter( and_( install_model.ToolDependency.table.c.tool_shed_repository_id == repository_id, - install_model.ToolDependency.table.c.status == status ) ) \ - .all() + .filter( and_( install_model.ToolDependency.table.c.tool_shed_repository_id == repository_id, + install_model.ToolDependency.table.c.status == status ) ) \ + .all() elif exclude_status is not None: return database_contexts.install_context.query( install_model.ToolDependency ) \ - .filter( and_( install_model.ToolDependency.table.c.tool_shed_repository_id == repository_id, - install_model.ToolDependency.table.c.status != exclude_status ) ) \ - .all() + .filter( and_( install_model.ToolDependency.table.c.tool_shed_repository_id == repository_id, + install_model.ToolDependency.table.c.status != exclude_status ) ) \ + .all() else: return database_contexts.install_context.query( install_model.ToolDependency ) \ - .filter( install_model.ToolDependency.table.c.tool_shed_repository_id == repository_id ) \ - .all() + .filter( install_model.ToolDependency.table.c.tool_shed_repository_id == repository_id ) \ + .all() def mark_obj_deleted( obj ): obj.deleted = True @@ -75,5 +74,5 @@ def get_user( email ): return database_contexts.galaxy_context.query( model.User ) \ - .filter( model.User.table.c.email==email ) \ - .first() + .filter( model.User.table.c.email==email ) \ + .first() diff -r f2ce61f7d1ad2ea38faedf952aa0abf60ca06e2c -r 10f283c957e5f8dbca6c93aae7655926600a3d12 test/install_and_test_tool_shed_repositories/base/twilltestcase.py --- a/test/install_and_test_tool_shed_repositories/base/twilltestcase.py +++ b/test/install_and_test_tool_shed_repositories/base/twilltestcase.py @@ -28,19 +28,14 @@ self.shed_tools_dict = {} self.home() - def deactivate_or_uninstall_repository( self, installed_repository, deactivate=False ): - url = '/admin_toolshed/deactivate_or_uninstall_repository?id=%s' % self.security.encode_id( installed_repository.id ) + def deactivate_repository( self, repository ): + """Deactivate a repository.""" + url = '/admin_toolshed/deactivate_or_uninstall_repository?id=%s' % self.security.encode_id( repository.id ) self.visit_url( url ) - if deactivate: - tc.fv ( 1, "remove_from_disk", 'false' ) - else: - tc.fv ( 1, "remove_from_disk", 'true' ) + tc.fv ( 1, "remove_from_disk", 'false' ) tc.submit( 'deactivate_or_uninstall_repository_button' ) strings_displayed = [ 'The repository named' ] - if deactivate: - strings_displayed.append( 'has been deactivated' ) - else: - strings_displayed.append( 'has been uninstalled' ) + strings_displayed.append( 'has been deactivated' ) self.check_for_strings( strings_displayed, strings_not_displayed=[] ) def initiate_installation_process( self, @@ -129,7 +124,18 @@ log.debug( 'No field %s in form %s, discarding from return value.' % ( str( control ), str( form_id ) ) ) del( kwd[ field_name ] ) return kwd - + + def uninstall_repository( self, repository ): + """Uninstall a repository.""" + # A repository can be uninstalled only if no dependent repositories are installed. + url = '/admin_toolshed/deactivate_or_uninstall_repository?id=%s' % self.security.encode_id( repository.id ) + self.visit_url( url ) + tc.fv ( 1, "remove_from_disk", 'true' ) + tc.submit( 'deactivate_or_uninstall_repository_button' ) + strings_displayed = [ 'The repository named' ] + strings_displayed.append( 'has been uninstalled' ) + self.check_for_strings( strings_displayed, strings_not_displayed=[] ) + def visit_url( self, url, allowed_codes=[ 200 ] ): new_url = tc.go( url ) return_code = tc.browser.get_code() @@ -145,11 +151,13 @@ if repository_ids: for repository_id in repository_ids: galaxy_repository = test_db_util.get_repository( self.security.decode_id( repository_id ) ) - log.debug( 'Repository %s with ID %s has initial state %s.' % ( str( galaxy_repository.name ), str( repository_id ), str( galaxy_repository.status ) ) ) + log.debug( 'Repository %s with ID %s has initial state %s.' % \ + ( str( galaxy_repository.name ), str( repository_id ), str( galaxy_repository.status ) ) ) timeout_counter = 0 while galaxy_repository.status not in final_states: test_db_util.refresh( galaxy_repository ) - log.debug( 'Repository %s with ID %s is in state %s, continuing to wait.' % ( str( galaxy_repository.name ), str( repository_id ), str( galaxy_repository.status ) ) ) + log.debug( 'Repository %s with ID %s is in state %s, continuing to wait.' % \ + ( str( galaxy_repository.name ), str( repository_id ), str( galaxy_repository.status ) ) ) timeout_counter = timeout_counter + 1 if timeout_counter % 10 == 0: log.debug( 'Waited %d seconds for repository %s.' % ( timeout_counter, str( galaxy_repository.name ) ) ) diff -r f2ce61f7d1ad2ea38faedf952aa0abf60ca06e2c -r 10f283c957e5f8dbca6c93aae7655926600a3d12 test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py --- a/test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py +++ b/test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py @@ -9,27 +9,74 @@ class InstallTestRepositories( InstallTestRepository ): """Abstract test case that installs and uninstalls a predefined list of repositories.""" - def do_installation( self, repository_info_dict ): + def do_deactivate( self, repository_dict ): + self.logout() + self.login( email='test(a)bx.psu.edu', username='test' ) + admin_user = test_db_util.get_user( 'test(a)bx.psu.edu' ) + assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email + # Get the repository defined by the received repository_dict along with all of its repository dependencies + # from the database. + name = repository_dict[ 'name' ] + owner = repository_dict[ 'owner' ] + changeset_revision = repository_dict[ 'changeset_revision' ] + repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) + admin_user_private_role = test_db_util.get_private_role( admin_user ) + # Uninstall the repository through the web interface using twill. + self.deactivate_repository( repository ) + + def do_install( self, repository_dict ): self.logout() self.login( email='test(a)bx.psu.edu', username='test' ) admin_user = test_db_util.get_user( 'test(a)bx.psu.edu' ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email admin_user_private_role = test_db_util.get_private_role( admin_user ) - # Install the repository through the web interface using twill. - self.install_repository( repository_info_dict ) + # Install the repository through the web interface using twill. The install_repository() method may + # actually install more than this singe repository because repository dependencies can be installed. + self.install_repository( repository_dict ) - def do_deactivate_or_uninstall( self, repository_info_dict, deactivate=False ): + def do_uninstall( self, repository_dict ): self.logout() self.login( email='test(a)bx.psu.edu', username='test' ) admin_user = test_db_util.get_user( 'test(a)bx.psu.edu' ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email - # Get the repository from the database. - repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( repository_info_dict[ 'name' ], - repository_info_dict[ 'owner' ], - repository_info_dict[ 'changeset_revision' ] ) + # Get the repository defined by the received repository_dict along with all of its repository dependencies + # from the database. + name = repository_dict[ 'name' ] + owner = repository_dict[ 'owner' ] + changeset_revision = repository_dict[ 'changeset_revision' ] + repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) admin_user_private_role = test_db_util.get_private_role( admin_user ) - # Uninstall the repository through the web interface using twill. - self.deactivate_or_uninstall_repository( repository, deactivate ) + # Uninstall the repository and all of its repository dependencies through the Galaxy web interface using twill. + self.uninstall_repository( repository ) + +def generate_deactivate_method( repository_dict=None ): + """Generate abstract test cases for the received repository_dict.""" + if repository_dict is None: + return + # Push all the toolbox tests to module level + G = globals() + # Eliminate all previous tests from G. + for key, val in G.items(): + if key.startswith( 'TestInstallRepository_' ) or key.startswith( 'TestForTool_' ): + del G[ key ] + # Create a new subclass with a method named install_repository_XXX that deactivates the + # repository specified by the received repository_dict. + name = "TestDeactivateRepository_%s_%s" % \ + ( str( repository_dict[ 'name' ] ), str( repository_dict[ 'changeset_revision' ] ) ) + baseclasses = ( InstallTestRepositories, ) + namespace = dict() + def make_deactivate_method( repository_dict ): + def test_deactivate_repository( self ): + self.do_deactivate( repository_dict ) + return test_deactivate_repository + test_method = make_deactivate_method( repository_dict ) + test_method.__doc__ = "Deactivate the repository %s." % str( repository_dict[ 'name' ] ) + namespace[ 'uninstall_repository_%s_%s' % ( str( repository_dict[ 'name' ] ), + str( repository_dict[ 'changeset_revision' ] ) ) ] = test_method + # The new.classobj function returns a new class object, with name name, derived + # from baseclasses (which should be a tuple of classes) and with namespace dict. + new_class_obj = new.classobj( str( name ), baseclasses, namespace ) + G[ name ] = new_class_obj def generate_install_method( repository_dict=None ): """Generate abstract test cases for the defined list of repositories.""" @@ -47,17 +94,18 @@ namespace = dict() def make_install_method( repository_dict ): def test_install_repository( self ): - self.do_installation( repository_dict ) + self.do_install( repository_dict ) return test_install_repository test_method = make_install_method( repository_dict ) - test_method.__doc__ = "Install the repository %s from %s." % ( repository_dict[ 'name' ], repository_dict[ 'tool_shed_url' ] ) - namespace[ 'install_repository_%s' % repository_dict[ 'name' ] ] = test_method - # The new.classobj function returns a new class object, with name name, derived + test_method.__doc__ = "Install the repository %s from %s." % \ + ( str( repository_dict[ 'name' ] ), str( repository_dict[ 'tool_shed_url' ] ) ) + namespace[ 'install_repository_%s' % str( repository_dict[ 'name' ] ) ] = test_method + # The new.classobj function returns a new class object with name name derived # from baseclasses (which should be a tuple of classes) and with namespace dict. new_class_obj = new.classobj( str( name ), baseclasses, namespace ) G[ name ] = new_class_obj -def generate_deactivate_or_uninstall_method( repository_dict=None, deactivate=False ): +def generate_uninstall_method( repository_dict=None ): """Generate abstract test cases for the received repository_dict.""" if repository_dict is None: return @@ -67,17 +115,20 @@ for key, val in G.items(): if key.startswith( 'TestInstallRepository_' ) or key.startswith( 'TestForTool_' ): del G[ key ] - # Create a new subclass with a method named install_repository_XXX that installs the repository specified by the provided dict. - name = "TestUninstallRepository_%s_%s" % ( repository_dict[ 'name' ], repository_dict[ 'changeset_revision' ] ) + # Create a new subclass with a method named install_repository_XXX that installs the repository + # specified by the received repository_dict. + name = "TestUninstallRepository_%s_%s" % \ + ( str( repository_dict[ 'name' ] ), str( repository_dict[ 'changeset_revision' ] ) ) baseclasses = ( InstallTestRepositories, ) namespace = dict() - def make_deactivate_or_uninstall_method( repository_dict ): - def test_install_repository( self ): - self.do_deactivate_or_uninstall( repository_dict, deactivate ) - return test_install_repository - test_method = make_deactivate_or_uninstall_method( repository_dict ) - test_method.__doc__ = "Deactivate or uninstall the repository %s." % repository_dict[ 'name' ] - namespace[ 'uninstall_repository_%s_%s' % ( str( repository_dict[ 'name' ] ), repository_dict[ 'changeset_revision' ] ) ] = test_method + def make_uninstall_method( repository_dict ): + def test_uninstall_repository( self ): + self.do_uninstall( repository_dict ) + return test_uninstall_repository + test_method = make_uninstall_method( repository_dict ) + test_method.__doc__ = "Uninstall the repository %s." % repository_dict[ 'name' ] + namespace[ 'uninstall_repository_%s_%s' % ( str( repository_dict[ 'name' ] ), + str( repository_dict[ 'changeset_revision' ] ) ) ] = test_method # The new.classobj function returns a new class object, with name name, derived # from baseclasses (which should be a tuple of classes) and with namespace dict. new_class_obj = new.classobj( str( name ), baseclasses, namespace ) diff -r f2ce61f7d1ad2ea38faedf952aa0abf60ca06e2c -r 10f283c957e5f8dbca6c93aae7655926600a3d12 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -42,6 +42,7 @@ import tool_shed.util.shed_util_common as suc import urllib +from base.tool_shed_util import log_reason_repository_cannot_be_uninstalled from base.tool_shed_util import parse_tool_panel_config from install_and_test_tool_shed_repositories.base.util import get_database_version from install_and_test_tool_shed_repositories.base.util import get_repository_current_revision @@ -64,7 +65,6 @@ from functional import database_contexts - log = logging.getLogger( 'install_and_test_repositories' ) assert sys.version_info[ :2 ] >= ( 2, 6 ) @@ -222,58 +222,62 @@ return [] def deactivate_repository( app, repository_dict ): + sa_session = app.install_model.context # Clean out any generated tests. This is necessary for Twill. remove_generated_tests( app ) - sa_session = app.install_model.context.current # The dict contains the only repository the app should have installed at this point. - repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( str( repository_dict[ 'name' ] ), - str( repository_dict[ 'owner' ] ), - str( repository_dict[ 'changeset_revision' ] ) ) - # We have to do this through Twill, in order to maintain app.toolbox and shed_tool_conf.xml in a state that is valid for future tests. - for required_repository in repository.repository_dependencies: - repository_dict = dict( name=required_repository.name, - owner=required_repository.owner, - changeset_revision=required_repository.changeset_revision ) - # Generate a test method to uninstall this repository through the embedded Galaxy application's web interface. - test_install_repositories.generate_deactivate_or_uninstall_method( repository_dict, deactivate=True ) - log.debug( 'Changeset revision %s of %s repository %s selected for deactivation.' % \ - ( str( required_repository.changeset_revision ), str( required_repository.status ), str( required_repository.name ) ) ) - repository_dict = dict( name=repository.name, - owner=repository.owner, - changeset_revision=repository.changeset_revision ) - test_install_repositories.generate_deactivate_or_uninstall_method( repository_dict, deactivate=True ) - log.debug( 'Changeset revision %s of %s repository %s selected for deactivation.' % \ - ( str( repository.changeset_revision ), str( repository.status ), str( repository.name ) ) ) + name = str( repository_dict[ 'name' ] ) + owner = str( repository_dict[ 'owner' ] ) + changeset_revision = str( repository_dict[ 'changeset_revision' ] ) + repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) + repository_dict_for_deactivation = dict( name=name, + owner=owner, + changeset_revision=changeset_revision ) + log.debug( 'Changeset revision %s of repository %s owned by %s selected for deactivation.' % ( changeset_revision, name, owner ) ) + test_install_repositories.generate_deactivate_method( repository_dict_for_deactivation ) # Set up nose to run the generated uninstall method as a functional test. test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) test_config.configure( sys.argv ) - # Run the uninstall method. This method uses the Galaxy web interface to uninstall the previously installed - # repository and delete it from disk. + # Run the deactivate method which uses the Galaxy web interface to deactivate the repository. result, _ = run_tests( test_config ) success = result.wasSuccessful() if not success: - log.debug( 'Failed deactivation of repository %s.' % str( repository.name ) ) + log.debug( 'Deactivation of revision %s repository %s owned by %s failed.' % ( changeset_revision, name, owner ) ) -def extract_log_data( test_result, from_tool_test=True ): - '''Extract any useful data from the test_result.failures and test_result.errors attributes.''' - log_data = [] +def get_api_url( base, parts=[], params=None ): + if 'api' in parts and parts.index( 'api' ) != 0: + parts.pop( parts.index( 'api' ) ) + parts.insert( 0, 'api' ) + elif 'api' not in parts: + parts.insert( 0, 'api' ) + url = suc.url_join( base, *parts ) + if params is not None: + query_string = urllib.urlencode( params ) + url += '?%s' % query_string + return url + +def get_failed_test_dicts( test_result, from_tool_test=True ): + """Extract any useful data from the test_result.failures and test_result.errors attributes.""" + failed_test_dicts = [] for failure in test_result.failures + test_result.errors: - # Record the twill test identifier and information about the tool, so the repository owner can discover which test is failing. - test_id = str( failure[0] ) + # Record the twill test identifier and information about the tool so the repository owner + # can discover which test is failing. + test_id = str( failure[ 0 ] ) if not from_tool_test: tool_id = None tool_version = None else: tool_id, tool_version = get_tool_info_from_test_id( test_id ) - test_status = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version ) - log_output = failure[1].replace( '\\n', '\n' ) - # Remove debug output that the reviewer or owner doesn't need. + test_status_dict = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version ) + log_output = failure[ 1 ].replace( '\\n', '\n' ) + # Remove debug output. log_output = re.sub( r'control \d+:.+', r'', log_output ) log_output = re.sub( r'\n+', r'\n', log_output ) appending_to = 'output' tmp_output = {} output = {} - # Iterate through the functional test output and extract only the important data. Captured logging and stdout are not recorded. + # Iterate through the functional test output and extract only the important data. Captured + # logging and stdout are not recorded. for line in log_output.split( '\n' ): if line.startswith( 'Traceback' ): appending_to = 'traceback' @@ -290,26 +294,15 @@ elif '>> begin captured stderr <<' in line or '>> begin tool stderr <<' in line: appending_to = 'stderr' continue - if appending_to not in tmp_output: - tmp_output[ appending_to ] = [] - tmp_output[ appending_to ].append( line ) + if appending_to in tmp_output: + tmp_output[ appending_to ].append( line ) + else: + tmp_output[ appending_to ] = [ line ] for output_type in [ 'stderr', 'traceback' ]: if output_type in tmp_output: - test_status[ output_type ] = '\n'.join( tmp_output[ output_type ] ) - log_data.append( test_status ) - return log_data - -def get_api_url( base, parts=[], params=None ): - if 'api' in parts and parts.index( 'api' ) != 0: - parts.pop( parts.index( 'api' ) ) - parts.insert( 0, 'api' ) - elif 'api' not in parts: - parts.insert( 0, 'api' ) - url = suc.url_join( base, *parts ) - if params is not None: - query_string = urllib.urlencode( params ) - url += '?%s' % query_string - return url + test_status_dict[ output_type ] = '\n'.join( tmp_output[ output_type ] ) + failed_test_dicts.append( test_status_dict ) + return failed_test_dicts def get_latest_downloadable_changeset_revision( url, name, owner ): error_message = '' @@ -415,16 +408,16 @@ if tsr_changeset_revision == rti_changeset_revision: return repository_dicts, error_message return repository_dicts, error_message - # Get a list of repositories to test from the tool shed specified in the GALAXY_INSTALL_TEST_TOOL_SHED_URL environment variable. + # Get a list of repositories to test from the tool shed specified in the GALAXY_INSTALL_TEST_TOOL_SHED_URL + # environment variable. log.debug( "The Tool Shed's API url...\n%s" % str( api_url ) ) log.debug( "...retrieved %d repository revisions for testing." % len( repository_dicts ) ) - #if '-list_repositories' in sys.argv: log.debug( "Repository revisions for testing:" ) for repository_dict in repository_dicts: - log.debug( "Revision %s of repository %s owned by %s" % \ - ( str( repository_dict.get( 'changeset_revision', None ) ), \ - str( repository_dict.get( 'name', None ) ), \ - str( repository_dict.get( 'owner', None ) ) ) ) + name = str( repository_dict.get( 'name', None ) ) + owner = str( repository_dict.get( 'owner', None ) ) + changeset_revision = str( repository_dict.get( 'changeset_revision', None ) ) + log.debug( "Revision %s of repository %s owned by %s" % ( changeset_revision, name, owner ) ) return repository_dicts, error_message def get_static_settings(): @@ -488,16 +481,23 @@ log.debug( 'The following dependencies of this repository are missing, so skipping functional tests.' ) # In keeping with the standard display layout, add the error message to the dict for each tool individually. for dependency in repository.missing_tool_dependencies: - log.debug( 'Missing tool dependency %s of type %s version %s: %s' % \ - ( str( dependency.name ), str( dependency.type ), str( dependency.version ), unicodify( dependency.error_message ) ) ) + name = str( dependency.name ) + type = str( dependency.type ) + version = str( dependency.version ) + error_message = unicodify( dependency.error_message ) + log.debug( 'Missing tool dependency %s of type %s version %s: %s' % ( name, type, version, error_message ) ) test_result = dict( type=dependency.type, name=dependency.name, version=dependency.version, error_message=dependency.error_message ) tool_test_results_dict[ 'installation_errors' ][ 'tool_dependencies' ].append( test_result ) for dependency in repository.missing_repository_dependencies: + name = str( dependency.name ) + owner = str( dependency.owner ) + changeset_revision = str( dependency.changeset_revision ) + error_message = unicodify( dependency.error_message ) log.debug( 'Missing repository dependency %s changeset revision %s owned by %s: %s' % \ - ( str( dependency.name ), str( dependency.changeset_revision ), str( dependency.owner ), unicodify( dependency.error_message ) ) ) + ( name, changeset_revision, owner, error_message ) ) test_result = dict( tool_shed=dependency.tool_shed, name=dependency.name, owner=dependency.owner, @@ -509,7 +509,11 @@ do_not_test=False, test_install_error=True ) # TODO: do something useful with response_dict - response_dict = register_test_result( galaxy_tool_shed_url, tool_test_results_dicts, tool_test_results_dict, repository_dict, params ) + response_dict = register_test_result( galaxy_tool_shed_url, + tool_test_results_dicts, + tool_test_results_dict, + repository_dict, + params ) # Since this repository is missing components, we do not want to test it, so deactivate it or uninstall it. # The deactivate flag is set to True if the environment variable GALAXY_INSTALL_TEST_KEEP_TOOL_DEPENDENCIES # is set to 'true'. @@ -533,19 +537,19 @@ # Initialize a dictionary for the summary that will be printed to stdout. results_dict = {} results_dict[ 'total_repositories_tested' ] = 0 - results_dict[ 'repositories_passed' ] = [] - results_dict[ 'repositories_failed' ] = [] + results_dict[ 'all_tests_passed' ] = [] + results_dict[ 'at_least_one_test_failed' ] = [] results_dict[ 'repositories_failed_install' ] = [] return results_dict def install_repository( app, repository_dict ): - """Attempt to install a repository defined by the received repository_dict from the tool shed into Galaxy.""" + """Install a repository defined by the received repository_dict from the tool shed into Galaxy.""" name = str( repository_dict[ 'name' ] ) owner = str( repository_dict[ 'owner' ] ) changeset_revision = str( repository_dict[ 'changeset_revision' ] ) error_message = '' repository = None - log.debug( "Installing revision %s of repository %s owned by %s..." % ( str( changeset_revision ), str( name ), str( owner ) ) ) + log.debug( "Installing revision %s of repository %s owned by %s." % ( changeset_revision, name, owner ) ) # Explicitly clear tests from twill's test environment. remove_generated_tests( app ) # Use the repository information dictionary to generate an install method that will install the repository into the @@ -560,8 +564,7 @@ try: repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) except Exception, e: - error_message = 'Error getting revision %s of installed repository %s owned by %s: %s' % \ - ( str( changeset_revision ), str( name ), str( owner ), str( e ) ) + error_message = 'Error getting revision %s of repository %s owned by %s: %s' % ( changeset_revision, name, owner, str( e ) ) log.exception( error_message ) return repository, error_message @@ -624,6 +627,7 @@ owner = str( repository_dict[ 'owner' ] ) changeset_revision = str( repository_dict[ 'changeset_revision' ] ) log.debug( "Processing revision %s of repository %s owned by %s..." % ( changeset_revision, name, owner ) ) + repository_identifier_dict = dict( name=name, owner=owner, changeset_revision=changeset_revision ) # Retrieve the stored list of tool_test_results_dicts. tool_test_results_dicts, error_message = get_tool_test_results_dicts( galaxy_tool_shed_url, encoded_repository_metadata_id ) if error_message: @@ -715,7 +719,11 @@ test_install_error=True, do_not_test=False ) # TODO: do something useful with response_dict - response_dict = register_test_result( galaxy_tool_shed_url, tool_test_results_dicts, tool_test_results_dict, repository_dict, params ) + response_dict = register_test_result( galaxy_tool_shed_url, + tool_test_results_dicts, + tool_test_results_dict, + repository_dict, + params ) try: if deactivate: # We are deactivating this repository and all of its repository dependencies. @@ -725,7 +733,7 @@ uninstall_repository( app, repository_dict ) except: log.exception( 'Encountered error attempting to deactivate or uninstall %s.', str( repository_dict[ 'name' ] ) ) - results_dict[ 'repositories_failed_install' ].append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + results_dict[ 'repositories_failed_install' ].append( repository_identifier_dict ) log.debug( 'Repository %s failed to install correctly.' % str( name ) ) else: # Configure and run functional tests for this repository. This is equivalent to sh run_functional_tests.sh -installed @@ -773,12 +781,12 @@ do_not_test=False, test_install_error=False ) # TODO: do something useful with response_dict - response_dict = register_test_result( galaxy_tool_shed_url, - tool_test_results_dicts, - tool_test_results_dict, - repository_dict, - params ) - results_dict[ 'repositories_failed' ].append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + response_dict = register_test_result( galaxy_tool_shed_url, + tool_test_results_dicts, + tool_test_results_dict, + repository_dict, + params ) + results_dict[ 'at_least_one_test_failed' ].append( repository_identifier_dict ) total_repositories_tested += 1 results_dict[ 'total_repositories_tested' ] = total_repositories_tested return results_dict, error_message @@ -1015,8 +1023,8 @@ log.debug( error_message ) else: total_repositories_tested = results_dict[ 'total_repositories_tested' ] - repositories_passed = results_dict[ 'repositories_passed' ] - repositories_failed = results_dict[ 'repositories_failed' ] + all_tests_passed = results_dict[ 'all_tests_passed' ] + at_least_one_test_failed = results_dict[ 'at_least_one_test_failed' ] repositories_failed_install = results_dict[ 'repositories_failed_install' ] now = time.strftime( "%Y-%m-%d %H:%M:%S" ) print "####################################################################################" @@ -1025,19 +1033,19 @@ if not can_update_tool_shed: print "# This run will not update the Tool Shed database." if total_repositories_tested > 0: - if repositories_passed: + if all_tests_passed: print '# ----------------------------------------------------------------------------------' - print "# %d repositories passed all tests:" % len( repositories_passed ) - show_summary_output( repositories_passed ) - if repositories_failed: + print "# %d repositories successfully passed all functional tests:" % len( all_tests_passed ) + show_summary_output( all_tests_passed ) + if at_least_one_test_failed: print '# ----------------------------------------------------------------------------------' - print "# %d repositories failed one or more tests:" % len( repositories_failed ) - show_summary_output( repositories_failed ) + print "# %d repositories failed at least 1 functional test:" % len( at_least_one_test_failed ) + show_summary_output( at_least_one_test_failed ) if repositories_failed_install: # Set success to False so that the return code will not be 0. success = False print '# ----------------------------------------------------------------------------------' - print "# %d repositories with installation errors:" % len( repositories_failed_install ) + print "# %d repositories have installation errors:" % len( repositories_failed_install ) show_summary_output( repositories_failed_install ) else: success = True @@ -1198,21 +1206,31 @@ return result, test_config.plugins._plugins def show_summary_output( repository_dicts ): - repositories_by_owner = {} - for repository in repository_dicts: - if repository[ 'owner' ] not in repositories_by_owner: - repositories_by_owner[ repository[ 'owner' ] ] = [] - repositories_by_owner[ repository[ 'owner' ] ].append( repository ) - for owner in repositories_by_owner: + # Group summary display by repository owner. + repository_dicts_by_owner = {} + for repository_dict in repository_dicts: + name = str( repository_dict[ 'name' ] ) + owner = str( repository_dict[ 'owner' ] ) + changeset_revision = str( repository_dict[ 'changeset_revision' ] ) + if owner in repository_dicts_by_owner: + repository_dicts_by_owner[ owner ].append( repository_dict ) + else: + repository_dicts_by_owner[ owner ] = [ repository_dict ] + # Display grouped summary. + for owner, grouped_repository_dicts in repository_dicts_by_owner.items(): print "# " - for repository in repositories_by_owner[ owner ]: - print "# %s owned by %s, changeset revision %s" % ( repository[ 'name' ], repository[ 'owner' ], repository[ 'changeset_revision' ] ) + for repository_dict in grouped_repository_dicts: + name = repository_dict[ 'name' ] + owner = repository_dict[ 'owner' ] + changeset_revision = repository_dict[ 'changeset_revision' ] + print "# Revision %s of repository %s owned by %s" % ( changeset_revision, name, owner ) def test_repository_tools( app, repository, repository_dict, tool_test_results_dicts, tool_test_results_dict, results_dict ): """Test tools contained in the received repository.""" name = str( repository.name ) owner = str( repository.owner ) changeset_revision = str( repository.changeset_revision ) + repository_identifier_dict = dict( name=name, owner=owner, changeset_revision=changeset_revision ) # Set the module-level variable 'toolbox', so that test.functional.test_toolbox will generate the appropriate test methods. test_toolbox.toolbox = app.toolbox # Generate the test methods for this installed repository. We need to pass in True here, or it will look @@ -1223,43 +1241,55 @@ test_config.configure( sys.argv ) # Run the configured tests. result, test_plugins = run_tests( test_config ) - success = result.wasSuccessful() - # Use the ReportResults nose plugin to get a list of tests that passed. - for plugin in test_plugins: - if hasattr( plugin, 'getTestStatus' ): - test_identifier = '%s/%s' % ( owner, name ) - passed_tests = plugin.getTestStatus( test_identifier ) - break - tool_test_results_dict[ 'passed_tests' ] = [] - for test_id in passed_tests: - # Normalize the tool ID and version display. - tool_id, tool_version = get_tool_info_from_test_id( test_id ) - test_result = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version ) - tool_test_results_dict[ 'passed_tests' ].append( test_result ) - if success: - # This repository's tools passed all functional tests. Update the repository_metadata table in the tool shed's database - # to reflect that. Call the register_test_result method, which executes a PUT request to the repository_revisions API - # controller with the status of the test. This also sets the do_not_test and tools_functionally correct flags, and - # updates the time_last_tested field to today's date. - results_dict[ 'repositories_passed' ].append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + if result.wasSuccessful(): + # This repository's tools passed all functional tests. Use the ReportResults nose plugin to get a list + # of tests that passed. + for plugin in test_plugins: + if hasattr( plugin, 'getTestStatus' ): + test_identifier = '%s/%s' % ( owner, name ) + passed_tests = plugin.getTestStatus( test_identifier ) + break + tool_test_results_dict[ 'passed_tests' ] = [] + for test_id in passed_tests: + # Normalize the tool ID and version display. + tool_id, tool_version = get_tool_info_from_test_id( test_id ) + test_result = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version ) + tool_test_results_dict[ 'passed_tests' ].append( test_result ) + # Update the repository_metadata table in the tool shed's database to include the passed tests. + passed_repository_dict = repository_identifier_dict + results_dict[ 'all_tests_passed' ].append( passed_repository_dict ) params = dict( tools_functionally_correct=True, do_not_test=False, test_install_error=False ) + # Call the register_test_result() method to execute a PUT request to the repository_revisions API + # controller with the status of the test. This also sets the do_not_test and tools_functionally + # correct flags and updates the time_last_tested field to today's date. # TODO: do something useful with response_dict - response_dict = register_test_result( galaxy_tool_shed_url, tool_test_results_dicts, tool_test_results_dict, repository_dict, params ) - log.debug( 'Revision %s of repository %s installed and passed functional tests.' % ( str( changeset_revision ), str( name ) ) ) + response_dict = register_test_result( galaxy_tool_shed_url, + tool_test_results_dicts, + tool_test_results_dict, + repository_dict, + params ) + log.debug( 'Revision %s of repository %s owned by %s installed and passed functional tests.' % \ + ( changeset_revision, name, owner ) ) else: - # The extract_log_data() netod returns a list. - tool_test_results_dict[ 'failed_tests' ] = extract_log_data( result, from_tool_test=True ) - results_dict[ 'repositories_failed' ].append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + # The get_failed_test_dicts() method returns a list. + failed_test_dicts = get_failed_test_dicts( result, from_tool_test=True ) + tool_test_results_dict[ 'failed_tests' ] = failed_test_dicts + failed_repository_dict = repository_identifier_dict + results_dict[ 'at_least_one_test_failed' ].append( failed_repository_dict ) set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_dict ) params = dict( tools_functionally_correct=False, test_install_error=False, do_not_test=str( set_do_not_test ) ) # TODO: do something useful with response_dict - response_dict = register_test_result( galaxy_tool_shed_url, tool_test_results_dicts, tool_test_results_dict, repository_dict, params ) - log.debug( 'Revision %s of repository %s installed successfully but did not pass functional tests.' % \ - ( str( changeset_revision ), str( name ) ) ) + response_dict = register_test_result( galaxy_tool_shed_url, + tool_test_results_dicts, + tool_test_results_dict, + repository_dict, + params ) + log.debug( 'Revision %s of repository %s owned by %s installed successfully but did not pass functional tests.' % \ + ( changeset_revision, name, owner ) ) # Run the uninstall method. This removes tool functional test methods from the test_toolbox module and uninstalls the # repository using Twill. deactivate = asbool( os.environ.get( 'GALAXY_INSTALL_TEST_KEEP_TOOL_DEPENDENCIES', False ) ) @@ -1270,47 +1300,81 @@ else: log.debug( 'Uninstalling changeset revision %s of repository %s' % ( str( changeset_revision ), str( name ) ) ) # We are uninstalling this repository and all of its repository dependencies. - uninstall_repository( app, repository_dict ) - + uninstall_repository_and_repository_dependencies( app, repository_dict ) # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. test_toolbox.toolbox = app.toolbox return results_dict -def uninstall_repository( app, repository_dict ): - """Attempt to uninstall a repository.""" - sa_session = app.model.context.current +def uninstall_repository_and_repository_dependencies( app, repository_dict ): + """Uninstall a repository and all of its repository dependencies.""" + # This method assumes that the repositor defined by the received repository_dict is not a repository + # dependency of another repository. + sa_session = app.install_model.context # Clean out any generated tests. This is necessary for Twill. remove_generated_tests( app ) # The dict contains the only repository the app should have installed at this point. name = str( repository_dict[ 'name' ] ) owner = str( repository_dict[ 'owner' ] ) changeset_revision = str( repository_dict[ 'changeset_revision' ] ) + # Since this install and test framework uninstalls repositories immediately after installing and testing + # them, the values of repository.installed_changeset_revision and repository.changeset_revision should be + # the same. repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) - # We have to do this through Twill, in order to maintain app.toolbox and shed_tool_conf.xml in a state that is valid for future tests. - for required_repository in repository.repository_dependencies: - repository_dict = dict( name=str( required_repository.name ), - owner=str( required_repository.owner ), - changeset_revision=str( required_repository.changeset_revision ) ) - # Generate a test method to uninstall this repository through the embedded Galaxy application's web interface. - test_install_repositories.generate_deactivate_or_uninstall_method( repository_dict, deactivate=False ) - log.debug( 'Changeset revision %s of %s repository %s selected for uninstallation.' % \ - ( str( required_repository.changeset_revision ), str( required_repository.status ), str( required_repository.name ) ) ) - repository_dict = dict( name=name, owner=owner, changeset_revision=changeset_revision ) - test_install_repositories.generate_deactivate_or_uninstall_method( repository_dict, deactivate=False ) - log.debug( 'Changeset revision %s of %s repository %s selected for uninstallation.' % ( changeset_revision, str( repository.status ), name ) ) - # Set up nose to run the generated uninstall method as a functional test. - test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) - test_config.configure( sys.argv ) - # Run the uninstall method. This method uses the Galaxy web interface to uninstall the previously installed - # repository and delete it from disk. - result, _ = run_tests( test_config ) - success = result.wasSuccessful() - if not success: - log.debug( 'Repository %s failed to uninstall.' % str( name ) ) + if repository.can_uninstall( app ): + # A repository can be uninstalled only if no dependent repositories are installed. So uninstallation order + # id critical. A repository is always uninstalled first, and the each of its dependencies is checked to see + # if it can be uninstalled. + uninstall_repository_dict = dict( name=name, + owner=owner, + changeset_revision=changeset_revision ) + log.debug( 'Revision %s of repository %s owned by %s selected for uninstallation.' % ( changeset_revision, name, owner ) ) + test_install_repositories.generate_uninstall_method( uninstall_repository_dict ) + # Set up nose to run the generated uninstall method as a functional test. + test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) + test_config.configure( sys.argv ) + # Run the uninstall method. This method uses the Galaxy web interface to uninstall the previously installed + # repository and all of its repository dependencies, deleting each of them from disk. + result, _ = run_tests( test_config ) + success = result.wasSuccessful() + if success: + # Now that the repository is uninstalled we can attempt to uninstall each of its repository dependencies. + # We have to do this through Twill in order to maintain app.toolbox and shed_tool_conf.xml in a state that + # is valid for future tests. Since some of the repository's repository dependencies may require other of + # the repository's repository dependencies, we'll keep track of the repositories we've been able to unistall. + processed_repository_dependency_ids = [] + while len( processed_repository_dependency_ids ) < len( repository.repository_dependencies ): + for repository_dependency in repository.repository_dependencies: + if repository_dependency.id not in processed_repository_dependency_ids and repository_dependency.can_uninstall( app ): + processed_repository_dependency_ids.append( repository_dependency.id ) + rd_name = str( repository_dependency.name ) + rd_owner = str( repository_dependency.owner ) + rd_changeset_revision = str( repository_dependency.changeset_revision ) + uninstall_repository_dict = dict( name=rd_name, + owner=rd_owner, + changeset_revision=rd_changeset_revision ) + log.debug( 'Revision %s of repository dependency %s owned by %s selected for uninstallation.' % \ + ( rd_changeset_revision, rd_name, rd_owner ) ) + # Generate a test method to uninstall the repository dependency through the embedded Galaxy application's + # web interface. + test_install_repositories.generate_uninstall_method( uninstall_repository_dict ) + # Set up nose to run the generated uninstall method as a functional test. + test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) + test_config.configure( sys.argv ) + # Run the uninstall method. + result, _ = run_tests( test_config ) + success = result.wasSuccessful() + if not success: + log.debug( 'Uninstallation of revision %s of repository %s owned by %s failed: %s' % \ + ( rd_changeset_revision, rd_name, rd_owner, str( e ) ) ) + break + else: + log.debug( 'Uninstallation of revision %s of repository %s owned by %s failed.' % ( changeset_revision, name, owner ) ) + else: + log_reason_repository_cannot_be_uninstalled( app, repository ) def uninstall_tool_dependency( app, tool_dependency ): """Attempt to uninstall a tool dependency.""" - sa_session = app.model.context.current + sa_session = app.install_model.context # Clean out any generated tests. This is necessary for Twill. tool_dependency_install_path = tool_dependency.installation_directory( app ) uninstalled, error_message = tool_dependency_util.remove_tool_dependency( app, tool_dependency ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: JavaScript MVC: remove BaseModel because it is not used.
by commits-noreply＠bitbucket.org 16 Dec '13

16 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f2ce61f7d1ad/ Changeset: f2ce61f7d1ad User: jgoecks Date: 2013-12-16 21:35:51 Summary: JavaScript MVC: remove BaseModel because it is not used. Affected #: 3 files diff -r c49caf699d9c5fdec10763ea01b3a167b9ac4bdd -r f2ce61f7d1ad2ea38faedf952aa0abf60ca06e2c static/scripts/mvc/base-mvc.js --- a/static/scripts/mvc/base-mvc.js +++ b/static/scripts/mvc/base-mvc.js @@ -1,26 +1,3 @@ -/** - * Simple base model for any visible element. Includes useful attributes and ability - * to set and track visibility. - */ -var BaseModel = Backbone.Model.extend({ - defaults: { - name: null, - hidden: false - }, - - show: function() { - this.set("hidden", false); - }, - - hide: function() { - this.set("hidden", true); - }, - - is_visible: function() { - return !this.attributes.hidden; - } -}); - /** * Base view that handles visibility based on model's hidden attribute. */ @@ -52,10 +29,10 @@ * * @example * // Add to your models/views at the definition using chaining: - * var MyModel = BaseModel.extend( LoggableMixin ).extend({ // ... }); + * var MyModel = Backbone.Model.extend( LoggableMixin ).extend({ // ... }); * * // or - more explicitly AFTER the definition: - * var MyModel = BaseModel.extend({ + * var MyModel = Backbone.Model.extend({ * logger : console * // ... * this.log( '$#%& it! - broken already...' ); diff -r c49caf699d9c5fdec10763ea01b3a167b9ac4bdd -r f2ce61f7d1ad2ea38faedf952aa0abf60ca06e2c static/scripts/mvc/history/history-model.js --- a/static/scripts/mvc/history/history-model.js +++ b/static/scripts/mvc/history/history-model.js @@ -57,7 +57,6 @@ * @param {Object} historyJSON model data for this History * @param {Object[]} hdaJSON array of model data for this History's HDAs * @param {Object} options any extra settings including logger - * @see BaseModel#initialize */ initialize : function( historyJSON, hdaJSON, options ){ options = options || {}; diff -r c49caf699d9c5fdec10763ea01b3a167b9ac4bdd -r f2ce61f7d1ad2ea38faedf952aa0abf60ca06e2c static/scripts/mvc/user/user-model.js --- a/static/scripts/mvc/user/user-model.js +++ b/static/scripts/mvc/user/user-model.js @@ -67,7 +67,7 @@ if( idOrCurrent === User.CURRENT_ID_STR ){ options.url = this.urlRoot + '/' + User.CURRENT_ID_STR; } - return BaseModel.prototype.fetch.call( this, options ); + return Backbone.Model.prototype.fetch.call( this, options ); }, /** Clears all data from the sessionStorage. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Placeholder in api documentation for Return Codes and Format standards.
by commits-noreply＠bitbucket.org 16 Dec '13

16 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c49caf699d9c/ Changeset: c49caf699d9c User: dannon Date: 2013-12-16 21:17:51 Summary: Placeholder in api documentation for Return Codes and Format standards. Affected #: 1 file diff -r 4fe46fd3181bfb63e8093200d71801cb507b586d -r c49caf699d9c5fdec10763ea01b3a167b9ac4bdd doc/source/lib/galaxy.webapps.galaxy.api.rst --- a/doc/source/lib/galaxy.webapps.galaxy.api.rst +++ b/doc/source/lib/galaxy.webapps.galaxy.api.rst @@ -205,6 +205,10 @@ The request and response format should be considered alpha and are subject to change. +API Return Codes and Formats +================== + +TODO (here, or add a new page and link it) API Controllers =============== Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Remove debugging code.
by commits-noreply＠bitbucket.org 14 Dec '13

14 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/4fe46fd3181b/ Changeset: 4fe46fd3181b User: inithello Date: 2013-12-14 05:23:32 Summary: Remove debugging code. Affected #: 2 files diff -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 -r 4fe46fd3181bfb63e8093200d71801cb507b586d test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -132,7 +132,6 @@ galaxy_model.ToolShedRepository.installation_status.DEACTIVATED ]: tool_panel_section = self.get_tool_panel_section_from_repository_metadata( metadata ) else: - log.debug( 'Status is %s' % repository.status ) tool_panel_section = self.get_tool_panel_section_from_api( metadata ) assert tool_panel_section == expected_tool_panel_section, 'Expected to find tool panel section *%s*, but instead found *%s*\nMetadata: %s\n' % \ ( expected_tool_panel_section, tool_panel_section, metadata ) @@ -1070,13 +1069,11 @@ def reset_metadata_on_selected_repositories( self, repository_ids ): self.visit_url( '/admin/reset_metadata_on_selected_repositories_in_tool_shed' ) - log.debug( repository_ids ) kwd = dict( repository_ids=repository_ids ) self.submit_form( form_no=1, button="reset_metadata_on_selected_repositories_button", **kwd ) def reset_metadata_on_selected_installed_repositories( self, repository_ids ): self.visit_galaxy_url( '/admin_toolshed/reset_metadata_on_selected_installed_repositories' ) - log.debug( repository_ids ) kwd = dict( repository_ids=repository_ids ) self.submit_form( form_no=1, button="reset_metadata_on_selected_repositories_button", **kwd ) diff -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 -r 4fe46fd3181bfb63e8093200d71801cb507b586d test/tool_shed/functional_tests.py --- a/test/tool_shed/functional_tests.py +++ b/test/tool_shed/functional_tests.py @@ -85,9 +85,6 @@ # should this serve static resources (scripts, images, styles, etc.) STATIC_ENABLED = True -global galaxyapp -galaxyapp = None - def get_static_settings(): """Returns dictionary of the settings necessary for a galaxy App to be wrapped in the static middleware. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Update functional tests to account for the changes to when repositories can be uninstalled. Add tool panel section to tool API return value. Move test_db_util import to twilltestcase. Update tool test metadata generation.
by commits-noreply＠bitbucket.org 14 Dec '13

14 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c2511b37db9b/ Changeset: c2511b37db9b User: inithello Date: 2013-12-14 05:07:07 Summary: Update functional tests to account for the changes to when repositories can be uninstalled. Add tool panel section to tool API return value. Move test_db_util import to twilltestcase. Update tool test metadata generation. Affected #: 69 files diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1075,6 +1075,22 @@ :returns: galaxy.jobs.JobDestination -- The destination definition and runner parameters. """ return self.app.job_config.get_destination(self.__get_job_tool_configuration(job_params=job_params).destination) + + def get_panel_section( self ): + for key, item in self.app.toolbox.integrated_tool_panel.items(): + if item: + if key.startswith( 'tool_' ): + if item.id == self.id: + return '', '' + if key.startswith( 'section_' ): + section_id = item.id or '' + section_name = item.name or '' + for section_key, section_item in item.elems.items(): + if section_key.startswith( 'tool_' ): + if section_item: + if section_item.id == self.id: + return section_id, section_name + return None, None def parse( self, root, guid=None ): """ @@ -3010,6 +3026,8 @@ if io_details: tool_dict[ 'inputs' ] = [ input.to_dict( trans ) for input in self.inputs.values() ] tool_dict[ 'outputs' ] = [ output.to_dict() for output in self.outputs.values() ] + + tool_dict[ 'panel_section_id' ], tool_dict[ 'panel_section_name' ] = self.get_panel_section() return tool_dict diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -940,7 +940,12 @@ for required_file in ttb.required_files: value, extra = required_file required_files.append( ( value ) ) - inputs = [ input for input in ttb.inputs ] + inputs = [] + for input_field, values in ttb.inputs.iteritems(): + if len( values ) == 1: + inputs.append( ( input_field, values[0] ) ) + else: + inputs.append( ( input_field, values ) ) outputs = [] for output in ttb.outputs: name, file_name, extra = output diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -9,6 +9,7 @@ import time import tempfile import tarfile +import urllib import galaxy.webapps.tool_shed.util.hgweb_config import galaxy.model.tool_shed_install as galaxy_model import galaxy.util as util @@ -46,6 +47,7 @@ self.tool_shed_test_file = None self.tool_data_path = os.environ.get( 'GALAXY_TEST_TOOL_DATA_PATH' ) self.shed_tool_conf = os.environ.get( 'GALAXY_TEST_SHED_TOOL_CONF' ) + self.test_db_util = test_db_util # TODO: Figure out a way to alter these attributes during tests. self.galaxy_tool_dependency_dir = os.environ.get( 'GALAXY_TEST_TOOL_DEPENDENCY_DIR' ) self.shed_tools_dict = {} @@ -121,20 +123,17 @@ def check_galaxy_repository_tool_panel_section( self, repository, expected_tool_panel_section ): metadata = repository.metadata - assert 'tools' in metadata, 'Tools not found in metadata: %s' % metadata + assert 'tools' in metadata, 'Tools not found in repository metadata: %s' % metadata tool_metadata = metadata[ 'tools' ] # If integrated_tool_panel.xml is to be tested, this test method will need to be enhanced to handle tools # from the same repository in different tool panel sections. Getting the first tool guid is ok, because # currently all tools contained in a single repository will be loaded into the same tool panel section. - tool_guid = tool_metadata[ 0 ][ 'guid' ] - assert 'tool_panel_section' in metadata, 'Tool panel section not found in metadata: %s' % metadata - tool_panel_section_metadata = metadata[ 'tool_panel_section' ] - # tool_section_dict = dict( tool_config=guids_and_configs[ guid ], - # id=section_id, - # name=section_name, - # version=section_version ) - # This dict is appended to tool_panel_section_metadata[ tool_guid ] - tool_panel_section = tool_panel_section_metadata[ tool_guid ][ 0 ][ 'name' ] + if repository.status in [ galaxy_model.ToolShedRepository.installation_status.UNINSTALLED, + galaxy_model.ToolShedRepository.installation_status.DEACTIVATED ]: + tool_panel_section = self.get_tool_panel_section_from_repository_metadata( metadata ) + else: + log.debug( 'Status is %s' % repository.status ) + tool_panel_section = self.get_tool_panel_section_from_api( metadata ) assert tool_panel_section == expected_tool_panel_section, 'Expected to find tool panel section *%s*, but instead found *%s*\nMetadata: %s\n' % \ ( expected_tool_panel_section, tool_panel_section, metadata ) @@ -765,6 +764,28 @@ invalid_tools.append( dict( tools=repository_metadata.metadata[ 'invalid_tools' ], changeset_revision=repository_metadata.changeset_revision ) ) return valid_tools, invalid_tools + def get_tool_panel_section_from_api( self, metadata ): + tool_metadata = metadata[ 'tools' ] + tool_guid = urllib.quote_plus( tool_metadata[ 0 ][ 'guid' ], safe='' ) + api_url = '/%s' % '/'.join( [ 'api', 'tools', tool_guid ] ) + self.visit_galaxy_url( api_url ) + tool_dict = from_json_string( self.last_page() ) + tool_panel_section = tool_dict[ 'panel_section_name' ] + return tool_panel_section + + def get_tool_panel_section_from_repository_metadata( self, metadata ): + tool_metadata = metadata[ 'tools' ] + tool_guid = tool_metadata[ 0 ][ 'guid' ] + assert 'tool_panel_section' in metadata, 'Tool panel section not found in metadata: %s' % metadata + tool_panel_section_metadata = metadata[ 'tool_panel_section' ] + # tool_section_dict = dict( tool_config=guids_and_configs[ guid ], + # id=section_id, + # name=section_name, + # version=section_version ) + # This dict is appended to tool_panel_section_metadata[ tool_guid ] + tool_panel_section = tool_panel_section_metadata[ tool_guid ][ 0 ][ 'name' ] + return tool_panel_section + def grant_role_to_user( self, user, role ): strings_displayed = [ self.security.encode_id( role.id ), role.name ] strings_not_displayed = [] @@ -1049,11 +1070,13 @@ def reset_metadata_on_selected_repositories( self, repository_ids ): self.visit_url( '/admin/reset_metadata_on_selected_repositories_in_tool_shed' ) + log.debug( repository_ids ) kwd = dict( repository_ids=repository_ids ) self.submit_form( form_no=1, button="reset_metadata_on_selected_repositories_button", **kwd ) def reset_metadata_on_selected_installed_repositories( self, repository_ids ): self.visit_galaxy_url( '/admin_toolshed/reset_metadata_on_selected_installed_repositories' ) + log.debug( repository_ids ) kwd = dict( repository_ids=repository_ids ) self.submit_form( form_no=1, button="reset_metadata_on_selected_repositories_button", **kwd ) @@ -1173,16 +1196,15 @@ self.visit_url( url ) self.check_for_strings( strings_displayed, strings_not_displayed ) - def uninstall_repository( self, installed_repository, remove_from_disk=True ): + def uninstall_repository( self, installed_repository, remove_from_disk=True, is_required=False, strings_displayed=[], strings_not_displayed=[] ): url = '/admin_toolshed/deactivate_or_uninstall_repository?id=%s' % self.security.encode_id( installed_repository.id ) self.visit_galaxy_url( url ) - if remove_from_disk: - tc.fv ( 1, "remove_from_disk", 'true' ) - else: - tc.fv ( 1, "remove_from_disk", 'false' ) + self.check_for_strings( strings_displayed, strings_not_displayed ) + form = tc.browser.get_form( 'deactivate_or_uninstall_repository' ) + kwd = self.set_form_value( form, {}, 'remove_from_disk', remove_from_disk ) tc.submit( 'deactivate_or_uninstall_repository_button' ) strings_displayed = [ 'The repository named' ] - if remove_from_disk: + if remove_from_disk and not is_required: strings_displayed.append( 'has been uninstalled' ) else: strings_displayed.append( 'has been deactivated' ) diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0000_basic_repository_features.py --- a/test/tool_shed/functional/test_0000_basic_repository_features.py +++ b/test/tool_shed/functional/test_0000_basic_repository_features.py @@ -1,11 +1,8 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util - repository_name = 'filtering_0000' repository_description = "Galaxy's filtering tool for test 0000" repository_long_description = "Long description of Galaxy's filtering tool for test 0000" - class TestBasicRepositoryFeatures( ShedTwillTestCase ): '''Test core repository features.''' @@ -13,19 +10,19 @@ """Create necessary user accounts and login as an admin user.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % common.test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.test_user_2_email, username=common.test_user_2_name ) - test_user_2 = test_db_util.get_user( common.test_user_2_email ) + test_user_2 = self.test_db_util.get_user( common.test_user_2_email ) assert test_user_2 is not None, 'Problem retrieving user with email %s from the database' % common.test_user_2_email - test_user_2_private_role = test_db_util.get_private_role( test_user_2 ) + test_user_2_private_role = self.test_db_util.get_private_role( test_user_2 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_categories( self ): """Create categories for this test suite""" @@ -36,7 +33,7 @@ """Create the filtering repository""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - category = test_db_util.get_category_by_name( 'Test 0000 Basic Repository Features 1' ) + category = self.test_db_util.get_category_by_name( 'Test 0000 Basic Repository Features 1' ) strings_displayed = [ 'Repository %s' % "'%s'" % repository_name, 'Repository %s has been created' % "<b>%s</b>" % repository_name ] self.get_or_create_repository( name=repository_name, @@ -48,7 +45,7 @@ def test_0015_edit_repository( self ): """Edit the repository name, description, and long description""" - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) new_name = "renamed_filtering" new_description = "Edited filtering tool" new_long_description = "Edited long description" @@ -56,20 +53,20 @@ def test_0020_change_repository_category( self ): """Change the categories associated with the filtering repository""" - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.edit_repository_categories( repository, categories_to_add=[ "Test 0000 Basic Repository Features 2" ], categories_to_remove=[ "Test 0000 Basic Repository Features 1" ] ) def test_0025_grant_write_access( self ): '''Grant write access to another user''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.grant_write_access( repository, usernames=[ common.test_user_2_name ] ) self.revoke_write_access( repository, common.test_user_2_name ) def test_0030_upload_filtering_1_1_0( self ): """Upload filtering_1.1.0.tar to the repository""" - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.upload_file( repository, filename='filtering/filtering_1.1.0.tar', filepath=None, @@ -82,7 +79,7 @@ def test_0035_verify_repository( self ): '''Display basic repository pages''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) latest_changeset_revision = self.get_repository_tip( repository ) self.check_for_valid_tools( repository, strings_displayed=[ 'Filter1' ] ) self.check_count_of_metadata_revisions_associated_with_repository( repository, metadata_count=1 ) @@ -106,7 +103,7 @@ def test_0040_alter_repository_states( self ): '''Test toggling the malicious and deprecated repository flags.''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) self.set_repository_malicious( repository, @@ -132,7 +129,7 @@ def test_0045_display_repository_tip_file( self ): '''Display the contents of filtering.xml in the repository tip revision''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.display_repository_file_contents( repository=repository, filename='filtering.xml', filepath=None, @@ -141,7 +138,7 @@ def test_0050_upload_filtering_txt_file( self ): '''Upload filtering.txt file associated with tool version 1.1.0.''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.upload_file( repository, filename='filtering/filtering_0000.txt', filepath=None, @@ -155,7 +152,7 @@ def test_0055_upload_filtering_test_data( self ): '''Upload filtering test data.''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.upload_file( repository, filename='filtering/filtering_test_data.tar', filepath=None, @@ -174,7 +171,7 @@ def test_0060_upload_filtering_2_2_0( self ): '''Upload filtering version 2.2.0''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.upload_file( repository, filename='filtering/filtering_2.2.0.tar', filepath=None, @@ -187,8 +184,8 @@ def test_0065_verify_filtering_repository( self ): '''Verify the new tool versions and repository metadata.''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) - category = test_db_util.get_category_by_name( 'Test 0000 Basic Repository Features 1' ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + category = self.test_db_util.get_category_by_name( 'Test 0000 Basic Repository Features 1' ) tip = self.get_repository_tip( repository ) self.check_for_valid_tools( repository ) strings_displayed = [ 'Select a revision' ] @@ -209,7 +206,7 @@ def test_0070_upload_readme_txt_file( self ): '''Upload readme.txt file associated with tool version 2.2.0.''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.upload_file( repository, filename='readme.txt', filepath=None, @@ -228,26 +225,26 @@ def test_0075_delete_readme_txt_file( self ): '''Delete the readme.txt file.''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.delete_files_from_repository( repository, filenames=[ 'readme.txt' ] ) self.check_count_of_metadata_revisions_associated_with_repository( repository, metadata_count=2 ) self.display_manage_repository_page( repository, strings_displayed=[ 'Readme file for filtering 1.1.0' ] ) def test_0080_search_for_valid_filter_tool( self ): '''Search for the filtering tool by tool ID, name, and version.''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) tip_changeset = self.get_repository_tip( repository ) search_fields = dict( tool_id='Filter1', tool_name='filter', tool_version='2.2.0' ) self.search_for_valid_tools( search_fields=search_fields, strings_displayed=[ tip_changeset ], strings_not_displayed=[] ) def test_0085_verify_repository_metadata( self ): '''Verify that resetting the metadata does not change it.''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.verify_unchanged_repository_metadata( repository ) def test_0090_verify_reserved_repository_name_handling( self ): '''Check that reserved repository names are handled correctly.''' - category = test_db_util.get_category_by_name( 'Test 0000 Basic Repository Features 1' ) + category = self.test_db_util.get_category_by_name( 'Test 0000 Basic Repository Features 1' ) error_message = 'The term <b>repos</b> is a reserved word in the tool shed, so it cannot be used as a repository name.' self.get_or_create_repository( name='repos', description=repository_description, @@ -260,7 +257,7 @@ '''Check that reserved usernames are handled correctly.''' self.logout() self.login( email='baduser(a)bx.psu.edu', username='repos' ) - test_user_1 = test_db_util.get_user( 'baduser(a)bx.psu.edu' ) + test_user_1 = self.test_db_util.get_user( 'baduser(a)bx.psu.edu' ) assert test_user_1 is None, 'Creating user with public name "repos" succeeded.' error_message = 'The term <b>repos</b> is a reserved word in the tool shed, so it cannot be used as a public user name.' self.check_for_strings( strings_displayed=[ error_message ] ) @@ -275,7 +272,7 @@ ''' self.logout() self.login( email=common.test_user_2_email, username=common.test_user_2_name ) - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) message = 'This is a test message.' strings_displayed = [ 'Contact the owner of the repository named', repository.name, 'streamline appropriate communication' ] post_submit_strings_displayed = [ 'An error occurred sending your message by email' ] @@ -286,25 +283,25 @@ def test_0110_delete_filtering_repository( self ): '''Delete the filtering_0000 repository and verify that it no longer has any downloadable revisions.''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) self.delete_repository( repository ) # Explicitly reload all metadata revisions from the database, to ensure that we have the current status of the downloadable flag. for metadata_revision in repository.metadata_revisions: - test_db_util.refresh( metadata_revision ) + self.test_db_util.refresh( metadata_revision ) # Marking a repository as deleted should result in no metadata revisions being downloadable. assert True not in [ metadata.downloadable for metadata in repository.metadata_revisions ] def test_0115_undelete_filtering_repository( self ): '''Undelete the filtering_0000 repository and verify that it now has two downloadable revisions.''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) self.undelete_repository( repository ) # Explicitly reload all metadata revisions from the database, to ensure that we have the current status of the downloadable flag. for metadata_revision in repository.metadata_revisions: - test_db_util.refresh( metadata_revision ) + self.test_db_util.refresh( metadata_revision ) # Marking a repository as undeleted should result in all previously downloadable metadata revisions being downloadable again. # In this case, there should be two downloadable revisions, one for filtering 1.1.0 and one for filtering 2.2.0. assert True in [ metadata.downloadable for metadata in repository.metadata_revisions ] @@ -316,7 +313,7 @@ self.logout() self.login( email=common.test_user_2_email, username=common.test_user_2_name ) # Get the repository, so we can pass the encoded repository id and browse_repositories method to the set_email_alerts method. - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) strings_displayed = [ 'Total alerts added: 1, total alerts removed: 0' ] self.enable_email_alerts( repository, strings_displayed=strings_displayed ) @@ -324,7 +321,7 @@ '''Upload a new readme file to the filtering_0000 repository and verify that there is no error.''' self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) # Upload readme.txt to the filtering_0000 repository and verify that it is now displayed. self.upload_file( repository, filename='filtering/readme.txt', @@ -339,7 +336,7 @@ def test_0130_verify_handling_of_invalid_characters( self ): '''Load the above changeset in the change log and confirm that there is no server error displayed.''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) changeset_revision = self.get_repository_tip( repository ) repository_id = self.security.encode_id( repository.id ) changelog_tuples = self.get_repository_changelog_tuples( repository ) diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0010_repository_with_tool_dependencies.py --- a/test/tool_shed/functional/test_0010_repository_with_tool_dependencies.py +++ b/test/tool_shed/functional/test_0010_repository_with_tool_dependencies.py @@ -1,6 +1,4 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util - repository_name = 'freebayes_0010' repository_description = "Galaxy's freebayes tool" repository_long_description = "Long description of Galaxy's freebayes tool" @@ -15,7 +13,6 @@ 7. Check for the appropriate strings on the manage repository page. ''' - class TestFreebayesRepository( ShedTwillTestCase ): '''Testing freebayes with tool data table entries, .loc files, and tool dependencies.''' @@ -23,14 +20,14 @@ """Create necessary user accounts and login as an admin user.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_category( self ): """Create a category for this test suite""" @@ -45,7 +42,7 @@ ''' self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - category = test_db_util.get_category_by_name( 'Test 0010 Repository With Tool Dependencies' ) + category = self.test_db_util.get_category_by_name( 'Test 0010 Repository With Tool Dependencies' ) repository = self.get_or_create_repository( name=repository_name, description=repository_description, long_description=repository_long_description, @@ -72,7 +69,7 @@ We are at step 2 - Upload the tool_data_table_conf.xml.sample file. Uploading the tool_data_table_conf.xml.sample alone should not make the tool valid, but the error message should change. ''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.upload_file( repository, filename='freebayes/tool_data_table_conf.xml.sample', filepath=None, @@ -93,7 +90,7 @@ We are at step 3 - Upload the tool_data_table_conf.xml.sample file. Uploading the tool_data_table_conf.xml.sample alone should not make the tool valid, but the error message should change. ''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.upload_file( repository, filename='freebayes/sam_fa_indices.loc.sample', filepath=None, @@ -110,7 +107,7 @@ We are at step 4 - Upload a tool_dependencies.xml file that should not parse correctly. Upload a tool_dependencies.xml file that contains <> in the text of the readme tag. This should show an error message about malformed xml. ''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.upload_file( repository, filename=os.path.join( 'freebayes', 'malformed_tool_dependencies', 'tool_dependencies.xml' ), filepath=None, @@ -127,7 +124,7 @@ We are at step 5 - Upload a tool_dependencies.xml file that specifies a version that does not match the tool's requirements. This should result in a message about the tool dependency configuration not matching the tool's requirements. ''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.upload_file( repository, filename=os.path.join( 'freebayes', 'invalid_tool_dependencies', 'tool_dependencies.xml' ), filepath=None, @@ -144,7 +141,7 @@ We are at step 6 - Upload a valid tool_dependencies.xml file. At this stage, there should be no errors on the upload page, as every missing or invalid file has been corrected. ''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.upload_file( repository, filename=os.path.join( 'freebayes', 'tool_dependencies.xml' ), filepath=None, @@ -161,7 +158,7 @@ We are at step 7 - Check for the appropriate strings on the manage repository page. Verify that the manage repository page now displays the valid tool dependencies, and that there are no invalid tools shown on the manage page. ''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) strings_displayed = [ 'freebayes', '0.9.4_9696d0ce8a9', 'samtools', '0.1.18', 'Valid tools', 'package' ] strings_not_displayed = [ 'Invalid tools' ] self.display_manage_repository_page( repository, strings_displayed=strings_displayed, strings_not_displayed=strings_not_displayed ) diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0020_basic_repository_dependencies.py --- a/test/tool_shed/functional/test_0020_basic_repository_dependencies.py +++ b/test/tool_shed/functional/test_0020_basic_repository_dependencies.py @@ -1,6 +1,4 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util - datatypes_repository_name = 'emboss_datatypes_0020' datatypes_repository_description = "Galaxy applicable data formats used by Emboss tools." datatypes_repository_long_description = "Galaxy applicable data formats used by Emboss tools. This repository contains no tools." @@ -9,7 +7,6 @@ emboss_repository_description = 'Galaxy wrappers for Emboss version 5.0.0 tools for test 0020' emboss_repository_long_description = 'Galaxy wrappers for Emboss version 5.0.0 tools for test 0020' - class TestBasicRepositoryDependencies( ShedTwillTestCase ): '''Testing emboss 5 with repository dependencies.''' @@ -17,14 +14,14 @@ """Create necessary user accounts and login as an admin user.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % common.test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_category( self ): """Create a category for this test suite""" @@ -34,7 +31,7 @@ '''Create and populate the emboss_datatypes repository.''' self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - category = test_db_util.get_category_by_name( 'Test 0020 Basic Repository Dependencies' ) + category = self.test_db_util.get_category_by_name( 'Test 0020 Basic Repository Dependencies' ) repository = self.get_or_create_repository( name=datatypes_repository_name, description=datatypes_repository_description, long_description=datatypes_repository_long_description, @@ -53,12 +50,12 @@ def test_0015_verify_datatypes_in_datatypes_repository( self ): '''Verify that the emboss_datatypes repository contains datatype entries.''' - repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) self.display_manage_repository_page( repository, strings_displayed=[ 'Datatypes', 'equicktandem', 'hennig86', 'vectorstrip' ] ) def test_0020_create_emboss_5_repository_and_upload_files( self ): '''Create and populate the emboss_5_0020 repository.''' - category = test_db_util.get_category_by_name( 'Test 0020 Basic Repository Dependencies' ) + category = self.test_db_util.get_category_by_name( 'Test 0020 Basic Repository Dependencies' ) repository = self.get_or_create_repository( name=emboss_repository_name, description=emboss_repository_description, long_description=emboss_repository_long_description, @@ -77,16 +74,16 @@ def test_0025_generate_and_upload_repository_dependencies_xml( self ): '''Generate and upload the repository_dependencies.xml file''' - repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) - datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + datatypes_repository = self.test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) repository_dependencies_path = self.generate_temp_path( 'test_0020', additional_paths=[ 'emboss', '5' ] ) repository_tuple = ( self.url, datatypes_repository.name, datatypes_repository.user.username, self.get_repository_tip( datatypes_repository ) ) self.create_repository_dependency( repository=repository, repository_tuples=[ repository_tuple ], filepath=repository_dependencies_path ) def test_0030_verify_emboss_5_dependencies( self ): '''Verify that the emboss_5 repository now depends on the emboss_datatypes repository with correct name, owner, and changeset revision.''' - repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) - datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + datatypes_repository = self.test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) changeset_revision = self.get_repository_tip( datatypes_repository ) strings_displayed = [ 'Tool dependencies', 'emboss', @@ -100,7 +97,7 @@ def test_0040_verify_repository_metadata( self ): '''Verify that resetting the metadata does not change it.''' - emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) - datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) + emboss_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + datatypes_repository = self.test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) self.verify_unchanged_repository_metadata( emboss_repository ) self.verify_unchanged_repository_metadata( datatypes_repository ) diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0030_repository_dependency_revisions.py --- a/test/tool_shed/functional/test_0030_repository_dependency_revisions.py +++ b/test/tool_shed/functional/test_0030_repository_dependency_revisions.py @@ -1,6 +1,4 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util - datatypes_repository_name = 'emboss_datatypes_0030' datatypes_repository_description = "Galaxy applicable data formats used by Emboss tools." datatypes_repository_long_description = "Galaxy applicable data formats used by Emboss tools. This repository contains no tools." @@ -11,7 +9,6 @@ emboss_repository_description = 'Galaxy wrappers for Emboss version 5.0.0 tools for test 0030' emboss_repository_long_description = 'Galaxy wrappers for Emboss version 5.0.0 tools for test 0030' - class TestRepositoryDependencyRevisions( ShedTwillTestCase ): '''Test dependencies on different revisions of a repository.''' @@ -19,14 +16,14 @@ """Create necessary user accounts.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_category( self ): """Create a category for this test suite""" @@ -36,7 +33,7 @@ '''Create and populate the emboss_5_0030 repository.''' self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - category = test_db_util.get_category_by_name( 'Test 0030 Repository Dependency Revisions' ) + category = self.test_db_util.get_category_by_name( 'Test 0030 Repository Dependency Revisions' ) repository = self.get_or_create_repository( name=emboss_5_repository_name, description=emboss_repository_description, long_description=emboss_repository_long_description, @@ -56,7 +53,7 @@ '''Create and populate the emboss_6_0030 repository.''' self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - category = test_db_util.get_category_by_name( 'Test 0030 Repository Dependency Revisions' ) + category = self.test_db_util.get_category_by_name( 'Test 0030 Repository Dependency Revisions' ) repository = self.get_or_create_repository( name=emboss_6_repository_name, description=emboss_repository_description, long_description=emboss_repository_long_description, @@ -76,7 +73,7 @@ '''Create and populate the emboss_datatypes_0030 repository.''' self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - category = test_db_util.get_category_by_name( 'Test 0030 Repository Dependency Revisions' ) + category = self.test_db_util.get_category_by_name( 'Test 0030 Repository Dependency Revisions' ) repository = self.get_or_create_repository( name=datatypes_repository_name, description=emboss_repository_description, long_description=emboss_repository_long_description, @@ -96,7 +93,7 @@ '''Create and populate the emboss_0030 repository.''' self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - category = test_db_util.get_category_by_name( 'Test 0030 Repository Dependency Revisions' ) + category = self.test_db_util.get_category_by_name( 'Test 0030 Repository Dependency Revisions' ) repository = self.get_or_create_repository( name=emboss_repository_name, description=emboss_repository_description, long_description=emboss_repository_long_description, @@ -114,41 +111,41 @@ def test_0030_generate_repository_dependencies_for_emboss_5( self ): '''Generate a repository_dependencies.xml file specifying emboss_datatypes and upload it to the emboss_5 repository.''' - datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) - emboss_5_repository = test_db_util.get_repository_by_name_and_owner( emboss_5_repository_name, common.test_user_1_name ) + datatypes_repository = self.test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) + emboss_5_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_5_repository_name, common.test_user_1_name ) repository_dependencies_path = self.generate_temp_path( 'test_0030', additional_paths=[ 'emboss5' ] ) datatypes_tuple = ( self.url, datatypes_repository.name, datatypes_repository.user.username, self.get_repository_tip( datatypes_repository ) ) self.create_repository_dependency( repository=emboss_5_repository, repository_tuples=[ datatypes_tuple ], filepath=repository_dependencies_path ) def test_0035_generate_repository_dependencies_for_emboss_6( self ): '''Generate a repository_dependencies.xml file specifying emboss_datatypes and upload it to the emboss_6 repository.''' - emboss_6_repository = test_db_util.get_repository_by_name_and_owner( emboss_6_repository_name, common.test_user_1_name ) - datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) + emboss_6_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_6_repository_name, common.test_user_1_name ) + datatypes_repository = self.test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) repository_dependencies_path = self.generate_temp_path( 'test_0030', additional_paths=[ 'emboss6' ] ) datatypes_tuple = ( self.url, datatypes_repository.name, datatypes_repository.user.username, self.get_repository_tip( datatypes_repository ) ) self.create_repository_dependency( repository=emboss_6_repository, repository_tuples=[ datatypes_tuple ], filepath=repository_dependencies_path ) def test_0040_generate_repository_dependency_on_emboss_5( self ): '''Create and upload repository_dependencies.xml for the emboss_5_0030 repository.''' - emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) - emboss_5_repository = test_db_util.get_repository_by_name_and_owner( emboss_5_repository_name, common.test_user_1_name ) + emboss_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + emboss_5_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_5_repository_name, common.test_user_1_name ) repository_dependencies_path = self.generate_temp_path( 'test_0030', additional_paths=[ 'emboss', '5' ] ) emboss_tuple = ( self.url, emboss_5_repository.name, emboss_5_repository.user.username, self.get_repository_tip( emboss_5_repository ) ) self.create_repository_dependency( repository=emboss_repository, repository_tuples=[ emboss_tuple ], filepath=repository_dependencies_path ) def test_0045_generate_repository_dependency_on_emboss_6( self ): '''Create and upload repository_dependencies.xml for the emboss_6_0030 repository.''' - emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) - emboss_6_repository = test_db_util.get_repository_by_name_and_owner( emboss_6_repository_name, common.test_user_1_name ) + emboss_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + emboss_6_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_6_repository_name, common.test_user_1_name ) repository_dependencies_path = self.generate_temp_path( 'test_0030', additional_paths=[ 'emboss', '5' ] ) emboss_tuple = ( self.url, emboss_6_repository.name, emboss_6_repository.user.username, self.get_repository_tip( emboss_6_repository ) ) self.create_repository_dependency( repository=emboss_repository, repository_tuples=[ emboss_tuple ], filepath=repository_dependencies_path ) def test_0050_verify_repository_dependency_revisions( self ): '''Verify that different metadata revisions of the emboss repository have different repository dependencies.''' - repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) repository_metadata = [ ( metadata.metadata, metadata.changeset_revision ) for metadata in self.get_repository_metadata( repository ) ] - datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) + datatypes_repository = self.test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) datatypes_tip = self.get_repository_tip( datatypes_repository ) strings_displayed = [] # Iterate through all metadata revisions and check for repository dependencies. @@ -161,9 +158,9 @@ strings_displayed=strings_displayed ) def test_0055_verify_repository_metadata( self ): '''Verify that resetting the metadata does not change it.''' - emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) - emboss_5_repository = test_db_util.get_repository_by_name_and_owner( emboss_5_repository_name, common.test_user_1_name ) - emboss_6_repository = test_db_util.get_repository_by_name_and_owner( emboss_6_repository_name, common.test_user_1_name ) - datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) + emboss_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + emboss_5_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_5_repository_name, common.test_user_1_name ) + emboss_6_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_6_repository_name, common.test_user_1_name ) + datatypes_repository = self.test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) for repository in [ emboss_repository, emboss_5_repository, emboss_6_repository, datatypes_repository ]: self.verify_unchanged_repository_metadata( repository ) diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0040_repository_circular_dependencies.py --- a/test/tool_shed/functional/test_0040_repository_circular_dependencies.py +++ b/test/tool_shed/functional/test_0040_repository_circular_dependencies.py @@ -1,6 +1,4 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util - freebayes_repository_name = 'freebayes_0040' freebayes_repository_description = "Galaxy's freebayes tool for test 0040" freebayes_repository_long_description = "Long description of Galaxy's freebayes tool for test 0040" @@ -9,7 +7,6 @@ filtering_repository_description = "Galaxy's filtering tool for test 0040" filtering_repository_long_description = "Long description of Galaxy's filtering tool for test 0040" - class TestRepositoryCircularDependencies( ShedTwillTestCase ): '''Verify that the code correctly displays repositories with circular repository dependencies.''' @@ -17,14 +14,14 @@ """Create necessary user accounts.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_category( self ): """Create a category for this test suite""" @@ -76,8 +73,8 @@ # Filtering revision 0 -> freebayes revision 0. # Freebayes revision 0 -> filtering revision 1. # Filtering will have two revisions, one with just the filtering tool, and one with the filtering tool and a dependency on freebayes. - repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) - filtering_repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) + filtering_repository = self.test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) repository_dependencies_path = self.generate_temp_path( 'test_0040', additional_paths=[ 'filtering' ] ) repository_tuple = ( self.url, repository.name, repository.user.username, self.get_repository_tip( repository ) ) self.create_repository_dependency( repository=filtering_repository, repository_tuples=[ repository_tuple ], filepath=repository_dependencies_path ) @@ -88,16 +85,16 @@ # Filtering revision 0 -> freebayes revision 0. # Freebayes revision 0 -> filtering revision 1. # Filtering will have two revisions, one with just the filtering tool, and one with the filtering tool and a dependency on freebayes. - repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) - freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) + freebayes_repository = self.test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) repository_dependencies_path = self.generate_temp_path( 'test_0040', additional_paths=[ 'freebayes' ] ) repository_tuple = ( self.url, repository.name, repository.user.username, self.get_repository_tip( repository ) ) self.create_repository_dependency( repository=freebayes_repository, repository_tuples=[ repository_tuple ], filepath=repository_dependencies_path ) def test_0030_verify_repository_dependencies( self ): '''Verify that each repository can depend on the other without causing an infinite loop.''' - filtering_repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) - freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) + filtering_repository = self.test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) + freebayes_repository = self.test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) # The dependency structure should look like: # Filtering revision 0 -> freebayes revision 0. # Freebayes revision 0 -> filtering revision 1. @@ -108,14 +105,14 @@ def test_0035_verify_repository_metadata( self ): '''Verify that resetting the metadata does not change it.''' - freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) - filtering_repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) + freebayes_repository = self.test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) + filtering_repository = self.test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) for repository in [ freebayes_repository, filtering_repository ]: self.verify_unchanged_repository_metadata( repository ) def test_0040_verify_tool_dependencies( self ): '''Verify that freebayes displays tool dependencies.''' - repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) self.display_manage_repository_page( repository, strings_displayed=[ 'freebayes', '0.9.4_9696d0ce8a9', 'samtools', '0.1.18', 'Valid tools', 'package' ], strings_not_displayed=[ 'Invalid tools' ] ) diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0050_circular_dependencies_4_levels.py --- a/test/tool_shed/functional/test_0050_circular_dependencies_4_levels.py +++ b/test/tool_shed/functional/test_0050_circular_dependencies_4_levels.py @@ -1,6 +1,4 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util - emboss_datatypes_repository_name = 'emboss_datatypes_0050' emboss_datatypes_repository_description = "Datatypes for emboss" emboss_datatypes_repository_long_description = "Long description of Emboss' datatypes" @@ -32,7 +30,6 @@ category_name = 'Test 0050 Circular Dependencies 5 Levels' category_description = 'Test circular dependency features' - class TestRepositoryCircularDependenciesToNLevels( ShedTwillTestCase ): '''Verify that the code correctly handles circular dependencies down to n levels.''' @@ -40,14 +37,14 @@ """Create necessary user accounts.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_convert_repository( self ): '''Create and populate convert_chars_0050.''' @@ -187,13 +184,13 @@ strings_not_displayed=[] ) def test_0040_create_and_upload_dependency_definitions( self ): - column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) - convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) - datatypes_repository = test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) - emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) - filtering_repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) - freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) - bismark_repository = test_db_util.get_repository_by_name_and_owner( bismark_repository_name, common.test_user_1_name ) + column_repository = self.test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) + convert_repository = self.test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) + datatypes_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) + emboss_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + filtering_repository = self.test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) + freebayes_repository = self.test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) + bismark_repository = self.test_db_util.get_repository_by_name_and_owner( bismark_repository_name, common.test_user_1_name ) dependency_xml_path = self.generate_temp_path( 'test_0050', additional_paths=[ 'freebayes' ] ) # convert_chars depends on column_maker # column_maker depends on convert_chars @@ -235,13 +232,13 @@ id: 7 key: http://toolshed.local:10001__ESEP__emboss_5__ESEP__test__ESEP__8de5fe0d7b04 ['http://toolshed.local:10001', 'emboss_datatypes', 'test', 'dbd4f68bf507'] ''' - emboss_datatypes_repository = test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) - emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) - filtering_repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) - freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) - column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) - convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) - bismark_repository = test_db_util.get_repository_by_name_and_owner( bismark_repository_name, common.test_user_1_name ) + emboss_datatypes_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) + emboss_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + filtering_repository = self.test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) + freebayes_repository = self.test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) + column_repository = self.test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) + convert_repository = self.test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) + bismark_repository = self.test_db_util.get_repository_by_name_and_owner( bismark_repository_name, common.test_user_1_name ) self.check_repository_dependency( convert_repository, column_repository ) self.check_repository_dependency( column_repository, convert_repository ) self.check_repository_dependency( emboss_datatypes_repository, bismark_repository ) @@ -256,17 +253,17 @@ def test_0050_verify_tool_dependencies( self ): '''Check that freebayes and emboss display tool dependencies.''' - freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) - emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + freebayes_repository = self.test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) + emboss_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) self.display_manage_repository_page( freebayes_repository, strings_displayed=[ 'freebayes', '0.9.4_9696d0ce8a9', 'samtools', '0.1.18', 'Tool dependencies', 'package' ] ) self.display_manage_repository_page( emboss_repository, strings_displayed=[ 'Tool dependencies', 'emboss', '5.0.0', 'package' ] ) def test_0055_verify_repository_metadata( self ): '''Verify that resetting the metadata does not change it.''' - emboss_datatypes_repository = test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) - emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) - filtering_repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) - freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) + emboss_datatypes_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) + emboss_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + filtering_repository = self.test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) + freebayes_repository = self.test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) for repository in [ emboss_datatypes_repository, emboss_repository, freebayes_repository, filtering_repository ]: self.verify_unchanged_repository_metadata( repository ) diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0060_workflows.py --- a/test/tool_shed/functional/test_0060_workflows.py +++ b/test/tool_shed/functional/test_0060_workflows.py @@ -1,6 +1,4 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util - repository_name = 'filtering_0060' repository_description="Galaxy's filtering tool for test 0060" repository_long_description="Long description of Galaxy's filtering tool for test 0060" @@ -15,20 +13,21 @@ workflow_repository_description="Workflow referencing the filtering tool for test 0060" workflow_repository_long_description="Long description of the workflow for test 0060" + class TestToolShedWorkflowFeatures( ShedTwillTestCase ): '''Test valid and invalid workflows.''' def test_0000_initiate_users( self ): """Create necessary user accounts and login as an admin user.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_categories( self ): """Create categories for this test suite""" self.create_category( name='Test 0060 Workflow Features', description='Test 0060 - Workflow Features' ) @@ -45,7 +44,7 @@ strings_displayed=[] ) def test_0015_upload_workflow( self ): '''Upload a workflow with a missing tool, and verify that the tool specified is marked as missing.''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) workflow = file( self.get_filename( 'filtering_workflow/Workflow_for_0060_filter_workflow_repository.ga' ), 'r' ).read() workflow = workflow.replace( '__TEST_TOOL_SHED_URL__', self.url.replace( 'http://', '' ) ) workflow_filepath = self.generate_temp_path( 'test_0060', additional_paths=[ 'filtering_workflow' ] ) @@ -64,7 +63,7 @@ self.load_workflow_image_in_tool_shed( repository, workflow_name, strings_displayed=[ '#EBBCB2' ] ) def test_0020_upload_tool( self ): '''Upload the missing tool for the workflow in the previous step, and verify that the error is no longer present.''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.upload_file( repository, filename='filtering/filtering_2.2.0.tar', filepath=None, @@ -94,7 +93,7 @@ if not os.path.exists( workflow_filepath ): os.makedirs( workflow_filepath ) file( os.path.join( workflow_filepath, workflow_filename ), 'w+' ).write( workflow ) - repository = test_db_util.get_repository_by_name_and_owner( workflow_repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( workflow_repository_name, common.test_user_1_name ) self.upload_file( repository, filename=workflow_filename, filepath=workflow_filepath, @@ -107,11 +106,11 @@ self.load_workflow_image_in_tool_shed( repository, workflow_name, strings_displayed=[ '#EBBCB2' ] ) def test_0030_check_workflow_repository( self ): """Check for strings on the manage page for the filtering_workflow_0060 repository.""" - repository = test_db_util.get_repository_by_name_and_owner( workflow_repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( workflow_repository_name, common.test_user_1_name ) strings_displayed = [ 'Workflows', 'New workflow for 0060_filter', '0.1' ] strings_not_displayed = [ 'Valid tools', 'Invalid tools' ] self.display_manage_repository_page( repository, strings_displayed=strings_displayed, strings_not_displayed=strings_not_displayed ) def test_0035_verify_repository_metadata( self ): '''Verify that resetting the metadata does not change it.''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.verify_unchanged_repository_metadata( repository ) diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0070_invalid_tool.py --- a/test/tool_shed/functional/test_0070_invalid_tool.py +++ b/test/tool_shed/functional/test_0070_invalid_tool.py @@ -1,13 +1,10 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util - repository_name = 'bismark_0070' repository_description = "Galaxy's bismark wrapper" repository_long_description = "Long description of Galaxy's bismark wrapper" category_name = 'Test 0070 Invalid Tool Revisions' category_description = 'Tests for a repository with invalid tool revisions.' - class TestBismarkRepository( ShedTwillTestCase ): '''Testing bismark with valid and invalid tool entries.''' @@ -15,14 +12,14 @@ """Create necessary user accounts and login as an admin user.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_category_and_repository( self ): """Create a category for this test suite, then create and populate a bismark repository. It should contain at least one each valid and invalid tool.""" @@ -56,7 +53,7 @@ strings_displayed=[], strings_not_displayed=[] ) valid_revision = self.get_repository_tip( repository ) - test_db_util.refresh( repository ) + self.test_db_util.refresh( repository ) tool_guid = '%s/repos/user1/bismark_0070/bismark_methylation_extractor/0.7.7.3' % self.url.replace( 'http://', '' ).rstrip( '/' ) tool_metadata_strings_displayed = [ tool_guid, '0.7.7.3', # The tool version. diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0080_advanced_circular_dependencies.py --- a/test/tool_shed/functional/test_0080_advanced_circular_dependencies.py +++ b/test/tool_shed/functional/test_0080_advanced_circular_dependencies.py @@ -1,6 +1,4 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util - column_repository_name = 'column_maker_0080' column_repository_description = "Add column" column_repository_long_description = "Compute an expression on every row" @@ -12,7 +10,6 @@ category_name = 'Test 0080 Advanced Circular Dependencies' category_description = 'Test circular dependency features' - class TestRepositoryCircularDependencies( ShedTwillTestCase ): '''Verify that the code correctly handles circular dependencies.''' @@ -20,14 +17,14 @@ """Create necessary user accounts.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_column_repository( self ): """Create and populate the column_maker repository.""" @@ -75,30 +72,30 @@ def test_0020_create_repository_dependencies( self ): '''Upload a repository_dependencies.xml file that specifies the current revision of convert_chars_0080 to the column_maker_0080 repository.''' - convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) - column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) + convert_repository = self.test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) + column_repository = self.test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) repository_dependencies_path = self.generate_temp_path( 'test_0080', additional_paths=[ 'convert' ] ) repository_tuple = ( self.url, convert_repository.name, convert_repository.user.username, self.get_repository_tip( convert_repository ) ) self.create_repository_dependency( repository=column_repository, repository_tuples=[ repository_tuple ], filepath=repository_dependencies_path ) def test_0025_create_dependency_on_filtering( self ): '''Upload a repository_dependencies.xml file that specifies the current revision of filtering to the freebayes_0040 repository.''' - convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) - column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) + convert_repository = self.test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) + column_repository = self.test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) repository_dependencies_path = self.generate_temp_path( 'test_0080', additional_paths=[ 'convert' ] ) repository_tuple = ( self.url, column_repository.name, column_repository.user.username, self.get_repository_tip( column_repository ) ) self.create_repository_dependency( repository=convert_repository, repository_tuples=[ repository_tuple ], filepath=repository_dependencies_path ) def test_0030_verify_repository_dependencies( self ): '''Verify that each repository can depend on the other without causing an infinite loop.''' - convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) - column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) + convert_repository = self.test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) + column_repository = self.test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) self.check_repository_dependency( convert_repository, column_repository, self.get_repository_tip( column_repository ) ) self.check_repository_dependency( column_repository, convert_repository, self.get_repository_tip( convert_repository ) ) def test_0035_verify_repository_metadata( self ): '''Verify that resetting the metadata does not change it.''' - column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) - convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) + column_repository = self.test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) + convert_repository = self.test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) for repository in [ column_repository, convert_repository ]: self.verify_unchanged_repository_metadata( repository ) diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0090_tool_search.py --- a/test/tool_shed/functional/test_0090_tool_search.py +++ b/test/tool_shed/functional/test_0090_tool_search.py @@ -1,6 +1,4 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util - emboss_datatypes_repository_name = 'emboss_datatypes_0090' emboss_datatypes_repository_description = "Datatypes for emboss" emboss_datatypes_repository_long_description = "Long description of Emboss' datatypes" @@ -28,7 +26,6 @@ category_name = 'Test 0090 Tool Search And Installation' category_description = 'Test 0090 Tool Search And Installation' - class TestRepositoryCircularDependenciesAgain( ShedTwillTestCase ): '''Test more features related to repository dependencies.''' @@ -36,14 +33,14 @@ """Create necessary user accounts.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_bwa_base_repository( self ): '''Create and populate bwa_base_0090.''' @@ -167,12 +164,12 @@ def test_0035_create_and_upload_dependency_definitions( self ): '''Create and upload repository dependency definitions.''' - bwa_color_repository = test_db_util.get_repository_by_name_and_owner( bwa_color_repository_name, common.test_user_1_name ) - bwa_base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) - datatypes_repository = test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) - emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) - filtering_repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) - freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) + bwa_color_repository = self.test_db_util.get_repository_by_name_and_owner( bwa_color_repository_name, common.test_user_1_name ) + bwa_base_repository = self.test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) + datatypes_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) + emboss_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + filtering_repository = self.test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) + freebayes_repository = self.test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) dependency_xml_path = self.generate_temp_path( 'test_0090', additional_paths=[ 'freebayes' ] ) freebayes_tuple = ( self.url, freebayes_repository.name, freebayes_repository.user.username, self.get_repository_tip( freebayes_repository ) ) emboss_tuple = ( self.url, emboss_repository.name, emboss_repository.user.username, self.get_repository_tip( emboss_repository ) ) @@ -185,12 +182,12 @@ def test_0040_verify_repository_dependencies( self ): '''Verify the generated dependency structure.''' - bwa_color_repository = test_db_util.get_repository_by_name_and_owner( bwa_color_repository_name, common.test_user_1_name ) - bwa_base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) - emboss_datatypes_repository = test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) - emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) - filtering_repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) - freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) + bwa_color_repository = self.test_db_util.get_repository_by_name_and_owner( bwa_color_repository_name, common.test_user_1_name ) + bwa_base_repository = self.test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) + emboss_datatypes_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) + emboss_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + filtering_repository = self.test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) + freebayes_repository = self.test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) self.check_repository_dependency( emboss_repository, emboss_datatypes_repository ) self.check_repository_dependency( filtering_repository, freebayes_repository ) self.check_repository_dependency( bwa_base_repository, emboss_repository ) diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0100_complex_repository_dependencies.py --- a/test/tool_shed/functional/test_0100_complex_repository_dependencies.py +++ b/test/tool_shed/functional/test_0100_complex_repository_dependencies.py @@ -1,5 +1,4 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util import logging log = logging.getLogger( __name__ ) @@ -15,7 +14,6 @@ category_name = 'Test 0100 Complex Repository Dependencies' category_description = 'Test 0100 Complex Repository Dependencies' - class TestComplexRepositoryDependencies( ShedTwillTestCase ): '''Test features related to complex repository dependencies.''' @@ -23,14 +21,14 @@ """Create necessary user accounts.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_bwa_package_repository( self ): '''Create and populate package_bwa_0_5_9_0100.''' @@ -84,9 +82,9 @@ dependency_path = self.generate_temp_path( 'test_0100', additional_paths=[ 'complex', 'invalid' ] ) xml_filename = self.get_filename( 'tool_dependencies.xml', filepath=dependency_path ) # The repository named bwa_base_repository_0100 is the dependent repository. - base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) + base_repository = self.test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) # The repository named package_bwa_0_5_9_0100 is the required repository. - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) + tool_repository = self.test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) url = 'http://http://this is not an url!' name = 'package_bwa_0_5_9_0100' owner = 'user1' @@ -107,9 +105,9 @@ dependency_path = self.generate_temp_path( 'test_0100', additional_paths=[ 'complex', 'invalid' ] ) xml_filename = self.get_filename( 'tool_dependencies.xml', filepath=dependency_path ) # The base_repository named bwa_base_repository_0100 is the dependent repository. - base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) + base_repository = self.test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) # The repository named package_bwa_0_5_9_0100 is the required repository. - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) + tool_repository = self.test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) url = self.url name = 'invalid_repository!?' owner = 'user1' @@ -130,9 +128,9 @@ dependency_path = self.generate_temp_path( 'test_0100', additional_paths=[ 'complex', 'invalid' ] ) xml_filename = self.get_filename( 'tool_dependencies.xml', filepath=dependency_path ) # The base_repository named bwa_base_repository_0100 is the dependent repository. - base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) + base_repository = self.test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) # The repository named package_bwa_0_5_9_0100 is the required repository. - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) + tool_repository = self.test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) url = self.url name = 'package_bwa_0_5_9_0100' owner = 'invalid_owner!?' @@ -152,9 +150,9 @@ dependency_path = self.generate_temp_path( 'test_0100', additional_paths=[ 'complex', 'invalid' ] ) xml_filename = self.get_filename( 'tool_dependencies.xml', filepath=dependency_path ) # The base_repository named bwa_base_repository_0100 is the dependent repository. - base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) + base_repository = self.test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) # The repository named package_bwa_0_5_9_0100 is the required repository. - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) + tool_repository = self.test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) url = self.url name = 'package_bwa_0_5_9_0100' owner = 'user1' @@ -172,9 +170,9 @@ def test_0035_generate_complex_repository_dependency( self ): '''Generate and upload a valid tool_dependencies.xml file that specifies package_bwa_0_5_9_0100.''' # The base_repository named bwa_base_repository_0100 is the dependent repository. - base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) + base_repository = self.test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) # The repository named package_bwa_0_5_9_0100 is the required repository. - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) + tool_repository = self.test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) dependency_path = self.generate_temp_path( 'test_0100', additional_paths=[ 'complex' ] ) url = self.url name = 'package_bwa_0_5_9_0100' @@ -193,9 +191,9 @@ def test_0040_generate_tool_dependency( self ): '''Generate and upload a new tool_dependencies.xml file that specifies an arbitrary file on the filesystem, and verify that bwa_base depends on the new changeset revision.''' # The base_repository named bwa_base_repository_0100 is the dependent repository. - base_repository = test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) + base_repository = self.test_db_util.get_repository_by_name_and_owner( bwa_base_repository_name, common.test_user_1_name ) # The repository named package_bwa_0_5_9_0100 is the required repository. - tool_repository = test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) + tool_repository = self.test_db_util.get_repository_by_name_and_owner( bwa_package_repository_name, common.test_user_1_name ) previous_changeset = self.get_repository_tip( tool_repository ) old_tool_dependency = self.get_filename( os.path.join( 'bwa', 'complex', 'readme', 'tool_dependencies.xml' ) ) new_tool_dependency_path = self.generate_temp_path( 'test_1100', additional_paths=[ 'tool_dependency' ] ) diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0110_invalid_simple_repository_dependencies.py --- a/test/tool_shed/functional/test_0110_invalid_simple_repository_dependencies.py +++ b/test/tool_shed/functional/test_0110_invalid_simple_repository_dependencies.py @@ -1,6 +1,4 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util - datatypes_repository_name = 'emboss_datatypes_0110' datatypes_repository_description = "Galaxy applicable data formats used by Emboss tools." datatypes_repository_long_description = "Galaxy applicable data formats used by Emboss tools. This repository contains no tools." @@ -12,7 +10,6 @@ category_name = 'Test 0110 Invalid Repository Dependencies' category_desc = 'Test 0110 Invalid Repository Dependencies' - class TestBasicRepositoryDependencies( ShedTwillTestCase ): '''Testing emboss 5 with repository dependencies.''' @@ -20,14 +17,14 @@ """Create necessary user accounts and login as an admin user.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % common.test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_category( self ): """Create a category for this test suite""" @@ -37,7 +34,7 @@ '''Create and populate the emboss_datatypes repository.''' self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - category = test_db_util.get_category_by_name( category_name ) + category = self.test_db_util.get_category_by_name( category_name ) repository = self.get_or_create_repository( name=datatypes_repository_name, description=datatypes_repository_description, long_description=datatypes_repository_long_description, @@ -56,12 +53,12 @@ def test_0015_verify_datatypes_in_datatypes_repository( self ): '''Verify that the emboss_datatypes repository contains datatype entries.''' - repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) self.display_manage_repository_page( repository, strings_displayed=[ 'Datatypes', 'equicktandem', 'hennig86', 'vectorstrip' ] ) def test_0020_create_emboss_5_repository_and_upload_files( self ): '''Create and populate the emboss_5_0110 repository.''' - category = test_db_util.get_category_by_name( category_name ) + category = self.test_db_util.get_category_by_name( category_name ) repository = self.get_or_create_repository( name=emboss_repository_name, description=emboss_repository_description, long_description=emboss_repository_long_description, @@ -82,8 +79,8 @@ '''Generate a repository dependency for emboss 5 with an invalid URL.''' dependency_path = self.generate_temp_path( 'test_0110', additional_paths=[ 'simple' ] ) xml_filename = self.get_filename( 'repository_dependencies.xml', filepath=dependency_path ) - datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) - emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + datatypes_repository = self.test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) + emboss_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) url = 'http://http://this is not an url!' name = datatypes_repository.name owner = datatypes_repository.user.username @@ -100,8 +97,8 @@ '''Generate a repository dependency for emboss 5 with an invalid name.''' dependency_path = self.generate_temp_path( 'test_0110', additional_paths=[ 'simple' ] ) xml_filename = self.get_filename( 'repository_dependencies.xml', filepath=dependency_path ) - repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) - emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) + emboss_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) url = self.url name = '!?invalid?!' owner = repository.user.username @@ -118,8 +115,8 @@ '''Generate a repository dependency for emboss 5 with an invalid owner.''' dependency_path = self.generate_temp_path( 'test_0110', additional_paths=[ 'simple' ] ) xml_filename = self.get_filename( 'repository_dependencies.xml', filepath=dependency_path ) - repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) - emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) + emboss_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) url = self.url name = repository.name owner = '!?invalid?!' @@ -136,8 +133,8 @@ '''Generate a repository dependency for emboss 5 with an invalid changeset revision.''' dependency_path = self.generate_temp_path( 'test_0110', additional_paths=[ 'simple', 'invalid' ] ) xml_filename = self.get_filename( 'repository_dependencies.xml', filepath=dependency_path ) - repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) - emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) + emboss_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) url = self.url name = repository.name owner = repository.user.username diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0120_simple_repository_dependency_multiple_owners.py --- a/test/tool_shed/functional/test_0120_simple_repository_dependency_multiple_owners.py +++ b/test/tool_shed/functional/test_0120_simple_repository_dependency_multiple_owners.py @@ -1,6 +1,4 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util - datatypes_repository_name = 'blast_datatypes_0120' datatypes_repository_description = 'Galaxy applicable datatypes for BLAST' datatypes_repository_long_description = 'Galaxy datatypes for the BLAST top hit descriptons tool' @@ -23,7 +21,6 @@ base_datatypes_count = 0 repository_datatypes_count = 0 - class TestRepositoryMultipleOwners( ShedTwillTestCase ): def test_0000_initiate_users( self ): @@ -34,19 +31,19 @@ """ self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % common.test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.test_user_2_email, username=common.test_user_2_name ) - test_user_2 = test_db_util.get_user( common.test_user_1_email ) + test_user_2 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_2 is not None, 'Problem retrieving user with email %s from the database' % common.test_user_2_email - test_user_2_private_role = test_db_util.get_private_role( test_user_2 ) + test_user_2_private_role = self.test_db_util.get_private_role( test_user_2 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_datatypes_repository( self ): """Create and populate the blast_datatypes_0120 repository""" @@ -83,7 +80,7 @@ the datatypes that are defined in datatypes_conf.xml. ''' global repository_datatypes_count - repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_2_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_2_name ) strings_displayed = [ 'BlastXml', 'BlastNucDb', 'BlastProtDb', 'application/xml', 'text/html', 'blastxml', 'blastdbn', 'blastdbp'] self.display_manage_repository_page( repository, strings_displayed=strings_displayed ) repository_datatypes_count = int( self.get_repository_datatypes_count( repository ) ) @@ -121,7 +118,7 @@ We are at step 2a. Check for appropriate strings, such as tool name, description, and version. ''' - repository = test_db_util.get_repository_by_name_and_owner( tool_repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( tool_repository_name, common.test_user_1_name ) strings_displayed = [ 'blastxml_to_top_descr_0120', 'BLAST top hit descriptions', 'Make a table from BLAST XML' ] strings_displayed.extend( [ '0.0.1', 'Valid tools'] ) self.display_manage_repository_page( repository, strings_displayed=strings_displayed ) @@ -132,8 +129,8 @@ We are at step 3. Create a simple repository dependency for blastxml_to_top_descr_0120 that defines a dependency on blast_datatypes_0120. ''' - datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_2_name ) - tool_repository = test_db_util.get_repository_by_name_and_owner( tool_repository_name, common.test_user_1_name ) + datatypes_repository = self.test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_2_name ) + tool_repository = self.test_db_util.get_repository_by_name_and_owner( tool_repository_name, common.test_user_1_name ) dependency_xml_path = self.generate_temp_path( 'test_0120', additional_paths=[ 'dependencies' ] ) datatypes_tuple = ( self.url, datatypes_repository.name, datatypes_repository.user.username, self.get_repository_tip( datatypes_repository ) ) self.create_repository_dependency( repository=tool_repository, repository_tuples=[ datatypes_tuple ], filepath=dependency_xml_path ) @@ -144,6 +141,6 @@ We are at step 3a. Check the newly created repository dependency to ensure that it was defined and displays correctly. ''' - datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_2_name ) - tool_repository = test_db_util.get_repository_by_name_and_owner( tool_repository_name, common.test_user_1_name ) + datatypes_repository = self.test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_2_name ) + tool_repository = self.test_db_util.get_repository_by_name_and_owner( tool_repository_name, common.test_user_1_name ) self.check_repository_dependency( tool_repository, datatypes_repository ) diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0130_datatype_converters.py --- a/test/tool_shed/functional/test_0130_datatype_converters.py +++ b/test/tool_shed/functional/test_0130_datatype_converters.py @@ -1,6 +1,4 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util - repository_name = 'bed_to_gff_0130' repository_description = "Converter: BED to GFF" repository_long_description = "Convert bed to gff" @@ -14,7 +12,6 @@ 3) Visit the view tool metadata page and make sure that "Display in tool panel" is False ''' - class TestDatatypeConverters( ShedTwillTestCase ): '''Test features related to datatype converters.''' @@ -22,14 +19,14 @@ """Create necessary user accounts.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_bed_to_gff_repository( self ): '''Create and populate bed_to_gff_0130.''' @@ -64,7 +61,7 @@ We are at step 2 - Visit the manage repository page and make sure there is the appropriate valid tool and datatype. There should be a 'Convert BED to GFF' tool and a 'galaxy.datatypes.interval:Bed' datatype with extension 'bed' ''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) strings_displayed = [ 'Convert BED to GFF', 'galaxy.datatypes.interval:Bed', 'bed', 'Valid tools', 'Datatypes' ] strings_not_displayed = [ 'Invalid tools' ] self.display_manage_repository_page( repository, strings_displayed=strings_displayed, strings_not_displayed=strings_not_displayed ) @@ -75,7 +72,7 @@ We are at step 3 Datatype converters that are associated with a datatype should have display in tool panel = False in the tool metadata. ''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) metadata = self.get_repository_metadata_by_changeset_revision( repository, self.get_repository_tip( repository ) ) tool_metadata_strings_displayed = '<label>Display in tool panel:</label>\n False' self.check_repository_tools_for_changeset_revision( repository, diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0140_tool_help_images.py --- a/test/tool_shed/functional/test_0140_tool_help_images.py +++ b/test/tool_shed/functional/test_0140_tool_help_images.py @@ -1,5 +1,6 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util + + import logging log = logging.getLogger(__name__) @@ -20,7 +21,6 @@ ''' - class TestToolHelpImages( ShedTwillTestCase ): '''Test features related to tool help images.''' @@ -28,14 +28,14 @@ """Create necessary user accounts.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_htseq_count_repository( self ): '''Create and populate htseq_count_0140.''' @@ -73,7 +73,7 @@ src="/repository/static/images/<id>/count_modes.png" ''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) # Get the repository tip. changeset_revision = self.get_repository_tip( repository ) self.display_manage_repository_page( repository ) diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0150_prior_installation_required.py --- a/test/tool_shed/functional/test_0150_prior_installation_required.py +++ b/test/tool_shed/functional/test_0150_prior_installation_required.py @@ -1,6 +1,4 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util - column_repository_name = 'column_maker_0150' column_repository_description = "Add column" column_repository_long_description = "Compute an expression on every row" @@ -26,7 +24,6 @@ Verify that convert_chars was installed first, contrary to the ordering that would be present without prior_installation_required. ''' - class TestSimplePriorInstallation( ShedTwillTestCase ): '''Test features related to datatype converters.''' @@ -34,14 +31,14 @@ """Create necessary user accounts.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_convert_repository( self ): '''Create and populate convert_chars_0150.''' @@ -89,8 +86,8 @@ Column maker repository dependency: <repository toolshed="self.url" name="convert_chars" owner="test" changeset_revision="<tip>" prior_installation_required="True" /> ''' - column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) - convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) + column_repository = self.test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) + convert_repository = self.test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) dependency_xml_path = self.generate_temp_path( 'test_0150', additional_paths=[ 'column' ] ) convert_tuple = ( self.url, convert_repository.name, convert_repository.user.username, self.get_repository_tip( convert_repository ) ) self.create_repository_dependency( repository=column_repository, @@ -100,8 +97,8 @@ def test_0020_verify_repository_dependency( self ): '''Verify that the previously generated repositiory dependency displays correctly.''' - column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) - convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) + column_repository = self.test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) + convert_repository = self.test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) self.check_repository_dependency( repository=column_repository, depends_on_repository=convert_repository, depends_on_changeset_revision=None, diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0160_circular_prior_installation_required.py --- a/test/tool_shed/functional/test_0160_circular_prior_installation_required.py +++ b/test/tool_shed/functional/test_0160_circular_prior_installation_required.py @@ -1,6 +1,4 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util - filter_repository_name = 'filtering_0160' filter_repository_description = "Galaxy's filtering tool for test 0160" filter_repository_long_description = "Long description of Galaxy's filtering tool for test 0160" @@ -25,7 +23,6 @@ Verify display. ''' - class TestSimplePriorInstallation( ShedTwillTestCase ): '''Test features related to datatype converters.''' @@ -33,14 +30,14 @@ """Create necessary user accounts.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_convert_repository( self ): '''Create and populate convert_chars_0160.''' @@ -106,9 +103,9 @@ ''' Each of the three repositories should depend on the other two, to make this as circular as possible. ''' - filter_repository = test_db_util.get_repository_by_name_and_owner( filter_repository_name, common.test_user_1_name ) - column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) - convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) + filter_repository = self.test_db_util.get_repository_by_name_and_owner( filter_repository_name, common.test_user_1_name ) + column_repository = self.test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) + convert_repository = self.test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) dependency_xml_path = self.generate_temp_path( 'test_0160', additional_paths=[ 'column' ] ) filter_revision = self.get_repository_tip( filter_repository ) column_revision = self.get_repository_tip( column_repository ) @@ -131,9 +128,9 @@ def test_0025_verify_repository_dependency( self ): '''Verify that the previously generated repositiory dependency displays correctly.''' - filter_repository = test_db_util.get_repository_by_name_and_owner( filter_repository_name, common.test_user_1_name ) - column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) - convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) + filter_repository = self.test_db_util.get_repository_by_name_and_owner( filter_repository_name, common.test_user_1_name ) + column_repository = self.test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) + convert_repository = self.test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) self.check_repository_dependency( repository=column_repository, depends_on_repository=convert_repository, depends_on_changeset_revision=None, diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0170_complex_prior_installation_required.py --- a/test/tool_shed/functional/test_0170_complex_prior_installation_required.py +++ b/test/tool_shed/functional/test_0170_complex_prior_installation_required.py @@ -1,6 +1,4 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util - matplotlib_repository_name = 'package_matplotlib_1_2_0170' matplotlib_repository_description = "Contains a tool dependency definition that downloads and compiles version 1.2.x of the the python matplotlib package." matplotlib_repository_long_description = "This repository is intended to be defined as a complex repository dependency within a separate repository." @@ -18,7 +16,6 @@ 3. Verify that package_matplotlib_1_2_0170 now depends on package_numpy_1_7_0170, and that the inherited tool dependency displays correctly. ''' - class TestComplexPriorInstallation( ShedTwillTestCase ): '''Test features related to datatype converters.''' @@ -26,14 +23,14 @@ """Create necessary user accounts.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_matplotlib_repository( self ): '''Create and populate the package_matplotlib_1_2_0170 repository.''' @@ -91,8 +88,8 @@ set to True. When matplotlib is selected for installation, the result should be that numpy is compiled and installed first. ''' - numpy_repository = test_db_util.get_repository_by_name_and_owner( numpy_repository_name, common.test_user_1_name ) - matplotlib_repository = test_db_util.get_repository_by_name_and_owner( matplotlib_repository_name, common.test_user_1_name ) + numpy_repository = self.test_db_util.get_repository_by_name_and_owner( numpy_repository_name, common.test_user_1_name ) + matplotlib_repository = self.test_db_util.get_repository_by_name_and_owner( matplotlib_repository_name, common.test_user_1_name ) # Generate the new dependency XML. Normally, the create_repository_dependency method would be used for this, but # it replaces any existing tool or repository dependency XML file with the generated contents. This is undesirable # in this case, because matplotlib already has an additional tool dependency definition that we don't want to @@ -128,8 +125,8 @@ 'Inhherited' in this case means that matplotlib should show a package tool dependency on numpy version 1.7, and a repository dependency on the latest revision of package_numpy_1_7_0170. ''' - numpy_repository = test_db_util.get_repository_by_name_and_owner( numpy_repository_name, common.test_user_1_name ) - matplotlib_repository = test_db_util.get_repository_by_name_and_owner( matplotlib_repository_name, common.test_user_1_name ) + numpy_repository = self.test_db_util.get_repository_by_name_and_owner( numpy_repository_name, common.test_user_1_name ) + matplotlib_repository = self.test_db_util.get_repository_by_name_and_owner( matplotlib_repository_name, common.test_user_1_name ) changeset_revision = self.get_repository_tip( numpy_repository ) self.check_repository_dependency( matplotlib_repository, depends_on_repository=numpy_repository ) self.display_manage_repository_page( matplotlib_repository, strings_displayed=[ 'numpy', '1.7', 'package', changeset_revision ] ) diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0300_reset_all_metadata.py --- a/test/tool_shed/functional/test_0300_reset_all_metadata.py +++ b/test/tool_shed/functional/test_0300_reset_all_metadata.py @@ -1,6 +1,4 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util - datatypes_repository_name = 'emboss_datatypes_0020' datatypes_repository_description = "Galaxy applicable data formats used by Emboss tools." datatypes_repository_long_description = "Galaxy applicable data formats used by Emboss tools. This repository contains no tools." @@ -41,6 +39,7 @@ running_standalone = False + class TestResetAllRepositoryMetadata( ShedTwillTestCase ): '''Verify that the "Reset selected metadata" feature works.''' @@ -48,14 +47,14 @@ """Create necessary user accounts.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_filtering_repository( self ): '''Create and populate the filtering_0000 repository.''' @@ -298,10 +297,10 @@ '''Create the dependency structure for test 0030.''' global running_standalone if running_standalone: - datatypes_repository = test_db_util.get_repository_by_name_and_owner( 'emboss_datatypes_0030', common.test_user_1_name ) - emboss_repository = test_db_util.get_repository_by_name_and_owner( 'emboss_0030', common.test_user_1_name ) - emboss_5_repository = test_db_util.get_repository_by_name_and_owner( 'emboss_5_0030', common.test_user_1_name ) - emboss_6_repository = test_db_util.get_repository_by_name_and_owner( 'emboss_6_0030', common.test_user_1_name ) + datatypes_repository = self.test_db_util.get_repository_by_name_and_owner( 'emboss_datatypes_0030', common.test_user_1_name ) + emboss_repository = self.test_db_util.get_repository_by_name_and_owner( 'emboss_0030', common.test_user_1_name ) + emboss_5_repository = self.test_db_util.get_repository_by_name_and_owner( 'emboss_5_0030', common.test_user_1_name ) + emboss_6_repository = self.test_db_util.get_repository_by_name_and_owner( 'emboss_6_0030', common.test_user_1_name ) repository_dependencies_path = self.generate_temp_path( 'test_0330', additional_paths=[ 'emboss' ] ) datatypes_tuple = ( self.url, datatypes_repository.name, datatypes_repository.user.username, self.get_repository_tip( datatypes_repository ) ) emboss_5_tuple = ( self.url, emboss_5_repository.name, emboss_5_repository.user.username, self.get_repository_tip( emboss_5_repository ) ) @@ -365,8 +364,8 @@ '''Create the dependency structure for test 0040.''' global running_standalone if running_standalone: - freebayes_repository = test_db_util.get_repository_by_name_and_owner( 'freebayes_0040', common.test_user_1_name ) - filtering_repository = test_db_util.get_repository_by_name_and_owner( 'filtering_0040', common.test_user_1_name ) + freebayes_repository = self.test_db_util.get_repository_by_name_and_owner( 'freebayes_0040', common.test_user_1_name ) + filtering_repository = self.test_db_util.get_repository_by_name_and_owner( 'filtering_0040', common.test_user_1_name ) repository_dependencies_path = self.generate_temp_path( 'test_0340', additional_paths=[ 'dependencies' ] ) freebayes_tuple = ( self.url, freebayes_repository.name, freebayes_repository.user.username, self.get_repository_tip( freebayes_repository ) ) filtering_tuple = ( self.url, filtering_repository.name, filtering_repository.user.username, self.get_repository_tip( filtering_repository ) ) @@ -554,13 +553,13 @@ if running_standalone: self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - column_repository = test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) - convert_repository = test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) - datatypes_repository = test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) - emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) - filtering_repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) - freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) - bismark_repository = test_db_util.get_repository_by_name_and_owner( bismark_repository_name, common.test_user_1_name ) + column_repository = self.test_db_util.get_repository_by_name_and_owner( column_repository_name, common.test_user_1_name ) + convert_repository = self.test_db_util.get_repository_by_name_and_owner( convert_repository_name, common.test_user_1_name ) + datatypes_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) + emboss_repository = self.test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + filtering_repository = self.test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) + freebayes_repository = self.test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) + bismark_repository = self.test_db_util.get_repository_by_name_and_owner( bismark_repository_name, common.test_user_1_name ) dependency_xml_path = self.generate_temp_path( 'test_0050', additional_paths=[ 'freebayes' ] ) # convert_chars depends on column_maker # column_maker depends on convert_chars @@ -629,7 +628,7 @@ self.login( email=common.admin_email, username=common.admin_username ) old_metadata = dict() new_metadata = dict() - repositories = test_db_util.get_all_repositories() + repositories = self.test_db_util.get_all_repositories() for repository in repositories: old_metadata[ self.security.encode_id( repository.id ) ] = dict() for metadata in self.get_repository_metadata( repository ): diff -r d0c56b58064bd73bd6ca408796453abb0f5404c2 -r c2511b37db9bfbc6ec07aba5ebc5795b2c8206a3 test/tool_shed/functional/test_0310_hg_push_from_api.py --- a/test/tool_shed/functional/test_0310_hg_push_from_api.py +++ b/test/tool_shed/functional/test_0310_hg_push_from_api.py @@ -1,5 +1,6 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os -import tool_shed.base.test_db_util as test_db_util + + import logging log = logging.getLogger( __name__ ) @@ -22,7 +23,6 @@ 5. Verify that the changesets have been applied. ''' - class TestHgWebFeatures( ShedTwillTestCase ): '''Test http mercurial interface.''' @@ -30,19 +30,19 @@ """Create necessary user accounts and login as an admin user.""" self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) - test_user_1 = test_db_util.get_user( common.test_user_1_email ) + test_user_1 = self.test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email - test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + test_user_1_private_role = self.test_db_util.get_private_role( test_user_1 ) self.logout() self.login( email=common.test_user_2_email, username=common.test_user_2_name ) - test_user_2 = test_db_util.get_user( common.test_user_2_email ) + test_user_2 = self.test_db_util.get_user( common.test_user_2_email ) assert test_user_2 is not None, 'Problem retrieving user with email %s from the database' % common.test_user_2_email - test_user_2_private_role = test_db_util.get_private_role( test_user_2 ) + test_user_2_private_role = self.test_db_util.get_private_role( test_user_2 ) self.logout() self.login( email=common.admin_email, username=common.admin_username ) - admin_user = test_db_util.get_user( common.admin_email ) + admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) + admin_user_private_role = self.test_db_util.get_private_role( admin_user ) def test_0005_create_filtering_repository( self ): '''Create and populate the filtering_0310 repository.''' @@ -98,7 +98,7 @@ We will be prepending a comment to filtering.py. ''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) clone_path = self.generate_temp_path( 'test_0310', additional_paths=[ 'filtering_0310', 'user2' ] ) self.clone_repository( repository, clone_path ) hgrepo = self.get_hg_repo( clone_path ) @@ -133,7 +133,7 @@ We will be prepending a second comment to filtering.py. ''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) clone_path = self.generate_temp_path( 'test_0310', additional_paths=[ 'filtering_0310', 'user1' ] ) self.clone_repository( repository, clone_path ) hgrepo = self.get_hg_repo( clone_path ) @@ -161,7 +161,7 @@ The commit from test_user_2 should not be present in the changelog, since the repositories were cloned to separate locations. ''' - repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) strings_displayed = [ 'Uploaded filtering 1.1.0.', 'Uploaded filtering test data.', 'Added another line to filtering.py.' ] strings_not_displayed = [ 'Added a line to filtering.py' ] This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Use null object pattern to simplify logic related to no dependency manager being set.
by commits-noreply＠bitbucket.org 13 Dec '13

13 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d0c56b58064b/ Changeset: d0c56b58064b User: jmchilton Date: 2013-12-13 22:43:26 Summary: Use null object pattern to simplify logic related to no dependency manager being set. Fixes bug introduced in 4241eff related to dependency_manager being None if tool_dependency_dir is not set. Affected #: 4 files diff -r 97f810044c531393a56cb84898ab5e7b759ee3e2 -r d0c56b58064bd73bd6ca408796453abb0f5404c2 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -688,10 +688,7 @@ if self.command_line and self.command_line.startswith( 'python' ): self.galaxy_lib_dir = os.path.abspath( "lib" ) # cwd = galaxy root # Shell fragment to inject dependencies - if self.app.config.use_tool_dependencies: - self.dependency_shell_commands = self.tool.build_dependency_shell_commands() - else: - self.dependency_shell_commands = None + self.dependency_shell_commands = self.tool.build_dependency_shell_commands() # We need command_line persisted to the db in order for Galaxy to re-queue the job # if the server was stopped and restarted before the job finished job.command_line = self.command_line @@ -1435,10 +1432,7 @@ if self.command_line and self.command_line.startswith( 'python' ): self.galaxy_lib_dir = os.path.abspath( "lib" ) # cwd = galaxy root # Shell fragment to inject dependencies - if self.app.config.use_tool_dependencies: - self.dependency_shell_commands = self.tool.build_dependency_shell_commands() - else: - self.dependency_shell_commands = None + self.dependency_shell_commands = self.tool.build_dependency_shell_commands() # We need command_line persisted to the db in order for Galaxy to re-queue the job # if the server was stopped and restarted before the job finished task.command_line = self.command_line diff -r 97f810044c531393a56cb84898ab5e7b759ee3e2 -r d0c56b58064bd73bd6ca408796453abb0f5404c2 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -40,7 +40,7 @@ from galaxy.tools.actions import DefaultToolAction from galaxy.tools.actions.data_source import DataSourceToolAction from galaxy.tools.actions.data_manager import DataManagerToolAction -from galaxy.tools.deps import DependencyManager, INDETERMINATE_DEPENDENCY +from galaxy.tools.deps import build_dependency_manager from galaxy.tools.deps.requirements import parse_requirements_from_xml from galaxy.tools.parameters import check_param, params_from_strings, params_to_strings from galaxy.tools.parameters.basic import (BaseURLToolParameter, @@ -700,14 +700,7 @@ return stored.latest_workflow def init_dependency_manager( self ): - if self.app.config.use_tool_dependencies: - dependency_manager_kwds = { - 'default_base_path': self.app.config.tool_dependency_dir, - 'conf_file': self.app.config.dependency_resolvers_config_file, - } - self.dependency_manager = DependencyManager( **dependency_manager_kwds ) - else: - self.dependency_manager = None + self.dependency_manager = build_dependency_manager( self.app.config ) @property def sa_session( self ): diff -r 97f810044c531393a56cb84898ab5e7b759ee3e2 -r d0c56b58064bd73bd6ca408796453abb0f5404c2 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -15,6 +15,31 @@ from galaxy.util.submodules import submodules +def build_dependency_manager( config ): + if config.use_tool_dependencies: + dependency_manager_kwds = { + 'default_base_path': config.tool_dependency_dir, + 'conf_file': config.dependency_resolvers_config_file, + } + dependency_manager = DependencyManager( **dependency_manager_kwds ) + else: + dependency_manager = NullDependencyManager() + + return dependency_manager + + +class NullDependencyManager( object ): + + def uses_tool_shed_dependencies(self): + return False + + def dependency_shell_commands( self, requirements, **kwds ): + return [] + + def find_dep( self, name, version=None, type='package', **kwds ): + return INDETERMINATE_DEPENDENCY + + class DependencyManager( object ): """ A DependencyManager attempts to resolve named and versioned dependencies by diff -r 97f810044c531393a56cb84898ab5e7b759ee3e2 -r d0c56b58064bd73bd6ca408796453abb0f5404c2 lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -462,7 +462,7 @@ if tool_dependency.can_install: # The database record is currently in a state that allows us to install the package on the file system. try: - dependencies_ignored = app.toolbox.dependency_manager and not app.toolbox.dependency_manager.uses_tool_shed_dependencies() + dependencies_ignored = not app.toolbox.dependency_manager.uses_tool_shed_dependencies() if dependencies_ignored: log.debug( "Skipping package %s because tool shed dependency resolver not enabled." % str( package_name ) ) # Tool dependency resolves have been configured and they do not include the tool shed. Do not install package. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: guerler: Masthead: Prevent navbar collapse
by commits-noreply＠bitbucket.org 13 Dec '13

13 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/97f810044c53/ Changeset: 97f810044c53 User: guerler Date: 2013-12-13 22:10:21 Summary: Masthead: Prevent navbar collapse Affected #: 2 files diff -r a32dcf32dffc6dd7175092f4fda7476258fe061e -r 97f810044c531393a56cb84898ab5e7b759ee3e2 static/style/blue/base.css --- a/static/style/blue/base.css +++ b/static/style/blue/base.css @@ -80,7 +80,7 @@ dt,dd{line-height:1.428571429} dt{font-weight:bold} dd{margin-left:0} -@media (min-width:768px){.dl-horizontal dt{float:left;width:160px;clear:left;text-align:right;overflow:hidden;text-overflow:ellipsis;white-space:nowrap} .dl-horizontal dd{margin-left:180px}.dl-horizontal dd:before,.dl-horizontal dd:after{content:" ";display:table;} .dl-horizontal dd:after{clear:both} .dl-horizontal dd:before,.dl-horizontal dd:after{content:" ";display:table;} .dl-horizontal dd:after{clear:both}}abbr[title],abbr[data-original-title]{cursor:help;border-bottom:1px dotted #999} +@media (min-width:0){.dl-horizontal dt{float:left;width:160px;clear:left;text-align:right;overflow:hidden;text-overflow:ellipsis;white-space:nowrap} .dl-horizontal dd{margin-left:180px}.dl-horizontal dd:before,.dl-horizontal dd:after{content:" ";display:table;} .dl-horizontal dd:after{clear:both} .dl-horizontal dd:before,.dl-horizontal dd:after{content:" ";display:table;} .dl-horizontal dd:after{clear:both}}abbr[title],abbr[data-original-title]{cursor:help;border-bottom:1px dotted #999} abbr.initialism{font-size:90%;text-transform:uppercase} blockquote{padding:8.5px 17px;margin:0 0 17px;border-left:5px solid #eee}blockquote p{font-size:15px;font-weight:300;line-height:1.25} blockquote p:last-child{margin-bottom:0} @@ -433,7 +433,7 @@ .pull-right>.dropdown-menu{right:0;left:auto} .dropup .caret,.navbar-fixed-bottom .dropdown .caret{border-top:0 dotted;border-bottom:4px solid #000;content:""} .dropup .dropdown-menu,.navbar-fixed-bottom .dropdown .dropdown-menu{top:auto;bottom:100%;margin-bottom:1px} -@media (min-width:768px){.navbar-right .dropdown-menu{right:0;left:auto}}.btn-default .caret{border-top-color:#333} +@media (min-width:0){.navbar-right .dropdown-menu{right:0;left:auto}}.btn-default .caret{border-top-color:#333} .btn-primary .caret,.btn-success .caret,.btn-warning .caret,.btn-danger .caret,.btn-info .caret{border-top-color:#fff} .dropup .btn-default .caret{border-bottom-color:#333} .dropup .btn-primary .caret,.dropup .btn-success .caret,.dropup .btn-warning .caret,.dropup .btn-danger .caret,.dropup .btn-info .caret{border-bottom-color:#fff} @@ -533,37 +533,37 @@ .navbar:after{clear:both} .navbar:before,.navbar:after{content:" ";display:table;} .navbar:after{clear:both} -@media (min-width:768px){.navbar{border-radius:3px}} +@media (min-width:0){.navbar{border-radius:3px}} .navbar-header:before,.navbar-header:after{content:" ";display:table;} .navbar-header:after{clear:both} .navbar-header:before,.navbar-header:after{content:" ";display:table;} .navbar-header:after{clear:both} -@media (min-width:768px){.navbar-header{float:left}} +@media (min-width:0){.navbar-header{float:left}} .navbar-collapse{max-height:340px;overflow-x:visible;padding-right:15px;padding-left:15px;border-top:1px solid transparent;box-shadow:inset 0 1px 0 rgba(255,255,255,0.1);-webkit-overflow-scrolling:touch}.navbar-collapse:before,.navbar-collapse:after{content:" ";display:table;} .navbar-collapse:after{clear:both} .navbar-collapse:before,.navbar-collapse:after{content:" ";display:table;} .navbar-collapse:after{clear:both} .navbar-collapse.in{overflow-y:auto} -@media (min-width:768px){.navbar-collapse{width:auto;border-top:0;box-shadow:none}.navbar-collapse.collapse{display:block !important;height:auto !important;padding-bottom:0;overflow:visible !important} .navbar-collapse.in{overflow-y:visible} .navbar-collapse .navbar-nav.navbar-left:first-child{margin-left:-15px} .navbar-collapse .navbar-nav.navbar-right:last-child{margin-right:-15px} .navbar-collapse .navbar-text:last-child{margin-right:0}} -.container>.navbar-header,.container>.navbar-collapse{margin-right:-15px;margin-left:-15px}@media (min-width:768px){.container>.navbar-header,.container>.navbar-collapse{margin-right:0;margin-left:0}} -.navbar-static-top{border-width:0 0 1px}@media (min-width:768px){.navbar-static-top{border-radius:0}} -.navbar-fixed-top,.navbar-fixed-bottom{position:fixed;right:0;left:0;border-width:0 0 1px}@media (min-width:768px){.navbar-fixed-top,.navbar-fixed-bottom{border-radius:0}} +@media (min-width:0){.navbar-collapse{width:auto;border-top:0;box-shadow:none}.navbar-collapse.collapse{display:block !important;height:auto !important;padding-bottom:0;overflow:visible !important} .navbar-collapse.in{overflow-y:visible} .navbar-collapse .navbar-nav.navbar-left:first-child{margin-left:-15px} .navbar-collapse .navbar-nav.navbar-right:last-child{margin-right:-15px} .navbar-collapse .navbar-text:last-child{margin-right:0}} +.container>.navbar-header,.container>.navbar-collapse{margin-right:-15px;margin-left:-15px}@media (min-width:0){.container>.navbar-header,.container>.navbar-collapse{margin-right:0;margin-left:0}} +.navbar-static-top{border-width:0 0 1px}@media (min-width:0){.navbar-static-top{border-radius:0}} +.navbar-fixed-top,.navbar-fixed-bottom{position:fixed;right:0;left:0;border-width:0 0 1px}@media (min-width:0){.navbar-fixed-top,.navbar-fixed-bottom{border-radius:0}} .navbar-fixed-top{z-index:1030;top:0} .navbar-fixed-bottom{bottom:0;margin-bottom:0} .navbar-brand{float:left;padding:8px 15px;font-size:15px;line-height:17px}.navbar-brand:hover,.navbar-brand:focus{text-decoration:none} -@media (min-width:768px){.navbar>.container .navbar-brand{margin-left:-15px}} +@media (min-width:0){.navbar>.container .navbar-brand{margin-left:-15px}} .navbar-toggle{position:relative;float:right;margin-right:15px;padding:9px 10px;margin-top:-0.5px;margin-bottom:-0.5px;background-color:transparent;border:1px solid transparent;border-radius:3px}.navbar-toggle .icon-bar{display:block;width:22px;height:2px;border-radius:1px} .navbar-toggle .icon-bar+.icon-bar{margin-top:4px} -@media (min-width:768px){.navbar-toggle{display:none}} +@media (min-width:0){.navbar-toggle{display:none}} .navbar-nav{margin:4px -15px}.navbar-nav>li>a{padding-top:10px;padding-bottom:10px;line-height:17px} -@media (max-width:767px){.navbar-nav .open .dropdown-menu{position:static;float:none;width:auto;margin-top:0;background-color:transparent;border:0;box-shadow:none}.navbar-nav .open .dropdown-menu>li>a,.navbar-nav .open .dropdown-menu .dropdown-header{padding:5px 15px 5px 25px} .navbar-nav .open .dropdown-menu>li>a{line-height:17px}.navbar-nav .open .dropdown-menu>li>a:hover,.navbar-nav .open .dropdown-menu>li>a:focus{background-image:none}}@media (min-width:768px){.navbar-nav{float:left;margin:0}.navbar-nav>li{float:left}.navbar-nav>li>a{padding-top:8px;padding-bottom:8px}} -@media (min-width:768px){.navbar-left{float:left !important;float:left} .navbar-right{float:right !important;float:right}}.navbar-form{margin-left:-15px;margin-right:-15px;padding:10px 15px;border-top:1px solid transparent;border-bottom:1px solid transparent;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);margin-top:3px;margin-bottom:3px}@media (min-width:768px){.navbar-form .form-group{display:inline-block;margin-bottom:0;vertical-align:middle} .navbar-form .form-control{display:inline-block} .navbar-form .radio,.navbar-form .checkbox{display:inline-block;margin-top:0;margin-bottom:0;padding-left:0} .navbar-form .radio input[type="radio"],.navbar-form .checkbox input[type="checkbox"]{float:none;margin-left:0}}@media (max-width:767px){.navbar-form .form-group{margin-bottom:5px}} -@media (min-width:768px){.navbar-form{width:auto;border:0;margin-left:0;margin-right:0;padding-top:0;padding-bottom:0;-webkit-box-shadow:none;box-shadow:none}} +@media (max-width:767px){.navbar-nav .open .dropdown-menu{position:static;float:none;width:auto;margin-top:0;background-color:transparent;border:0;box-shadow:none}.navbar-nav .open .dropdown-menu>li>a,.navbar-nav .open .dropdown-menu .dropdown-header{padding:5px 15px 5px 25px} .navbar-nav .open .dropdown-menu>li>a{line-height:17px}.navbar-nav .open .dropdown-menu>li>a:hover,.navbar-nav .open .dropdown-menu>li>a:focus{background-image:none}}@media (min-width:0){.navbar-nav{float:left;margin:0}.navbar-nav>li{float:left}.navbar-nav>li>a{padding-top:8px;padding-bottom:8px}} +@media (min-width:0){.navbar-left{float:left !important;float:left} .navbar-right{float:right !important;float:right}}.navbar-form{margin-left:-15px;margin-right:-15px;padding:10px 15px;border-top:1px solid transparent;border-bottom:1px solid transparent;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);margin-top:3px;margin-bottom:3px}@media (min-width:768px){.navbar-form .form-group{display:inline-block;margin-bottom:0;vertical-align:middle} .navbar-form .form-control{display:inline-block} .navbar-form .radio,.navbar-form .checkbox{display:inline-block;margin-top:0;margin-bottom:0;padding-left:0} .navbar-form .radio input[type="radio"],.navbar-form .checkbox input[type="checkbox"]{float:none;margin-left:0}}@media (max-width:767px){.navbar-form .form-group{margin-bottom:5px}} +@media (min-width:0){.navbar-form{width:auto;border:0;margin-left:0;margin-right:0;padding-top:0;padding-bottom:0;-webkit-box-shadow:none;box-shadow:none}} .navbar-nav>li>.dropdown-menu{margin-top:0;border-top-right-radius:0;border-top-left-radius:0} .navbar-fixed-bottom .navbar-nav>li>.dropdown-menu{border-bottom-right-radius:0;border-bottom-left-radius:0} .navbar-nav.pull-right>li>.dropdown-menu,.navbar-nav>li>.dropdown-menu.pull-right{left:auto;right:0} .navbar-btn{margin-top:3px;margin-bottom:3px} -.navbar-text{float:left;margin-top:8px;margin-bottom:8px}@media (min-width:768px){.navbar-text{margin-left:15px;margin-right:15px}} +.navbar-text{float:left;margin-top:8px;margin-bottom:8px}@media (min-width:0){.navbar-text{margin-left:15px;margin-right:15px}} .navbar-default{background-color:#eee;border-color:#888}.navbar-default .navbar-brand{color:#777}.navbar-default .navbar-brand:hover,.navbar-default .navbar-brand:focus{color:#5e5e5e;background-color:transparent} .navbar-default .navbar-text{color:#777} .navbar-default .navbar-nav>li>a{color:#777}.navbar-default .navbar-nav>li>a:hover,.navbar-default .navbar-nav>li>a:focus{color:#333;background-color:transparent} diff -r a32dcf32dffc6dd7175092f4fda7476258fe061e -r 97f810044c531393a56cb84898ab5e7b759ee3e2 static/style/src/less/galaxy_bootstrap/variables.less --- a/static/style/src/less/galaxy_bootstrap/variables.less +++ b/static/style/src/less/galaxy_bootstrap/variables.less @@ -295,7 +295,7 @@ // Padding, to be divided by two and applied to the left and right of all columns @grid-gutter-width: 30px; // Point at which the navbar stops collapsing -@grid-float-breakpoint: @screen-sm; +@grid-float-breakpoint: 0px; // Navbar Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0