galaxy-commits December 2013

galaxy-commits@lists.galaxyproject.org

1 participants
207 discussions

commit/galaxy-central: greg: Correctly use the initialized test_environment dict when running the tool shed's install and test framework and additional code cleanup for the framework.
by commits-noreply＠bitbucket.org 04 Dec '13

04 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/dc3944242df9/ Changeset: dc3944242df9 User: greg Date: 2013-12-04 21:07:39 Summary: Correctly use the initialized test_environment dict when running the tool shed's install and test framework and additional code cleanup for the framework. Affected #: 8 files diff -r ca25579a90e97157e32b527e7471cde90e9beed0 -r dc3944242df936a5026988aaf11f0a90fbe7b115 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -4,7 +4,7 @@ import sys new_path = [ os.path.join( os.getcwd(), "lib" ), - os.path.join( os.getcwd(), "test" ) ] + os.path.join( os.getcwd(), "test" ) ] new_path.extend( sys.path[ 1: ] ) sys.path = new_path @@ -21,8 +21,8 @@ import tempfile import time -from base.util import get_database_version -from base.util import get_repository_current_revision +from install_and_test_tool_shed_repositories.base.util import get_database_version +from install_and_test_tool_shed_repositories.base.util import get_repository_current_revision from galaxy.model.orm import and_ from galaxy.model.orm import not_ from galaxy.model.orm import select diff -r ca25579a90e97157e32b527e7471cde90e9beed0 -r dc3944242df936a5026988aaf11f0a90fbe7b115 lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py --- a/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py +++ b/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py @@ -25,9 +25,8 @@ import galaxy.webapps.tool_shed.config as tool_shed_config import galaxy.webapps.tool_shed.model.mapping -from base.util import get_database_version -from base.util import get_repository_current_revision -from base.util import get_test_environment +from install_and_test_tool_shed_repositories.base.util import get_database_version +from install_and_test_tool_shed_repositories.base.util import get_repository_current_revision from galaxy.model.orm import and_ from galaxy.model.orm import not_ from galaxy.model.orm import select diff -r ca25579a90e97157e32b527e7471cde90e9beed0 -r dc3944242df936a5026988aaf11f0a90fbe7b115 scripts/functional_tests.py --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -13,7 +13,7 @@ new_path.extend( sys.path[1:] ) sys.path = new_path -from base.util import parse_tool_panel_config +from base.tool_shed_util import parse_tool_panel_config from galaxy import eggs diff -r ca25579a90e97157e32b527e7471cde90e9beed0 -r dc3944242df936a5026988aaf11f0a90fbe7b115 test/base/tool_shed_util.py --- /dev/null +++ b/test/base/tool_shed_util.py @@ -0,0 +1,90 @@ +import logging +import os +import sys + +cwd = os.getcwd() +if cwd not in sys.path: + sys.path.append( cwd ) + +new_path = [ os.path.join( cwd, "lib" ) ] +if new_path not in sys.path: + new_path.extend( sys.path ) + sys.path = new_path + +from galaxy.util import parse_xml + +log = logging.getLogger(__name__) + +def get_installed_repository_info( elem, last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision, tool_path ): + """ + Return the GALAXY_TEST_FILE_DIR, the containing repository name and the change set revision for the tool elem. + This only happens when testing tools installed from the tool shed. + """ + tool_config_path = elem.get( 'file' ) + installed_tool_path_items = tool_config_path.split( '/repos/' ) + sans_shed = installed_tool_path_items[ 1 ] + path_items = sans_shed.split( '/' ) + repository_owner = path_items[ 0 ] + repository_name = path_items[ 1 ] + changeset_revision = path_items[ 2 ] + if repository_name != last_tested_repository_name or changeset_revision != last_tested_changeset_revision: + # Locate the test-data directory. + installed_tool_path = os.path.join( installed_tool_path_items[ 0 ], 'repos', repository_owner, repository_name, changeset_revision ) + for root, dirs, files in os.walk( os.path.join(tool_path, installed_tool_path )): + if '.' in dirs: + dirs.remove( '.hg' ) + if 'test-data' in dirs: + return os.path.join( root, 'test-data' ), repository_name, changeset_revision + return None, repository_name, changeset_revision + return last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision + +def parse_tool_panel_config( config, shed_tools_dict ): + """ + Parse a shed-related tool panel config to generate the shed_tools_dict. This only happens when testing tools installed from the tool shed. + """ + last_galaxy_test_file_dir = None + last_tested_repository_name = None + last_tested_changeset_revision = None + tool_path = None + has_test_data = False + tree = parse_xml( config ) + root = tree.getroot() + tool_path = root.get('tool_path') + for elem in root: + if elem.tag == 'tool': + galaxy_test_file_dir, \ + last_tested_repository_name, \ + last_tested_changeset_revision = get_installed_repository_info( elem, + last_galaxy_test_file_dir, + last_tested_repository_name, + last_tested_changeset_revision, + tool_path ) + if galaxy_test_file_dir: + if not has_test_data: + has_test_data = True + if galaxy_test_file_dir != last_galaxy_test_file_dir: + if not os.path.isabs( galaxy_test_file_dir ): + galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) + guid = elem.get( 'guid' ) + shed_tools_dict[ guid ] = galaxy_test_file_dir + last_galaxy_test_file_dir = galaxy_test_file_dir + elif elem.tag == 'section': + for section_elem in elem: + if section_elem.tag == 'tool': + galaxy_test_file_dir, \ + last_tested_repository_name, \ + last_tested_changeset_revision = get_installed_repository_info( section_elem, + last_galaxy_test_file_dir, + last_tested_repository_name, + last_tested_changeset_revision, + tool_path ) + if galaxy_test_file_dir: + if not has_test_data: + has_test_data = True + if galaxy_test_file_dir != last_galaxy_test_file_dir: + if not os.path.isabs( galaxy_test_file_dir ): + galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) + guid = section_elem.get( 'guid' ) + shed_tools_dict[ guid ] = galaxy_test_file_dir + last_galaxy_test_file_dir = galaxy_test_file_dir + return has_test_data, shed_tools_dict diff -r ca25579a90e97157e32b527e7471cde90e9beed0 -r dc3944242df936a5026988aaf11f0a90fbe7b115 test/base/util.py --- a/test/base/util.py +++ /dev/null @@ -1,140 +0,0 @@ -import logging -import os -import platform -import sys - -cwd = os.getcwd() -if cwd not in sys.path: - sys.path.append( cwd ) - -new_path = [ os.path.join( cwd, "lib" ) ] -if new_path not in sys.path: - new_path.extend( sys.path ) - sys.path = new_path - -from galaxy.util import parse_xml -from galaxy import eggs - -eggs.require( 'mercurial' ) - -from mercurial import hg -from mercurial import ui - -log = logging.getLogger(__name__) - -def get_database_version( app ): - ''' - This method returns the value of the version column from the migrate_version table, using the provided app's SQLAlchemy session to determine - which table to get that from. This way, it's provided with an instance of a Galaxy UniverseApplication, it will return the Galaxy instance's - database migration version. If a tool shed UniverseApplication is provided, it returns the tool shed's database migration version. - ''' - sa_session = app.model.context.current - result = sa_session.execute( 'SELECT version FROM migrate_version LIMIT 1' ) - # This query will return the following structure: - # row = [ column 0, column 1, ..., column n ] - # rows = [ row 0, row 1, ..., row n ] - # The first column in the first row is the version number we want. - for row in result: - version = row[ 0 ] - break - return version - -def get_installed_repository_info( elem, last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision, tool_path ): - """ - Return the GALAXY_TEST_FILE_DIR, the containing repository name and the change set revision for the tool elem. - This only happens when testing tools installed from the tool shed. - """ - tool_config_path = elem.get( 'file' ) - installed_tool_path_items = tool_config_path.split( '/repos/' ) - sans_shed = installed_tool_path_items[ 1 ] - path_items = sans_shed.split( '/' ) - repository_owner = path_items[ 0 ] - repository_name = path_items[ 1 ] - changeset_revision = path_items[ 2 ] - if repository_name != last_tested_repository_name or changeset_revision != last_tested_changeset_revision: - # Locate the test-data directory. - installed_tool_path = os.path.join( installed_tool_path_items[ 0 ], 'repos', repository_owner, repository_name, changeset_revision ) - for root, dirs, files in os.walk( os.path.join(tool_path, installed_tool_path )): - if '.hg' in dirs: - dirs.remove( '.hg' ) - if 'test-data' in dirs: - return os.path.join( root, 'test-data' ), repository_name, changeset_revision - return None, repository_name, changeset_revision - return last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision - -def get_repository_current_revision( repo_path ): - ''' - This method uses the python mercurial API to get the current working directory's mercurial changeset hash. Note that if the author of mercurial - changes the API, this method will have to be updated or replaced. - ''' - # Initialize a mercurial repo object from the provided path. - repo = hg.repository( ui.ui(), repo_path ) - # Get the working directory's change context. - ctx = repo[ None ] - # Extract the changeset hash of the first parent of that change context (the most recent changeset to which the working directory was updated). - changectx = ctx.parents()[ 0 ] - # Also get the numeric revision, so we can return the customary id:hash changeset identifiers. - ctx_rev = changectx.rev() - hg_id = '%d:%s' % ( ctx_rev, str( changectx ) ) - return hg_id - -def get_test_environment( current_environment=None ): - if current_environment is None: - current_environment = {} - rval = current_environment - rval[ 'python_version' ] = platform.python_version() - rval[ 'architecture' ] = platform.machine() - os, hostname, os_version, uname, arch, processor = platform.uname() - rval[ 'system' ] = '%s %s' % ( os, os_version ) - return rval - -def parse_tool_panel_config( config, shed_tools_dict ): - """ - Parse a shed-related tool panel config to generate the shed_tools_dict. This only happens when testing tools installed from the tool shed. - """ - last_galaxy_test_file_dir = None - last_tested_repository_name = None - last_tested_changeset_revision = None - tool_path = None - has_test_data = False - tree = parse_xml( config ) - root = tree.getroot() - tool_path = root.get('tool_path') - for elem in root: - if elem.tag == 'tool': - galaxy_test_file_dir, \ - last_tested_repository_name, \ - last_tested_changeset_revision = get_installed_repository_info( elem, - last_galaxy_test_file_dir, - last_tested_repository_name, - last_tested_changeset_revision, - tool_path ) - if galaxy_test_file_dir: - if not has_test_data: - has_test_data = True - if galaxy_test_file_dir != last_galaxy_test_file_dir: - if not os.path.isabs( galaxy_test_file_dir ): - galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) - guid = elem.get( 'guid' ) - shed_tools_dict[ guid ] = galaxy_test_file_dir - last_galaxy_test_file_dir = galaxy_test_file_dir - elif elem.tag == 'section': - for section_elem in elem: - if section_elem.tag == 'tool': - galaxy_test_file_dir, \ - last_tested_repository_name, \ - last_tested_changeset_revision = get_installed_repository_info( section_elem, - last_galaxy_test_file_dir, - last_tested_repository_name, - last_tested_changeset_revision, - tool_path ) - if galaxy_test_file_dir: - if not has_test_data: - has_test_data = True - if galaxy_test_file_dir != last_galaxy_test_file_dir: - if not os.path.isabs( galaxy_test_file_dir ): - galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) - guid = section_elem.get( 'guid' ) - shed_tools_dict[ guid ] = galaxy_test_file_dir - last_galaxy_test_file_dir = galaxy_test_file_dir - return has_test_data, shed_tools_dict diff -r ca25579a90e97157e32b527e7471cde90e9beed0 -r dc3944242df936a5026988aaf11f0a90fbe7b115 test/install_and_test_tool_shed_repositories/base/util.py --- /dev/null +++ b/test/install_and_test_tool_shed_repositories/base/util.py @@ -0,0 +1,54 @@ +import logging +import os +import sys + +cwd = os.getcwd() +if cwd not in sys.path: + sys.path.append( cwd ) + +new_path = [ os.path.join( cwd, "lib" ) ] +if new_path not in sys.path: + new_path.extend( sys.path ) + sys.path = new_path + +from galaxy import eggs + +eggs.require( 'mercurial' ) + +from mercurial import hg +from mercurial import ui + +log = logging.getLogger(__name__) + +def get_database_version( app ): + ''' + This method returns the value of the version column from the migrate_version table, using the provided app's SQLAlchemy session to determine + which table to get that from. This way, it's provided with an instance of a Galaxy UniverseApplication, it will return the Galaxy instance's + database migration version. If a tool shed UniverseApplication is provided, it returns the tool shed's database migration version. + ''' + sa_session = app.model.context.current + result = sa_session.execute( 'SELECT version FROM migrate_version LIMIT 1' ) + # This query will return the following structure: + # row = [ column 0, column 1, ..., column n ] + # rows = [ row 0, row 1, ..., row n ] + # The first column in the first row is the version number we want. + for row in result: + version = row[ 0 ] + break + return version + +def get_repository_current_revision( repo_path ): + ''' + This method uses the python mercurial API to get the current working directory's mercurial changeset hash. Note that if the author of mercurial + changes the API, this method will have to be updated or replaced. + ''' + # Initialize a mercurial repo object from the provided path. + repo = hg.repository( ui.ui(), repo_path ) + # Get the working directory's change context. + ctx = repo[ None ] + # Extract the changeset hash of the first parent of that change context (the most recent changeset to which the working directory was updated). + changectx = ctx.parents()[ 0 ] + # Also get the numeric revision, so we can return the customary id:hash changeset identifiers. + ctx_rev = changectx.rev() + hg_id = '%d:%s' % ( ctx_rev, str( changectx ) ) + return hg_id diff -r ca25579a90e97157e32b527e7471cde90e9beed0 -r dc3944242df936a5026988aaf11f0a90fbe7b115 test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py --- a/test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py +++ b/test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py @@ -1,6 +1,8 @@ -import new, logging +import logging +import new import install_and_test_tool_shed_repositories.base.test_db_util as test_db_util from install_and_test_tool_shed_repositories.base.twilltestcase import InstallTestRepository + log = logging.getLogger(__name__) @@ -80,4 +82,3 @@ # from baseclasses (which should be a tuple of classes) and with namespace dict. new_class_obj = new.classobj( str( name ), baseclasses, namespace ) G[ name ] = new_class_obj - diff -r ca25579a90e97157e32b527e7471cde90e9beed0 -r dc3944242df936a5026988aaf11f0a90fbe7b115 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -10,9 +10,9 @@ cwd = os.getcwd() sys.path.append( cwd ) new_path = [ os.path.join( cwd, "scripts" ), - os.path.join( cwd, "lib" ), - os.path.join( cwd, 'test' ), - os.path.join( cwd, 'scripts', 'api' ) ] + os.path.join( cwd, "lib" ), + os.path.join( cwd, 'test' ), + os.path.join( cwd, 'scripts', 'api' ) ] new_path.extend( sys.path ) sys.path = new_path @@ -30,6 +30,7 @@ import install_and_test_tool_shed_repositories.functional.test_install_repositories as test_install_repositories import logging import nose +import platform import random import re import shutil @@ -41,10 +42,9 @@ import tool_shed.util.shed_util_common as suc import urllib -from base.util import get_database_version -from base.util import get_repository_current_revision -from base.util import get_test_environment -from base.util import parse_tool_panel_config +from base.tool_shed_util import parse_tool_panel_config +from install_and_test_tool_shed_repositories.base.util import get_database_version +from install_and_test_tool_shed_repositories.base.util import get_repository_current_revision from common import update from datetime import datetime from galaxy.app import UniverseApplication @@ -614,31 +614,38 @@ owner = str( repository_dict[ 'owner' ] ) changeset_revision = str( repository_dict[ 'changeset_revision' ] ) log.debug( "Processing revision %s of repository %s owned by %s..." % ( changeset_revision, name, owner ) ) - # Populate the tool_test_results_dict. + # Retrieve the stored list of tool_test_results_dicts. tool_test_results_dicts, error_message = get_tool_test_results_dicts( galaxy_tool_shed_url, encoded_repository_metadata_id ) if error_message: log.debug( error_message ) else: - # The preparation script ~/tool_shed/scripts/check_repositories_for_functional_tests.py will have entered - # information in the 'test_environment' and possibly the 'missing_test_components' entries of the first - # tool_test_results_dict in the list of tool_test_results_dicts. We need to be careful to not lose this - # information. - try: - tool_test_results_dict = tool_test_results_dicts.pop( 0 ) - except Exception, e: - log.exception( "Invalid list of tool_test_results_dicts %s: %s" % ( str( tool_test_results_dicts ), str( e ) ) ) - continue + if tool_test_results_dicts: + # Inspect the tool_test_results_dict for the last test run to make sure it contains only a test_environment + # entry. If it contains more entries, then the script ~/tool_shed/api/check_repositories_for_functional_tests.py + # was not executed in preparation for this script's execution, so we'll just create an empty dictionary. + tool_test_results_dict = tool_test_results_dicts[ 0 ] + if len( tool_test_results_dict ) <= 1: + # We can re-use the mostly empty tool_test_results_dict for this run because it is either empty or it contains only + # a test_environment entry. If we use it we need to temporarily eliminate it from the list of tool_test_results_dicts + # since it will be re-inserted later. + tool_test_results_dict = tool_test_results_dicts.pop( 0 ) + else: + # The latest tool_test_results_dict has been populated with the results of a test run, so it cannot be used. + tool_test_results_dict = {} + else: + # Create a new dictionary for this test test run, + tool_test_results_dict = {} # See if this repository should be skipped for any reason. - this_repository_is_in_the_exclude_lost = False + this_repository_is_in_the_exclude_list = False skip_reason = None for exclude_dict in exclude_list: reason = exclude_dict[ 'reason' ] exclude_repositories = exclude_dict[ 'repositories' ] if ( name, owner, changeset_revision ) in exclude_repositories or ( name, owner, None ) in exclude_repositories: - this_repository_is_in_the_exclude_lost = True + this_repository_is_in_the_exclude_list = True skip_reason = reason break - if this_repository_is_in_the_exclude_lost: + if this_repository_is_in_the_exclude_list: tool_test_results_dict[ 'not_tested' ] = dict( reason=skip_reason ) params = dict( tools_functionally_correct=False, do_not_test=False ) @@ -647,19 +654,32 @@ log.debug( "Not testing revision %s of repository %s owned by %s because it is in the exclude list for this test run." % \ ( changeset_revision, name, owner ) ) else: - test_environment_dict = tool_test_results_dict.get( 'test_environment', None ) - test_environment_dict = get_test_environment( test_environment_dict ) + test_environment_dict = tool_test_results_dict.get( 'test_environment', {} ) + if len( test_environment_dict ) == 0: + # Set information about the tool shed to nothing since we cannot currently determine it from here. + # We could eventually add an API method... + test_environment_dict = dict( tool_shed_database_version='', + tool_shed_mercurial_version='', + tool_shed_revision='' ) # Add the current time as the approximate time that this test run occurs. A similar value will also be # set to the repository_metadata.time_last_tested column, but we also store it here because the Tool Shed # may be configured to store multiple test run results, so each must be associated with a time stamp. now = time.strftime( "%Y-%m-%d %H:%M:%S" ) + # Add information about the current platform. test_environment_dict[ 'time_tested' ] = now + test_environment_dict[ 'python_version' ] = platform.python_version() + test_environment_dict[ 'architecture' ] = platform.machine() + operating_system, hostname, operating_system_version, uname, arch, processor = platform.uname() + test_environment_dict[ 'system' ] = '%s %s' % ( operating_system, operating_system_version ) + # Add information about the current Galaxy environment. test_environment_dict[ 'galaxy_database_version' ] = get_database_version( app ) test_environment_dict[ 'galaxy_revision' ] = get_repository_current_revision( os.getcwd() ) + # Initialize and populate the tool_test_results_dict. tool_test_results_dict[ 'test_environment' ] = test_environment_dict tool_test_results_dict[ 'passed_tests' ] = [] tool_test_results_dict[ 'failed_tests' ] = [] tool_test_results_dict[ 'installation_errors' ] = dict( current_repository=[], repository_dependencies=[], tool_dependencies=[] ) + # Proceed with installing repositories and testing contained tools. repository, error_message = install_repository( app, repository_dict ) if error_message: tool_test_results_dict[ 'installation_errors' ][ 'current_repository' ] = error_message Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Actually skip inspection of repository revisions that have the skip_tool_test set in the Tool Shed instead of just stating we're going to...
by commits-noreply＠bitbucket.org 04 Dec '13

04 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ca25579a90e9/ Changeset: ca25579a90e9 User: greg Date: 2013-12-04 17:01:32 Summary: Actually skip inspection of repository revisions that have the skip_tool_test set in the Tool Shed instead of just stating we're going to... Affected #: 1 file diff -r 13f8c99da1cd462841b268b5beaf2148086bd19d -r ca25579a90e97157e32b527e7471cde90e9beed0 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -128,6 +128,7 @@ print '# Checking revision %s of %s owned by %s.' % ( changeset_revision, name, owner ) if repository_metadata.id in skip_metadata_ids: print'# Skipping revision %s of %s owned by %s because the skip_tool_test setting has been set.' % ( changeset_revision, name, owner ) + continue # If this changeset revision has no tools, we don't need to do anything here, the install and test script has a filter for returning # only repositories that contain tools. tool_dicts = metadata.get( 'tools', None ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Final fixes for rendering installation errors in the tool test results container in the tool shed.
by commits-noreply＠bitbucket.org 04 Dec '13

04 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/13f8c99da1cd/ Changeset: 13f8c99da1cd User: greg Date: 2013-12-04 16:29:59 Summary: Final fixes for rendering installation errors in the tool test results container in the tool shed. Affected #: 2 files diff -r 8d62ee337534d00d8a3d74150d3ed71d5a1f9b40 -r 13f8c99da1cd462841b268b5beaf2148086bd19d lib/tool_shed/util/container_util.py --- a/lib/tool_shed/util/container_util.py +++ b/lib/tool_shed/util/container_util.py @@ -1199,6 +1199,7 @@ key='current_repository_installation_errors', label='This repository', parent=installation_error_base_folder ) + installation_error_base_folder.folders.append( current_repository_folder ) for current_repository_error_dict in current_repository_installation_error_dicts: repository_installation_error_id += 1 repository_installation_error = \ @@ -1209,13 +1210,13 @@ changeset_revision=str( current_repository_error_dict.get( 'changeset_revision', '' ) ), error_message=current_repository_error_dict.get( 'error_message', '' ) ) current_repository_folder.current_repository_installation_errors.append( repository_installation_error ) - installation_error_base_folder.folders.append( current_repository_folder ) if len( repository_dependency_installation_error_dicts ) > 0: folder_id += 1 repository_dependencies_folder = Folder( id=folder_id, key='repository_dependency_installation_errors', label='Repository dependencies', parent=installation_error_base_folder ) + installation_error_base_folder.folders.append( repository_dependencies_folder ) for repository_dependency_error_dict in repository_dependency_installation_error_dicts: repository_installation_error_id += 1 repository_installation_error = \ @@ -1226,7 +1227,6 @@ changeset_revision=str( repository_dependency_error_dict.get( 'changeset_revision', '' ) ), error_message=repository_dependency_error_dict.get( 'error_message', '' ) ) repository_dependencies_folder.repository_installation_errors.append( repository_installation_error ) - installation_error_base_folder.folders.append( repository_dependencies_folder ) if len( tool_dependency_installation_error_dicts ) > 0: # [{'error_message': 'some traceback string' 'type': 'package', 'name': 'MIRA', 'version': '4.0'}] folder_id += 1 @@ -1234,6 +1234,7 @@ key='tool_dependency_installation_errors', label='Tool dependencies', parent=installation_error_base_folder ) + installation_error_base_folder.folders.append( tool_dependencies_folder ) tool_dependency_error_id = 0 for tool_dependency_error_dict in tool_dependency_installation_error_dicts: tool_dependency_error_id += 1 @@ -1244,7 +1245,6 @@ version=str( tool_dependency_error_dict.get( 'version', '' ) ), error_message=tool_dependency_error_dict.get( 'error_message', '' ) ) tool_dependencies_folder.tool_dependency_installation_errors.append( tool_dependency_installation_error ) - installation_error_base_folder.folders.append( tool_dependencies_folder ) else: tool_test_results_root_folder = None return folder_id, tool_test_results_root_folder diff -r 8d62ee337534d00d8a3d74150d3ed71d5a1f9b40 -r 13f8c99da1cd462841b268b5beaf2148086bd19d templates/webapps/tool_shed/repository/common.mako --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -688,14 +688,13 @@ from galaxy.util import unicodify encoded_id = trans.security.encode_id( installation_error.id ) %> - ${render_table_wrap_style( "td_install_error_table" )} <tr class="datasetRow" %if parent is not None: parent="${parent}" %endif id="libraryItem-rtdie-${encoded_id}"><td style="padding-left: ${pad+20}px;"> - <table id="td_install_error_table"> + <table id="test_environment"><tr bgcolor="#FFFFCC"><th>Type</th><th>Name</th><th>Version</th></tr> @@ -720,14 +719,13 @@ from galaxy.util import unicodify encoded_id = trans.security.encode_id( installation_error.id ) %> - ${render_table_wrap_style( "rd_install_error_table" )} <tr class="datasetRow" %if parent is not None: parent="${parent}" %endif id="libraryItem-rrie-${encoded_id}"><td style="padding-left: ${pad+20}px;"> - <table id="rd_install_error_table"> + <table id="test_environment"> %if not is_current_repository: <tr bgcolor="#FFFFCC"><th>Tool shed</th><th>Name</th><th>Owner</th><th>Changeset revision</th> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: More fixes for rendering tests that resulted in installation errors within the tool test results container in the tool shed.
by commits-noreply＠bitbucket.org 04 Dec '13

04 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8d62ee337534/ Changeset: 8d62ee337534 User: greg Date: 2013-12-04 15:49:17 Summary: More fixes for rendering tests that resulted in installation errors within the tool test results container in the tool shed. Affected #: 2 files diff -r 6bbd8f13365063ee960969dea82c2b1ed2a6d6ae -r 8d62ee337534d00d8a3d74150d3ed71d5a1f9b40 lib/tool_shed/util/container_util.py --- a/lib/tool_shed/util/container_util.py +++ b/lib/tool_shed/util/container_util.py @@ -28,7 +28,6 @@ self.invalid_repository_dependencies = [] self.invalid_tool_dependencies = [] self.invalid_tools = [] - self.installation_errors = [] self.current_repository_installation_errors = [] self.repository_installation_errors = [] self.tool_dependency_installation_errors = [] @@ -1117,7 +1116,7 @@ tool_shed_revision=tool_shed_revision ) test_environment_folder.test_environments.append( test_environment ) not_tested_dict = tool_test_results_dict.get( 'not_tested', {} ) - if not_tested_dict: + if len( not_tested_dict ) > 0: folder_id += 1 not_tested_folder = Folder( id=folder_id, key='not_tested', label='Not tested', parent=containing_folder ) containing_folder.folders.append( not_tested_folder ) @@ -1126,7 +1125,7 @@ reason=not_tested_dict.get( 'reason', '' ) ) not_tested_folder.not_tested.append( not_tested ) passed_tests_dicts = tool_test_results_dict.get( 'passed_tests', [] ) - if passed_tests_dicts: + if len( passed_tests_dicts ) > 0: folder_id += 1 passed_tests_folder = Folder( id=folder_id, key='passed_tests', label='Tests that passed successfully', parent=containing_folder ) containing_folder.folders.append( passed_tests_folder ) @@ -1139,7 +1138,7 @@ tool_version=passed_tests_dict.get( 'tool_version', '' ) ) passed_tests_folder.passed_tests.append( passed_test ) failed_tests_dicts = tool_test_results_dict.get( 'failed_tests', [] ) - if failed_tests_dicts: + if len( failed_tests_dicts ) > 0: folder_id += 1 failed_tests_folder = Folder( id=folder_id, key='failed_tests', label='Tests that failed', parent=containing_folder ) containing_folder.folders.append( failed_tests_folder ) @@ -1174,28 +1173,33 @@ tool_version=missing_test_components_dict.get( 'tool_version', '' ) ) missing_test_components_folder.missing_test_components.append( missing_test_component ) installation_error_dict = tool_test_results_dict.get( 'installation_errors', {} ) - if installation_error_dict: - # 'installation_errors': {'current_repository': [], 'repository_dependencies': [], 'tool_dependencies': []}}, - current_repository_installation_errors = installation_error_dict.get( 'current_repository', [] ) - repository_dependency_installation_errors = installation_error_dict.get( 'repository_dependencies', [] ) - tool_dependency_installation_errors = installation_error_dict.get( 'tool_dependencies', [] ) - if len( current_repository_installation_errors ) > 0 or \ - len( repository_dependency_installation_errors ) > 0 or \ - len( tool_dependency_installation_errors ) > 0: + if len( installation_error_dict ) > 0: + # 'installation_errors': + # {'current_repository': [], + # 'repository_dependencies': [], + # 'tool_dependencies': + # [{'error_message': 'some traceback string' 'type': 'package', 'name': 'MIRA', 'version': '4.0'}] + # } + current_repository_installation_error_dicts = installation_error_dict.get( 'current_repository', [] ) + repository_dependency_installation_error_dicts = installation_error_dict.get( 'repository_dependencies', [] ) + tool_dependency_installation_error_dicts = installation_error_dict.get( 'tool_dependencies', [] ) + if len( current_repository_installation_error_dicts ) > 0 or \ + len( repository_dependency_installation_error_dicts ) > 0 or \ + len( tool_dependency_installation_error_dicts ) > 0: repository_installation_error_id = 0 folder_id += 1 installation_error_base_folder = Folder( id=folder_id, key='installation_errors', label='Installation errors', parent=containing_folder ) - containing_folder.installation_errors.append( installation_error_base_folder ) - if len( current_repository_installation_errors ) > 0: + containing_folder.folders.append( installation_error_base_folder ) + if len( current_repository_installation_error_dicts ) > 0: folder_id += 1 current_repository_folder = Folder( id=folder_id, key='current_repository_installation_errors', label='This repository', parent=installation_error_base_folder ) - for current_repository_error_dict in current_repository_installation_errors: + for current_repository_error_dict in current_repository_installation_error_dicts: repository_installation_error_id += 1 repository_installation_error = \ RepositoryInstallationError( id=repository_installation_error_id, @@ -1206,13 +1210,13 @@ error_message=current_repository_error_dict.get( 'error_message', '' ) ) current_repository_folder.current_repository_installation_errors.append( repository_installation_error ) installation_error_base_folder.folders.append( current_repository_folder ) - if len( repository_dependency_installation_errors ) > 0: + if len( repository_dependency_installation_error_dicts ) > 0: folder_id += 1 repository_dependencies_folder = Folder( id=folder_id, key='repository_dependency_installation_errors', label='Repository dependencies', parent=installation_error_base_folder ) - for repository_dependency_error_dict in repository_dependency_installation_errors: + for repository_dependency_error_dict in repository_dependency_installation_error_dicts: repository_installation_error_id += 1 repository_installation_error = \ RepositoryInstallationError( id=repository_installation_error_id, @@ -1223,14 +1227,15 @@ error_message=repository_dependency_error_dict.get( 'error_message', '' ) ) repository_dependencies_folder.repository_installation_errors.append( repository_installation_error ) installation_error_base_folder.folders.append( repository_dependencies_folder ) - if len( tool_dependency_installation_errors ) > 0: + if len( tool_dependency_installation_error_dicts ) > 0: + # [{'error_message': 'some traceback string' 'type': 'package', 'name': 'MIRA', 'version': '4.0'}] folder_id += 1 tool_dependencies_folder = Folder( id=folder_id, key='tool_dependency_installation_errors', label='Tool dependencies', parent=installation_error_base_folder ) tool_dependency_error_id = 0 - for tool_dependency_error_dict in tool_dependency_installation_errors: + for tool_dependency_error_dict in tool_dependency_installation_error_dicts: tool_dependency_error_id += 1 tool_dependency_installation_error = \ ToolDependencyInstallationError( id=tool_dependency_error_id, diff -r 6bbd8f13365063ee960969dea82c2b1ed2a6d6ae -r 8d62ee337534d00d8a3d74150d3ed71d5a1f9b40 templates/webapps/tool_shed/repository/common.mako --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -392,11 +392,6 @@ ${render_missing_test_component( missing_test_component, pad, my_row, row_counter )} %endfor %endif - %if folder.installation_errors: - %for installation_error in folder.installation_errors: - ${render_folder( installation_error, pad, my_row, row_counter )} - %endfor - %endif %if folder.tool_dependency_installation_errors: %for tool_dependency_installation_error in folder.tool_dependency_installation_errors: ${render_tool_dependency_installation_error( tool_dependency_installation_error, pad, my_row, row_counter )} @@ -690,6 +685,7 @@ <%def name="render_tool_dependency_installation_error( installation_error, pad, parent, row_counter, row_is_header=False )"><% + from galaxy.util import unicodify encoded_id = trans.security.encode_id( installation_error.id ) %> ${render_table_wrap_style( "td_install_error_table" )} @@ -709,7 +705,7 @@ <td>${installation_error.version | h}</td></tr><tr><th>Error</th></tr> - <tr><td colspan="3">${installation_error.error_message | h}</td></tr> + <tr><td colspan="3">${unicodify( installation_error.error_message ) | h}</td></tr></table></td></tr> @@ -721,6 +717,7 @@ <%def name="render_repository_installation_error( installation_error, pad, parent, row_counter, row_is_header=False, is_current_repository=False )"><% + from galaxy.util import unicodify encoded_id = trans.security.encode_id( installation_error.id ) %> ${render_table_wrap_style( "rd_install_error_table" )} @@ -743,7 +740,7 @@ </tr> %endif <tr><th>Error</th></tr> - <tr><td colspan="4">${installation_error.error_message | h}</td></tr> + <tr><td colspan="4">${unicodify( installation_error.error_message ) | h}</td></tr></table></td></tr> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Update test 0450 to also verify the list of testable revisions returned by the repository_revisions tool shed API controller.
by commits-noreply＠bitbucket.org 04 Dec '13

04 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6bbd8f133650/ Changeset: 6bbd8f133650 User: Dave Bouvier Date: 2013-12-04 14:41:23 Summary: Update test 0450 to also verify the list of testable revisions returned by the repository_revisions tool shed API controller. Affected #: 3 files diff -r 1ef1ec8346f176e050099767679e6df9effe28e5 -r 6bbd8f13365063ee960969dea82c2b1ed2a6d6ae test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -616,7 +616,7 @@ if not os.path.exists( temp_path ): os.makedirs( temp_path ) return temp_path - + def get_datatypes_count( self ): url = '/api/datatypes?upload_only=false' self.visit_galaxy_url( url ) @@ -887,23 +887,6 @@ log.debug( 'Waiting for the installation of repository IDs: %s' % str( repository_ids ) ) self.wait_for_repository_installation( repository_ids ) - def set_form_value( self, form, kwd, field_name, field_value ): - ''' - Set the form field field_name to field_value if it exists, and return the provided dict containing that value. If - the field does not exist in the provided form, return a dict without that index. - ''' - form_id = form.attrs.get( 'id' ) - controls = [ control for control in form.controls if str( control.name ) == field_name ] - if len( controls ) > 0: - log.debug( 'Setting field %s of form %s to %s.' % ( field_name, form_id, str( field_value ) ) ) - tc.formvalue( form_id, field_name, str( field_value ) ) - kwd[ field_name ] = str( field_value ) - else: - if field_name in kwd: - log.debug( 'No field %s in form %s, discarding from return value.' % ( str( control ), str( form_id ) ) ) - del( kwd[ field_name ] ) - return kwd - def load_citable_url( self, username, repository_name, @@ -941,6 +924,17 @@ self.visit_url( url ) self.check_for_strings( strings_displayed, strings_not_displayed ) + def load_checkable_revisions( self, strings_displayed=[], strings_not_displayed=[] ): + params = urllib.urlencode( dict( do_not_test='false', + downloadable='true', + includes_tools='true', + malicious='false', + missing_test_components='false', + skip_tool_test='false' ) ) + api_url = '%s?%s' % ( '/'.join( [ self.url, 'api', 'repository_revisions' ] ), params ) + self.visit_url( api_url ) + self.check_for_strings( strings_displayed, strings_not_displayed ) + def load_display_tool_page( self, repository, tool_xml_path, changeset_revision, strings_displayed=[], strings_not_displayed=[] ): url = '/repository/display_tool?repository_id=%s&tool_config=%s&changeset_revision=%s' % \ ( self.security.encode_id( repository.id ), tool_xml_path, changeset_revision ) @@ -1104,6 +1098,23 @@ tc.submit() self.check_for_strings( post_submit_strings_displayed, post_submit_strings_not_displayed ) + def set_form_value( self, form, kwd, field_name, field_value ): + ''' + Set the form field field_name to field_value if it exists, and return the provided dict containing that value. If + the field does not exist in the provided form, return a dict without that index. + ''' + form_id = form.attrs.get( 'id' ) + controls = [ control for control in form.controls if str( control.name ) == field_name ] + if len( controls ) > 0: + log.debug( 'Setting field %s of form %s to %s.' % ( field_name, form_id, str( field_value ) ) ) + tc.formvalue( form_id, field_name, str( field_value ) ) + kwd[ field_name ] = str( field_value ) + else: + if field_name in kwd: + log.debug( 'No field %s in form %s, discarding from return value.' % ( str( control ), str( form_id ) ) ) + del( kwd[ field_name ] ) + return kwd + def set_repository_deprecated( self, repository, set_deprecated=True, strings_displayed=[], strings_not_displayed=[] ): url = '/repository/deprecate?id=%s&mark_deprecated=%s' % ( self.security.encode_id( repository.id ), set_deprecated ) self.visit_url( url ) @@ -1116,21 +1127,25 @@ self.check_for_strings( strings_displayed, strings_not_displayed ) def set_skip_tool_tsts_flag( self, repository, flag_value, reason, changeset_revision=None ): - ''' - NOTE: As with certain other methods in this test suite, the form can't easily be filled out, so we're emulating the - form's expected behavior with url parameters, in particular the checkbox behavior. - ''' if changeset_revision is None: changeset_revision = self.get_repository_tip( repository ) self.display_manage_repository_page( repository, changeset_revision=changeset_revision ) - params = 'skip_tool_tests=true&%s' % ( urllib.urlencode( dict( changeset_revision=changeset_revision, - id=self.security.encode_id( repository.id ), - skip_tool_tests=str( flag_value ), - skip_tool_tests_comment=reason, - skip_tool_tests_button='Save' ) ) ) - url = '/repository/manage_repository?%s' % params - self.visit_url( url ) - self.check_for_strings( strings_displayed=[ 'Tools in this revision', 'automated test framework' ] ) + form = tc.browser.get_form( 'skip_tool_tests' ) + assert form is not None, 'Could not find form skip_tool_tests.' + for control in form.controls: + control_name = str( control.name ) + if control_name == 'skip_tool_tests' and control.type == 'checkbox': + checkbox = control.get() + checkbox.selected = flag_value + elif control_name == 'skip_tool_tests_comment': + tc.browser.clicked( form, control ) + tc.formvalue( 'skip_tool_tests', control_name, reason ) + kwd = dict() + self.submit_form( 'skip_tool_tests', 'skip_tool_tests_button', **kwd ) + if flag_value is True: + self.check_for_strings( strings_displayed=[ 'Tools in this revision will not be tested by the automated test framework' ] ) + else: + self.check_for_strings( strings_displayed=[ 'Tools in this revision will be tested by the automated test framework' ] ) def tip_has_metadata( self, repository ): tip = self.get_repository_tip( repository ) diff -r 1ef1ec8346f176e050099767679e6df9effe28e5 -r 6bbd8f13365063ee960969dea82c2b1ed2a6d6ae test/tool_shed/functional/test_0450_skip_tool_tests.py --- a/test/tool_shed/functional/test_0450_skip_tool_tests.py +++ b/test/tool_shed/functional/test_0450_skip_tool_tests.py @@ -131,6 +131,7 @@ ''' repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.display_manage_repository_page( repository, strings_displayed=[ 'Skip reason for first revision.' ] ) + self.load_checkable_revisions( strings_not_displayed=[ self.get_repository_tip( repository ) ] ) def test_0025_upload_new_tool_version( self ): '''Upload filtering 2.2.0.''' @@ -156,6 +157,7 @@ should not be checked and the reason should be empty. ''' repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + self.load_checkable_revisions( strings_displayed=[ self.get_repository_tip( repository ) ] ) self.display_manage_repository_page( repository, strings_not_displayed=[ 'Skip reason for first revision.' ] ) def test_0045_verify_previous_revision_skip_setting( self ): @@ -166,6 +168,7 @@ ''' global first_metadata_revision repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + self.load_checkable_revisions( strings_not_displayed=[ first_metadata_revision ] ) self.display_manage_repository_page( repository, changeset_revision=first_metadata_revision, strings_displayed=[ 'Skip reason for first revision.' ] ) @@ -177,6 +180,7 @@ ''' repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.set_skip_tool_tsts_flag( repository=repository, flag_value=True, reason='Skip reason for second revision.' ) + self.load_checkable_revisions( strings_not_displayed=[ self.get_repository_tip( repository ) ] ) self.display_manage_repository_page( repository, strings_displayed=[ 'Skip reason for second revision.' ] ) def test_0060_upload_another_readme_file( self ): @@ -198,6 +202,7 @@ strings_not_displayed=[] ) self.display_manage_repository_page( repository, strings_displayed=[ 'Skip reason for second revision.' ] ) + self.load_checkable_revisions( strings_not_displayed=[ self.get_repository_tip( repository ) ] ) def test_0065_verify_skip_setting_on_first_changeset( self ): '''Verify that the first changeset only displays its own skip reason.''' @@ -232,6 +237,7 @@ self.display_manage_repository_page( repository, changeset_revision=first_metadata_revision, strings_not_displayed=[ 'Skip reason for second revision.', 'Skip reason for first revision.' ] ) + self.load_checkable_revisions( strings_displayed=[ first_metadata_revision ] ) def test_0075_unset_skip_setting_on_second_revision( self ): '''Set the second revision not to skip tests, verify setting.''' @@ -242,7 +248,5 @@ self.set_skip_tool_tsts_flag( repository=repository, flag_value=False, reason='' ) self.display_manage_repository_page( repository, strings_not_displayed=[ 'Skip reason for second revision.', 'Skip reason for first revision.' ] ) + self.load_checkable_revisions( strings_displayed=[ self.get_repository_tip( repository ) ] ) - - - \ No newline at end of file diff -r 1ef1ec8346f176e050099767679e6df9effe28e5 -r 6bbd8f13365063ee960969dea82c2b1ed2a6d6ae test/tool_shed/functional/test_1100_install_repository_with_complex_dependencies.py --- a/test/tool_shed/functional/test_1100_install_repository_with_complex_dependencies.py +++ b/test/tool_shed/functional/test_1100_install_repository_with_complex_dependencies.py @@ -6,7 +6,7 @@ bwa_base_repository_long_description = "BWA tool that depends on bwa 0.5.9, with a complex repository dependency pointing at package_bwa_0_5_9_0100" bwa_package_repository_name = 'package_bwa_0_5_9_0100' -bwa_package_repository_description = "BWA Tool" +bwa_package_repository_description = "BWA Package" bwa_package_repository_long_description = "BWA repository with a package tool dependency defined to compile and install BWA 0.5.9." category_name = 'Test 0100 Complex Repository Dependencies' @@ -31,7 +31,7 @@ admin_user_private_role = test_db_util.get_private_role( admin_user ) def test_0005_create_bwa_package_repository( self ): - '''Create and populate bwa_tool_0100.''' + '''Create and populate package_bwa_0_5_9_0100.''' global running_standalone category = self.create_category( name=category_name, description=category_description ) self.logout() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix for displaying installation errors in the Tool Shed's tool test results container.
by commits-noreply＠bitbucket.org 03 Dec '13

03 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1ef1ec8346f1/ Changeset: 1ef1ec8346f1 User: greg Date: 2013-12-03 22:15:03 Summary: Fix for displaying installation errors in the Tool Shed's tool test results container. Affected #: 1 file diff -r 73658f181e3064c0220d7538432a14caa72aeebb -r 1ef1ec8346f176e050099767679e6df9effe28e5 lib/tool_shed/util/container_util.py --- a/lib/tool_shed/util/container_util.py +++ b/lib/tool_shed/util/container_util.py @@ -1065,7 +1065,8 @@ # script check_repositories_for_functional_tests.py has run for that entry. continue # We have a dictionary that looks something like this: - # {'missing_test_components': [], + # { + # 'missing_test_components': [], # 'failed_tests': [], # 'passed_tests': # [{'tool_id': 'effectiveT3', @@ -1078,8 +1079,8 @@ # {'python_version': '2.7.4', 'tool_shed_mercurial_version': '2.2.3', 'system': 'Linux 3.8.0-30-generic', # 'tool_shed_database_version': 21, 'architecture': 'x86_64', 'galaxy_revision': '11573:a62c54ddbe2a', # 'galaxy_database_version': 117, 'time_tested': '2013-12-03 09:11:48', 'tool_shed_revision': '11556:228156daa575'}, - # 'installation_errors': - # {'current_repository': [], 'repository_dependencies': [], 'tool_dependencies': []}}, + # 'installation_errors': {'current_repository': [], 'repository_dependencies': [], 'tool_dependencies': []} + # } test_environment_dict = tool_test_results_dict.get( 'test_environment', None ) if test_environment_dict is None: # The test environment entry will exist only if the preparation script check_repositories_for_functional_tests.py @@ -1156,7 +1157,7 @@ traceback=failed_tests_dict.get( 'traceback', '' ) ) failed_tests_folder.failed_tests.append( failed_test ) missing_test_components_dicts = tool_test_results_dict.get( 'missing_test_components', [] ) - if missing_test_components_dicts: + if len( missing_test_components_dicts ) > 0: folder_id += 1 missing_test_components_folder = Folder( id=folder_id, key='missing_test_components', @@ -1172,68 +1173,70 @@ tool_id=missing_test_components_dict.get( 'tool_id', '' ), tool_version=missing_test_components_dict.get( 'tool_version', '' ) ) missing_test_components_folder.missing_test_components.append( missing_test_component ) - installation_error_dicts = tool_test_results_dict.get( 'installation_errors', {} ) - if installation_error_dicts: - current_repository_errors = installation_error_dicts.get( 'current_repository', [] ) - repository_dependency_errors = installation_error_dicts.get( 'repository_dependencies', [] ) - tool_dependency_errors = installation_error_dicts.get( 'tool_dependencies', [] ) - if current_repository_errors or repository_dependency_errors or tool_dependency_errors: + installation_error_dict = tool_test_results_dict.get( 'installation_errors', {} ) + if installation_error_dict: + # 'installation_errors': {'current_repository': [], 'repository_dependencies': [], 'tool_dependencies': []}}, + current_repository_installation_errors = installation_error_dict.get( 'current_repository', [] ) + repository_dependency_installation_errors = installation_error_dict.get( 'repository_dependencies', [] ) + tool_dependency_installation_errors = installation_error_dict.get( 'tool_dependencies', [] ) + if len( current_repository_installation_errors ) > 0 or \ + len( repository_dependency_installation_errors ) > 0 or \ + len( tool_dependency_installation_errors ) > 0: + repository_installation_error_id = 0 folder_id += 1 installation_error_base_folder = Folder( id=folder_id, key='installation_errors', label='Installation errors', parent=containing_folder ) containing_folder.installation_errors.append( installation_error_base_folder ) - if current_repository_errors: + if len( current_repository_installation_errors ) > 0: folder_id += 1 current_repository_folder = Folder( id=folder_id, - key='current_repository_errors', + key='current_repository_installation_errors', label='This repository', parent=installation_error_base_folder ) - repository_error_id = 0 - for repository_error_dict in current_repository_errors: - repository_error_id += 1 + for current_repository_error_dict in current_repository_installation_errors: + repository_installation_error_id += 1 repository_installation_error = \ - RepositoryInstallationError( id=repository_error_id, - tool_shed=repository_error_dict.get( 'tool_shed', '' ), - name=repository_error_dict.get( 'name', '' ), - owner=repository_error_dict.get( 'owner', '' ), - changeset_revision=repository_error_dict.get( 'changeset_revision', '' ), - error_message=repository_error_dict.get( 'error_message', '' ) ) + RepositoryInstallationError( id=repository_installation_error_id, + tool_shed=str( current_repository_error_dict.get( 'tool_shed', '' ) ), + name=str( current_repository_error_dict.get( 'name', '' ) ), + owner=str( current_repository_error_dict.get( 'owner', '' ) ), + changeset_revision=str( current_repository_error_dict.get( 'changeset_revision', '' ) ), + error_message=current_repository_error_dict.get( 'error_message', '' ) ) current_repository_folder.current_repository_installation_errors.append( repository_installation_error ) installation_error_base_folder.folders.append( current_repository_folder ) - if repository_dependency_errors: + if len( repository_dependency_installation_errors ) > 0: folder_id += 1 repository_dependencies_folder = Folder( id=folder_id, - key='repository_dependency_errors', + key='repository_dependency_installation_errors', label='Repository dependencies', parent=installation_error_base_folder ) - repository_error_id = 0 - for repository_error_dict in repository_dependency_errors: - repository_error_id += 1 + for repository_dependency_error_dict in repository_dependency_installation_errors: + repository_installation_error_id += 1 repository_installation_error = \ - RepositoryInstallationError( id=repository_error_id, - tool_shed=repository_error_dict.get( 'tool_shed', '' ), - name=repository_error_dict.get( 'name', '' ), - owner=repository_error_dict.get( 'owner', '' ), - changeset_revision=repository_error_dict.get( 'changeset_revision', '' ), - error_message=repository_error_dict.get( 'error_message', '' ) ) + RepositoryInstallationError( id=repository_installation_error_id, + tool_shed=str( repository_dependency_error_dict.get( 'tool_shed', '' ) ), + name=str( repository_dependency_error_dict.get( 'name', '' ) ), + owner=str( repository_dependency_error_dict.get( 'owner', '' ) ), + changeset_revision=str( repository_dependency_error_dict.get( 'changeset_revision', '' ) ), + error_message=repository_dependency_error_dict.get( 'error_message', '' ) ) repository_dependencies_folder.repository_installation_errors.append( repository_installation_error ) installation_error_base_folder.folders.append( repository_dependencies_folder ) - if tool_dependency_errors: + if len( tool_dependency_installation_errors ) > 0: folder_id += 1 tool_dependencies_folder = Folder( id=folder_id, - key='tool_dependency_errors', + key='tool_dependency_installation_errors', label='Tool dependencies', parent=installation_error_base_folder ) tool_dependency_error_id = 0 - for tool_dependency_error_dict in tool_dependency_errors: + for tool_dependency_error_dict in tool_dependency_installation_errors: tool_dependency_error_id += 1 tool_dependency_installation_error = \ ToolDependencyInstallationError( id=tool_dependency_error_id, - type=tool_dependency_error_dict.get( 'type', '' ), - name=tool_dependency_error_dict.get( 'name', '' ), - version=tool_dependency_error_dict.get( 'version', '' ), + type=str( tool_dependency_error_dict.get( 'type', '' ) ), + name=str( tool_dependency_error_dict.get( 'name', '' ) ), + version=str( tool_dependency_error_dict.get( 'version', '' ) ), error_message=tool_dependency_error_dict.get( 'error_message', '' ) ) tool_dependencies_folder.tool_dependency_installation_errors.append( tool_dependency_installation_error ) installation_error_base_folder.folders.append( tool_dependencies_folder ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix for displaying installation errors in the Tool Shed's tool test results container.
by commits-noreply＠bitbucket.org 03 Dec '13

03 Dec '13

1 0

commit/galaxy-central: greg: Fix tool shed unit test I broke with a recent commit.
by commits-noreply＠bitbucket.org 03 Dec '13

03 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/73658f181e30/ Changeset: 73658f181e30 User: greg Date: 2013-12-03 21:52:46 Summary: Fix tool shed unit test I broke with a recent commit. Affected #: 2 files diff -r 41e739778945ac5f564785b59b2a90051676d4f2 -r 73658f181e3064c0220d7538432a14caa72aeebb lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -41,8 +41,9 @@ return_code = file_append( env_entry, env_file, skip_if_contained=skip_if_contained, make_executable=make_executable ) self.return_code = self.return_code or return_code return self.return_code - - def create_or_update_env_shell_file( self, install_dir, env_var_dict ): + + @staticmethod + def create_or_update_env_shell_file( install_dir, env_var_dict ): env_var_action = env_var_dict[ 'action' ] env_var_value = env_var_dict[ 'value' ] if env_var_action in [ 'prepend_to', 'set_to', 'append_to' ]: @@ -417,7 +418,8 @@ return tool_dependency # R libraries are installed to $INSTALL_DIR (install_dir), we now set the R_LIBS path to that directory env_file_builder = EnvFileBuilder( install_dir ) - handle_action_shell_file_paths( env_file_builder, action_dict ) # Pull in R environment (runtime). + # Pull in R environment (runtime). + handle_action_shell_file_paths( env_file_builder, action_dict ) env_file_builder.append_line( name="R_LIBS", action="prepend_to", value=install_dir ) return_code = env_file_builder.return_code if return_code: diff -r 41e739778945ac5f564785b59b2a90051676d4f2 -r 73658f181e3064c0220d7538432a14caa72aeebb test/unit/tool_shed/test_td_common_util.py --- a/test/unit/tool_shed/test_td_common_util.py +++ b/test/unit/tool_shed/test_td_common_util.py @@ -2,6 +2,7 @@ from contextlib import contextmanager from galaxy.util import parse_xml_string +from tool_shed.galaxy_install.tool_dependencies import fabric_util from tool_shed.galaxy_install.tool_dependencies import td_common_util @@ -14,26 +15,26 @@ def __init__( self ): pass - -def test_create_or_update_env_shell_file( ): +def test_create_or_update_env_shell_file(): test_path = "/usr/share/R/libs" - line, path = td_common_util.create_or_update_env_shell_file( TEST_INSTALL_DIR, dict(action="append_to", name="R_LIBS", value=test_path)) + env_file_builder = fabric_util.EnvFileBuilder( test_path ) + line, path = env_file_builder.create_or_update_env_shell_file( TEST_INSTALL_DIR, dict( action="append_to", name="R_LIBS", value=test_path ) ) assert path == join( TEST_INSTALL_DIR, "env.sh" ) assert line == "R_LIBS=$R_LIBS:/usr/share/R/libs; export R_LIBS" - line, path = td_common_util.create_or_update_env_shell_file( TEST_INSTALL_DIR, dict(action="prepend_to", name="R_LIBS", value=test_path)) + line, path = env_file_builder.create_or_update_env_shell_file( TEST_INSTALL_DIR, dict( action="prepend_to", name="R_LIBS", value=test_path ) ) assert path == join( TEST_INSTALL_DIR, "env.sh" ) assert line == "R_LIBS=/usr/share/R/libs:$R_LIBS; export R_LIBS" - line, path = td_common_util.create_or_update_env_shell_file( TEST_INSTALL_DIR, dict(action="set_to", name="R_LIBS", value=test_path)) + line, path = env_file_builder.create_or_update_env_shell_file( TEST_INSTALL_DIR, dict( action="set_to", name="R_LIBS", value=test_path ) ) assert path == join( TEST_INSTALL_DIR, "env.sh" ) assert line == "R_LIBS=/usr/share/R/libs; export R_LIBS" - line, path = td_common_util.create_or_update_env_shell_file( TEST_INSTALL_DIR, dict(action="source", value=test_path)) + line, path = env_file_builder.create_or_update_env_shell_file( TEST_INSTALL_DIR, dict( action="source", value=test_path ) ) assert path == join( TEST_INSTALL_DIR, "env.sh" ) assert line == ". /usr/share/R/libs" -def test_get_env_shell_file_paths_from_setup_environment_elem( ): +def test_get_env_shell_file_paths_from_setup_environment_elem(): xml = """<action name="setup_r_environment"><repository name="package_r_3_0_1" owner="bgruening" toolshed="toolshed.g2.bx.psu.edu" changeset_revision="1234567"><package name="R" version="3.0.1" /> @@ -70,10 +71,10 @@ ## Poor man's mocking. Need to get a real mocking library as real Galaxy development ## dependnecy. @contextmanager -def __mock_common_util_method(name, mock_method): - real_method = getattr(td_common_util, name) +def __mock_common_util_method( name, mock_method ): + real_method = getattr( td_common_util, name ) try: - setattr(td_common_util, name, mock_method) + setattr( td_common_util, name, mock_method ) yield finally: - setattr(td_common_util, name, real_method) + setattr( td_common_util, name, real_method ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix tool shed unit test I broke with a recent commit.
by commits-noreply＠bitbucket.org 03 Dec '13

03 Dec '13

1 0

commit/galaxy-central: greg: Fixes for filtering repository revision to be tested by the Tool Shed's install and test framework, and a new Tool Shed API script to enable analyzing the the Tool Shed's install and test framework.
by commits-noreply＠bitbucket.org 03 Dec '13

03 Dec '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/41e739778945/ Changeset: 41e739778945 User: greg Date: 2013-12-03 21:16:14 Summary: Fixes for filtering repository revision to be tested by the Tool Shed's install and test framework, and a new Tool Shed API script to enable analyzing the the Tool Shed's install and test framework. Affected #: 3 files diff -r 5830078f639cd0b145ca3975f9c621c3d3b69cf1 -r 41e739778945ac5f564785b59b2a90051676d4f2 lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -65,7 +65,7 @@ trans.response.status = 500 return message - @web.expose_api + @web.expose_api_anonymous def index( self, trans, **kwd ): """ GET /api/repository_revisions @@ -107,9 +107,10 @@ skip_tool_test = kwd.get( 'skip_tool_test', None ) if skip_tool_test is not None: skip_tool_test = util.string_as_bool( skip_tool_test ) - if skip_tool_test: skipped_metadata_ids_subquery = select( [ trans.app.model.SkipToolTest.table.c.repository_metadata_id ] ) - if skipped_metadata_ids_subquery: + if skip_tool_test: + clause_list.append( trans.model.RepositoryMetadata.id.in_( skipped_metadata_ids_subquery ) ) + else: clause_list.append( not_( trans.model.RepositoryMetadata.id.in_( skipped_metadata_ids_subquery ) ) ) # Generate and execute the query. try: @@ -131,7 +132,7 @@ trans.response.status = 500 return message - @web.expose_api + @web.expose_api_anonymous def show( self, trans, id, **kwd ): """ GET /api/repository_revisions/{encoded_repository_metadata_id} diff -r 5830078f639cd0b145ca3975f9c621c3d3b69cf1 -r 41e739778945ac5f564785b59b2a90051676d4f2 lib/tool_shed/scripts/api/get_filtered_repository_revisions.py --- /dev/null +++ b/lib/tool_shed/scripts/api/get_filtered_repository_revisions.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python +""" +Get a list of dictionaries, each of which defines a repository revision, that is filtered by a combination of one or more +of the following. + +- do_not_test : true / false +- downloadable : true / false +- includes_tools : true / false +- malicious : true / false +- missing_test_components : true / false +- skip_tool_test : true / false +- test_install_error : true / false +- tools_functionally_correct : true / false + +Results can also be restricted to the latest downloadable revision of each repository. + +This script is useful for analyzing the Tool Shed's install and test framework. + +Here is a working example of how to use this script. +./get_filtered_repository_revisions.py --url http://testtoolshed.g2.bx.psu.edu +""" + +import argparse +import os +import sys +import urllib + +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import get +from galaxy.util.json import from_json_string +import tool_shed.util.shed_util_common as suc + +def get_api_url( base, parts=[], params=None ): + if 'api' in parts and parts.index( 'api' ) != 0: + parts.pop( parts.index( 'api' ) ) + parts.insert( 0, 'api' ) + elif 'api' not in parts: + parts.insert( 0, 'api' ) + url = suc.url_join( base, *parts ) + if params: + url += '?%s' % params + return url + +def get_latest_downloadable_changeset_revision( url, name, owner ): + error_message = '' + parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ] + params = urllib.urlencode( dict( name=name, owner=owner ) ) + api_url = get_api_url( base=url, parts=parts, params=params ) + changeset_revisions, error_message = json_from_url( api_url ) + if error_message: + return None, error_message + if changeset_revisions: + return changeset_revisions[ -1 ], error_message + else: + return suc.INITIAL_CHANGELOG_HASH, error_message + +def get_repository_dict( url, repository_dict ): + error_message = '' + parts = [ 'api', 'repositories', repository_dict[ 'repository_id' ] ] + api_url = get_api_url( base=url, parts=parts ) + extended_dict, error_message = json_from_url( api_url ) + if error_message: + return None, error_message + name = str( extended_dict[ 'name' ] ) + owner = str( extended_dict[ 'owner' ] ) + latest_changeset_revision, error_message = get_latest_downloadable_changeset_revision( url, name, owner ) + if error_message: + print error_message + extended_dict[ 'latest_revision' ] = str( latest_changeset_revision ) + return extended_dict, error_message + +def json_from_url( url ): + error_message = '' + url_handle = urllib.urlopen( url ) + url_contents = url_handle.read() + try: + parsed_json = from_json_string( url_contents ) + except Exception, e: + error_message = str( url_contents ) + return None, error_message + return parsed_json, error_message + +def string_as_bool( string ): + if str( string ).lower() in [ 'true' ]: + return True + else: + return False + +def main( options ): + base_tool_shed_url = options.tool_shed_url.rstrip( '/' ) + latest_revision_only = string_as_bool( options.latest_revision_only ) + do_not_test = str( options.do_not_test ) + downloadable = str( options.downloadable ) + includes_tools = str( options.includes_tools ) + malicious = str( options.malicious ) + missing_test_components = str( options.missing_test_components ) + skip_tool_test = str( options.skip_tool_test ) + test_install_error = str( options.test_install_error ) + tools_functionally_correct = str( options.tools_functionally_correct ) + parts=[ 'repository_revisions' ] + params = urllib.urlencode( dict( do_not_test=do_not_test, + downloadable=downloadable, + includes_tools=includes_tools, + malicious=malicious, + missing_test_components=missing_test_components, + skip_tool_test=skip_tool_test, + test_install_error=test_install_error, + tools_functionally_correct=tools_functionally_correct ) ) + api_url = get_api_url( base=base_tool_shed_url, parts=parts, params=params ) + baseline_repository_dicts, error_message = json_from_url( api_url ) + if error_message: + print error_message + repository_dicts = [] + for baseline_repository_dict in baseline_repository_dicts: + # We need to get some details from the tool shed API, such as repository name and owner, to pass on to the + # module that will generate the install methods. + repository_dict, error_message = get_repository_dict( base_tool_shed_url, baseline_repository_dict ) + if error_message: + print 'Error getting additional details from the API: ', error_message + repository_dicts.append( baseline_repository_dict ) + else: + # Don't test empty repositories. + changeset_revision = baseline_repository_dict[ 'changeset_revision' ] + if changeset_revision != suc.INITIAL_CHANGELOG_HASH: + # Merge the dictionary returned from /api/repository_revisions with the detailed repository_dict and + # append it to the list of repository_dicts to install and test. + if latest_revision_only: + latest_revision = repository_dict[ 'latest_revision' ] + if changeset_revision == latest_revision: + repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) ) + else: + repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) ) + print '\n\n', repository_dicts + print '\nThe url:\n\n', api_url, '\n\nreturned ', len( repository_dicts ), ' repository dictionaries...' + +if __name__ == '__main__': + parser = argparse.ArgumentParser( description='Get a filtered list of repository dictionaries.' ) + parser.add_argument( "-u", "--url", dest="tool_shed_url", required=True, help="Tool Shed URL" ) + parser.add_argument( "-l", "--latest_revision_only", dest="latest_revision_only", default=True, + help="Restrict results to latest downloadable revision only" ) + parser.add_argument( "-n", "--do_not_test", help="do_not_test", default=False ) + parser.add_argument( "-d", "--downloadable", help="downloadable", default=True ) + parser.add_argument( "-i", "--includes_tools", help="includes_tools", default=True ) + parser.add_argument( "-m", "--malicious", help="malicious", default=False ) + parser.add_argument( "-c", "--missing_test_components", help="missing_test_components", default=False ) + parser.add_argument( "-s", "--skip_tool_test", help="skip_tool_test", default=False ) + parser.add_argument( "-e", "--test_install_error", help="test_install_error", default=False ) + parser.add_argument( "-t", "--tools_functionally_correct", help="tools_functionally_correct", default=True ) + options = parser.parse_args() + main( options ) diff -r 5830078f639cd0b145ca3975f9c621c3d3b69cf1 -r 41e739778945ac5f564785b59b2a90051676d4f2 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -292,26 +292,22 @@ log_data.append( test_status ) return log_data -def get_api_url( base, parts=[], params=None, key=None ): +def get_api_url( base, parts=[], params=None ): if 'api' in parts and parts.index( 'api' ) != 0: parts.pop( parts.index( 'api' ) ) parts.insert( 0, 'api' ) elif 'api' not in parts: parts.insert( 0, 'api' ) url = suc.url_join( base, *parts ) - if key: - url += '?%s' % urllib.urlencode( dict( key=key ) ) - else: - url += '?%s' % urllib.urlencode( dict( key=tool_shed_api_key ) ) if params: - url += '&%s' % params + url += '?%s' % params return url def get_latest_downloadable_changeset_revision( url, name, owner ): error_message = '' - api_url_parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ] + parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ] params = urllib.urlencode( dict( name=name, owner=owner ) ) - api_url = get_api_url( url, api_url_parts, params ) + api_url = get_api_url( base=url, parts=parts, params=params ) changeset_revisions, error_message = json_from_url( api_url ) if error_message: return None, error_message @@ -333,7 +329,7 @@ def get_repository_dict( url, repository_dict ): error_message = '' parts = [ 'api', 'repositories', repository_dict[ 'repository_id' ] ] - api_url = get_api_url( base=url, parts=parts ) + api_url = get_api_url( base=url, parts=parts, params=None ) extended_dict, error_message = json_from_url( api_url ) if error_message: return None, error_message @@ -366,13 +362,14 @@ if latest_revision_only: log.debug( 'Testing is restricted to the latest downloadable revision in this test run.' ) repository_dicts = [] + parts = [ 'repository_revisions' ] params = urllib.urlencode( dict( do_not_test='false', downloadable='true', includes_tools='true', malicious='false', missing_test_components='false', skip_tool_test='false' ) ) - api_url = get_api_url( base=tool_shed_url, parts=[ 'repository_revisions' ], params=params ) + api_url = get_api_url( base=tool_shed_url, parts=parts, params=params ) baseline_repository_dicts, error_message = json_from_url( api_url ) if error_message: return None, error_message @@ -457,8 +454,8 @@ column via the Tool Shed API. """ error_message = '' - api_path = [ 'api', 'repository_revisions', encoded_repository_metadata_id ] - api_url = get_api_url( base=tool_shed_url, parts=api_path ) + parts = [ 'api', 'repository_revisions', encoded_repository_metadata_id ] + api_url = get_api_url( base=tool_shed_url, parts=parts, params=None ) repository_metadata, error_message = json_from_url( api_url ) if error_message: return None, error_message Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

galaxy-commits December 2013