- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: greg: Set a repository revision as missing test components only if test components are missing for all tools in the revision.
by commits-noreply＠bitbucket.org 28 Nov '13

28 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/9ff030d0daee/ Changeset: 9ff030d0daee User: greg Date: 2013-11-28 23:38:38 Summary: Set a repository revision as missing test components only if test components are missing for all tools in the revision. Affected #: 2 files diff -r 5aede225ebef9801748b0f2948f5d1c387368ee7 -r 9ff030d0daee11b2187d9784a8b0a57742a671c2 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -104,8 +104,7 @@ app.model.RepositoryMetadata.table.c.includes_tools == True, app.model.RepositoryMetadata.table.c.do_not_test == False, not_( app.model.RepositoryMetadata.table.c.id.in_( skip_metadata_ids ) ) ) ): - # Clear any old invalid tests for this metadata revision, since this could lead to duplication of invalid test rows, - # or tests incorrectly labeled as invalid. + # Initialize some items. missing_test_components = [] repository = repository_metadata.repository records_checked += 1 @@ -154,8 +153,7 @@ tool_version = tool_dict[ 'version' ] tool_guid = tool_dict[ 'guid' ] if verbosity >= 1: - print "# Checking tool ID '%s' in changeset revision %s of %s." % \ - ( tool_id, changeset_revision, name ) + print "# Checking tool ID '%s' in changeset revision %s of %s." % ( tool_id, changeset_revision, name ) # If there are no tests, this tool should not be tested, since the tool functional tests only report failure if the test itself fails, # not if it's missing or undefined. Filtering out those repositories at this step will reduce the number of "false negatives" the # automated functional test framework produces. @@ -282,7 +280,11 @@ print "# and it is not the latest downloadable revision." repository_metadata.do_not_test = True repository_metadata.tools_functionally_correct = False - repository_metadata.missing_test_components = True + if not testable_revision: + # Even though some tools may be missing test components, it may be possible to test other tools. Since the + # install and test framework filters out repositories marked as missing test components, we'll set it only if + # no tools can be tested. + repository_metadata.missing_test_components = True tool_test_results_dict[ 'missing_test_components' ] = missing_test_components # Store only the configured number of test runs. num_tool_test_results_saved = int( app.config.num_tool_test_results_saved ) diff -r 5aede225ebef9801748b0f2948f5d1c387368ee7 -r 9ff030d0daee11b2187d9784a8b0a57742a671c2 lib/tool_shed/util/container_util.py --- a/lib/tool_shed/util/container_util.py +++ b/lib/tool_shed/util/container_util.py @@ -1122,6 +1122,9 @@ containing_folder.folders.append( folder ) failed_test_id = 0 for failed_tests_dict in failed_tests_dicts: + # TODO: Remove this when invalid test data is eliminated. + if isinstance( failed_tests_dict, list ): + failed_tests_dict = failed_tests_dict[ 0 ] failed_test_id += 1 failed_test = FailedTest( id=failed_test_id, stderr=failed_tests_dict.get( 'stderr', '' ), Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Provide more useful details when logging the results of the prep script for the Tool Shed's install and test framework.
by commits-noreply＠bitbucket.org 28 Nov '13

28 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/5aede225ebef/ Changeset: 5aede225ebef User: greg Date: 2013-11-28 14:41:23 Summary: Provide more useful details when logging the results of the prep script for the Tool Shed's install and test framework. Affected #: 1 file diff -r 7756b29bea292228d3472a7f613f5be5adee38a2 -r 5aede225ebef9801748b0f2948f5d1c387368ee7 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -117,9 +117,8 @@ repository = repository_metadata.repository if repository.id not in checked_repository_ids: checked_repository_ids.append( repository.id ) - if verbosity >= 1: - print '# -------------------------------------------------------------------------------------------' - print '# Now checking revision %s of %s, owned by %s.' % ( changeset_revision, name, owner ) + print '# -------------------------------------------------------------------------------------------' + print '# Now checking revision %s of %s, owned by %s.' % ( changeset_revision, name, owner ) # If this changeset revision has no tools, we don't need to do anything here, the install and test script has a filter for returning # only repositories that contain tools. tool_dicts = metadata.get( 'tools', None ) @@ -142,20 +141,19 @@ has_test_data = True test_data_path = os.path.join( root, dirs[ dirs.index( 'test-data' ) ] ) break - if verbosity >= 1: - if has_test_data: - print '# Test data directory found in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) - else: - print '# Test data directory missing in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) - print '# Checking for functional tests in changeset revision %s of %s, owned by %s.' % \ - ( changeset_revision, name, owner ) + if has_test_data: + print '# Test data directory found in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) + else: + print '# Test data directory missing in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) + print '# Checking for functional tests in changeset revision %s of %s, owned by %s.' % \ + ( changeset_revision, name, owner ) # Inspect each tool_dict for defined functional tests. for tool_dict in tool_dicts: tool_count += 1 tool_id = tool_dict[ 'id' ] tool_version = tool_dict[ 'version' ] tool_guid = tool_dict[ 'guid' ] - if verbosity >= 2: + if verbosity >= 1: print "# Checking tool ID '%s' in changeset revision %s of %s." % \ ( tool_id, changeset_revision, name ) # If there are no tests, this tool should not be tested, since the tool functional tests only report failure if the test itself fails, @@ -185,12 +183,12 @@ tool_has_tests = True break if tool_has_tests: - if verbosity >= 2: + if verbosity >= 1: print "# Tool ID '%s' in changeset revision %s of %s has one or more valid functional tests defined." % \ ( tool_id, changeset_revision, name ) has_tests += 1 else: - if verbosity >= 2: + if verbosity >= 1: print '# No functional tests defined for %s.' % tool_id no_tests += 1 failure_reason = '' @@ -200,7 +198,7 @@ if tool_has_tests and has_test_data: missing_test_files = check_for_missing_test_files( defined_test_dicts, test_data_path ) if missing_test_files: - if verbosity >= 2: + if verbosity >= 1: print "# Tool ID '%s' in changeset revision %s of %s is missing one or more required test files: %s" % \ ( tool_id, changeset_revision, name, ', '.join( missing_test_files ) ) else: @@ -226,16 +224,14 @@ shutil.rmtree( work_dir ) if not missing_test_components: valid_revisions += 1 - if verbosity >= 1: - print '# All tools have functional tests in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) + print '# All tools have functional tests in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) else: invalid_revisions += 1 + print '# Some tools have problematic functional tests in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) if verbosity >= 1: - print '# Some tools have problematic functional tests in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) - if verbosity >= 2: - for invalid_test in missing_test_components: - if 'missing_components' in invalid_test: - print '# %s' % invalid_test[ 'missing_components' ] + for invalid_test in missing_test_components: + if 'missing_components' in invalid_test: + print '# %s' % invalid_test[ 'missing_components' ] if not info_only: # Get or create the list of tool_test_results dictionaries. if repository_metadata.tool_test_results is not None: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix a debug statement.
by commits-noreply＠bitbucket.org 28 Nov '13

28 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7756b29bea29/ Changeset: 7756b29bea29 User: greg Date: 2013-11-28 03:25:03 Summary: Fix a debug statement. Affected #: 1 file diff -r 849898bb73009a5be2387a03e44ff8afdeed12cc -r 7756b29bea292228d3472a7f613f5be5adee38a2 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -380,7 +380,7 @@ # module that will generate the install methods. repository_dict, error_message = get_repository_dict( galaxy_tool_shed_url, baseline_repository_dict ) if error_message: - log.debug( 'Error getting additional details about repository %s from the API: %s' % ( str( name ), error_message ) ) + log.debug( 'Error getting additional details from the API: %s' % str( error_message ) ) else: # Don't test empty repositories. changeset_revision = baseline_repository_dict[ 'changeset_revision' ] Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Handle errors attempting to get the list of repositories to test for the Tool Shed's install and test framework.
by commits-noreply＠bitbucket.org 28 Nov '13

28 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/849898bb7300/ Changeset: 849898bb7300 User: greg Date: 2013-11-28 03:20:38 Summary: Handle errors attempting to get the list of repositories to test for the Tool Shed's install and test framework. Affected #: 1 file diff -r 3c7edca24e5aa0563e7078b56469730fd2bc27a1 -r 849898bb73009a5be2387a03e44ff8afdeed12cc test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -620,130 +620,137 @@ changeset_revision = str( repository_dict[ 'changeset_revision' ] ) # Populate the tool_test_results_dict. tool_test_results_dicts, error_message = get_tool_test_results_dicts( galaxy_tool_shed_url, encoded_repository_metadata_id ) - # The preparation script ~/tool_shed/scripts/check_repositories_for_functional_tests.py will have entered - # information in the 'test_environment' and possibly the 'missing_test_components' entries of the first - # tool_test_results_dict in the list of tool_test_results_dicts. We need to be careful to not lose this - # information. - tool_test_results_dict = tool_test_results_dicts.pop( 0 ) - # See if this repository should be skipped for any reason. - skip_this_repository = False - skip_reason = None - for exclude_dict in exclude_list: - reason = exclude_dict[ 'reason' ] - exclude_repositories = exclude_dict[ 'repositories' ] - if ( name, owner, changeset_revision ) in exclude_repositories or ( name, owner, None ) in exclude_repositories: - skip_this_repository = True - skip_reason = reason - break - if skip_this_repository: - tool_test_results_dict[ 'not_tested' ] = dict( reason=skip_reason ) - params = dict( tools_functionally_correct=False, - do_not_test=False ) - # TODO: do something usefule with response_dict - response_dict = register_test_result( galaxy_tool_shed_url, tool_test_results_dicts, tool_test_results_dict, repository_dict, params ) - log.debug( "Not testing revision %s of repository %s owned by %s." % ( changeset_revision, name, owner ) ) + if error_message: + log.debug( error_message ) else: - if error_message: - log.debug( error_message ) + # The preparation script ~/tool_shed/scripts/check_repositories_for_functional_tests.py will have entered + # information in the 'test_environment' and possibly the 'missing_test_components' entries of the first + # tool_test_results_dict in the list of tool_test_results_dicts. We need to be careful to not lose this + # information. + try: + tool_test_results_dict = tool_test_results_dicts.pop( 0 ) + except Exception, e: + log.exception( "Invalid list of tool_test_results_dicts %s: %s" % ( str( tool_test_results_dicts ), str( e ) ) ) + continue + # See if this repository should be skipped for any reason. + skip_this_repository = False + skip_reason = None + for exclude_dict in exclude_list: + reason = exclude_dict[ 'reason' ] + exclude_repositories = exclude_dict[ 'repositories' ] + if ( name, owner, changeset_revision ) in exclude_repositories or ( name, owner, None ) in exclude_repositories: + skip_this_repository = True + skip_reason = reason + break + if skip_this_repository: + tool_test_results_dict[ 'not_tested' ] = dict( reason=skip_reason ) + params = dict( tools_functionally_correct=False, + do_not_test=False ) + # TODO: do something usefule with response_dict + response_dict = register_test_result( galaxy_tool_shed_url, tool_test_results_dicts, tool_test_results_dict, repository_dict, params ) + log.debug( "Not testing revision %s of repository %s owned by %s." % ( changeset_revision, name, owner ) ) else: - test_environment_dict = tool_test_results_dict.get( 'test_environment', None ) - test_environment_dict = get_test_environment( test_environment_dict ) - # Add the current time as the approximate time that this test run occurs. A similar value will also be - # set to the repository_metadata.time_last_tested column, but we also store it here because the Tool Shed - # may be configured to store multiple test run results, so each must be associated with a time stamp. - now = time.strftime( "%Y-%m-%d %H:%M:%S" ) - test_environment_dict[ 'time_tested' ] = now - test_environment_dict[ 'galaxy_database_version' ] = get_database_version( app ) - test_environment_dict[ 'galaxy_revision' ] = get_repository_current_revision( os.getcwd() ) - tool_test_results_dict[ 'test_environment' ] = test_environment_dict - tool_test_results_dict[ 'passed_tests' ] = [] - tool_test_results_dict[ 'failed_tests' ] = [] - tool_test_results_dict[ 'installation_errors' ] = dict( current_repository=[], repository_dependencies=[], tool_dependencies=[] ) - repository, error_message = install_repository( app, repository_dict ) if error_message: - tool_test_results_dict[ 'installation_errors' ][ 'current_repository' ] = error_message - # Even if the repository failed to install, execute the uninstall method, in case a dependency did succeed. - log.debug( 'Attempting to uninstall repository %s owned by %s.' % ( name, owner ) ) - try: - repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) - except Exception, e: - error_message = 'Unable to find installed repository %s owned by %s: %s.' % ( name, owner, str( e ) ) - log.exception( error_message ) - test_result = dict( tool_shed=galaxy_tool_shed_url, - name=name, - owner=owner, - changeset_revision=changeset_revision, - error_message=error_message ) - tool_test_results_dict[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) - params = dict( tools_functionally_correct=False, - test_install_error=True, - do_not_test=False ) - # TODO: do something usefule with response_dict - response_dict = register_test_result( galaxy_tool_shed_url, tool_test_results_dicts, tool_test_results_dict, repository_dict, params ) - try: - if deactivate: - # We are deactivating this repository and all of its repository dependencies. - deactivate_repository( app, repository_dict ) + log.debug( error_message ) + else: + test_environment_dict = tool_test_results_dict.get( 'test_environment', None ) + test_environment_dict = get_test_environment( test_environment_dict ) + # Add the current time as the approximate time that this test run occurs. A similar value will also be + # set to the repository_metadata.time_last_tested column, but we also store it here because the Tool Shed + # may be configured to store multiple test run results, so each must be associated with a time stamp. + now = time.strftime( "%Y-%m-%d %H:%M:%S" ) + test_environment_dict[ 'time_tested' ] = now + test_environment_dict[ 'galaxy_database_version' ] = get_database_version( app ) + test_environment_dict[ 'galaxy_revision' ] = get_repository_current_revision( os.getcwd() ) + tool_test_results_dict[ 'test_environment' ] = test_environment_dict + tool_test_results_dict[ 'passed_tests' ] = [] + tool_test_results_dict[ 'failed_tests' ] = [] + tool_test_results_dict[ 'installation_errors' ] = dict( current_repository=[], repository_dependencies=[], tool_dependencies=[] ) + repository, error_message = install_repository( app, repository_dict ) + if error_message: + tool_test_results_dict[ 'installation_errors' ][ 'current_repository' ] = error_message + # Even if the repository failed to install, execute the uninstall method, in case a dependency did succeed. + log.debug( 'Attempting to uninstall repository %s owned by %s.' % ( name, owner ) ) + try: + repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) + except Exception, e: + error_message = 'Unable to find installed repository %s owned by %s: %s.' % ( name, owner, str( e ) ) + log.exception( error_message ) + test_result = dict( tool_shed=galaxy_tool_shed_url, + name=name, + owner=owner, + changeset_revision=changeset_revision, + error_message=error_message ) + tool_test_results_dict[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) + params = dict( tools_functionally_correct=False, + test_install_error=True, + do_not_test=False ) + # TODO: do something usefule with response_dict + response_dict = register_test_result( galaxy_tool_shed_url, tool_test_results_dicts, tool_test_results_dict, repository_dict, params ) + try: + if deactivate: + # We are deactivating this repository and all of its repository dependencies. + deactivate_repository( app, repository_dict ) + else: + # We are uninstalling this repository and all of its repository dependencies. + uninstall_repository( app, repository_dict ) + except: + log.exception( 'Encountered error attempting to deactivate or uninstall %s.', str( repository_dict[ 'name' ] ) ) + results_dict[ 'repositories_failed_install' ].append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + log.debug( 'Repository %s failed to install correctly.' % str( name ) ) + else: + # Configure and run functional tests for this repository. This is equivalent to sh run_functional_tests.sh -installed + remove_install_tests() + log.debug( 'Installation of %s succeeded, running all defined functional tests.' % str( repository.name ) ) + # Generate the shed_tools_dict that specifies the location of test data contained within this repository. If the repository + # does not have a test-data directory, this will return has_test_data = False, and we will set the do_not_test flag to True, + # and the tools_functionally_correct flag to False, as well as updating tool_test_results. + file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( {} ) ) + has_test_data, shed_tools_dict = parse_tool_panel_config( galaxy_shed_tool_conf_file, + from_json_string( file( galaxy_shed_tools_dict, 'r' ).read() ) ) + # Add an empty 'missing_test_results' entry if it is missing from the tool_test_results_dict. The + # ~/tool_shed/scripts/check_repositories_for_functional_tests.py will have entered information in the + # 'missing_test_components' entry of the tool_test_results_dict dictionary for repositories that are + # missing test components. + if 'missing_test_components' not in tool_test_results_dict: + tool_test_results_dict[ 'missing_test_components' ] = [] + missing_tool_dependencies = get_missing_tool_dependencies( repository ) + if missing_tool_dependencies or repository.missing_repository_dependencies: + results_dict = handle_missing_dependencies( app, + repository, + missing_tool_dependencies, + repository_dict, + tool_test_results_dicts, + tool_test_results_dict, + results_dict ) else: - # We are uninstalling this repository and all of its repository dependencies. - uninstall_repository( app, repository_dict ) - except: - log.exception( 'Encountered error attempting to deactivate or uninstall %s.', str( repository_dict[ 'name' ] ) ) - results_dict[ 'repositories_failed_install' ].append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) - log.debug( 'Repository %s failed to install correctly.' % str( name ) ) - else: - # Configure and run functional tests for this repository. This is equivalent to sh run_functional_tests.sh -installed - remove_install_tests() - log.debug( 'Installation of %s succeeded, running all defined functional tests.' % str( repository.name ) ) - # Generate the shed_tools_dict that specifies the location of test data contained within this repository. If the repository - # does not have a test-data directory, this will return has_test_data = False, and we will set the do_not_test flag to True, - # and the tools_functionally_correct flag to False, as well as updating tool_test_results. - file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( {} ) ) - has_test_data, shed_tools_dict = parse_tool_panel_config( galaxy_shed_tool_conf_file, - from_json_string( file( galaxy_shed_tools_dict, 'r' ).read() ) ) - # Add an empty 'missing_test_results' entry if it is missing from the tool_test_results_dict. The - # ~/tool_shed/scripts/check_repositories_for_functional_tests.py will have entered information in the - # 'missing_test_components' entry of the tool_test_results_dict dictionary for repositories that are - # missing test components. - if 'missing_test_components' not in tool_test_results_dict: - tool_test_results_dict[ 'missing_test_components' ] = [] - missing_tool_dependencies = get_missing_tool_dependencies( repository ) - if missing_tool_dependencies or repository.missing_repository_dependencies: - results_dict = handle_missing_dependencies( app, - repository, - missing_tool_dependencies, - repository_dict, - tool_test_results_dicts, - tool_test_results_dict, - results_dict ) - else: - # If the repository has a test-data directory we write the generated shed_tools_dict to a file, so the functional - # test framework can find it. - file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( shed_tools_dict ) ) - log.debug( 'Saved generated shed_tools_dict to %s\nContents: %s' % ( str( galaxy_shed_tools_dict ), str( shed_tools_dict ) ) ) - try: - results_dict = test_repository_tools( app, - repository, - repository_dict, - tool_test_results_dicts, - tool_test_results_dict, - results_dict ) - except Exception, e: - exception_message = 'Error executing tests for repository %s: %s' % ( name, str( e ) ) - log.exception( exception_message ) - tool_test_results_dict[ 'failed_tests' ].append( exception_message ) - # Record the status of this repository in the tool shed. - params = dict( tools_functionally_correct=False, - do_not_test=False, - test_install_error=False ) - # TODO: do something usefule with response_dict - response_dict = register_test_result( galaxy_tool_shed_url, - tool_test_results_dicts, - tool_test_results_dict, - repository_dict, - params ) - results_dict[ 'repositories_failed' ].append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) - total_repositories_tested += 1 + # If the repository has a test-data directory we write the generated shed_tools_dict to a file, so the functional + # test framework can find it. + file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( shed_tools_dict ) ) + log.debug( 'Saved generated shed_tools_dict to %s\nContents: %s' % ( str( galaxy_shed_tools_dict ), str( shed_tools_dict ) ) ) + try: + results_dict = test_repository_tools( app, + repository, + repository_dict, + tool_test_results_dicts, + tool_test_results_dict, + results_dict ) + except Exception, e: + exception_message = 'Error executing tests for repository %s: %s' % ( name, str( e ) ) + log.exception( exception_message ) + tool_test_results_dict[ 'failed_tests' ].append( exception_message ) + # Record the status of this repository in the tool shed. + params = dict( tools_functionally_correct=False, + do_not_test=False, + test_install_error=False ) + # TODO: do something usefule with response_dict + response_dict = register_test_result( galaxy_tool_shed_url, + tool_test_results_dicts, + tool_test_results_dict, + repository_dict, + params ) + results_dict[ 'repositories_failed' ].append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + total_repositories_tested += 1 results_dict[ 'total_repositories_tested' ] = total_repositories_tested return results_dict, error_message Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Moinor fixes and adjustments to the tool shed's install and test framework.
by commits-noreply＠bitbucket.org 28 Nov '13

28 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3c7edca24e5a/ Changeset: 3c7edca24e5a User: greg Date: 2013-11-28 03:10:52 Summary: Moinor fixes and adjustments to the tool shed's install and test framework. Affected #: 3 files diff -r 32205f911e7442fb6eed7155f8306df7aafb731e -r 3c7edca24e5aa0563e7078b56469730fd2bc27a1 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -259,6 +259,11 @@ # Create a new dictionary for the most recent test run. tool_test_results_dict = {} test_environment_dict = tool_test_results_dict.get( 'test_environment', {} ) + # Add the current time as the approximate time that this test run occurs. A similar value will also be + # set to the repository_metadata.time_last_tested column, but we also store it here because the Tool Shed + # may be configured to store multiple test run results, so each must be associated with a time stamp. + now = time.strftime( "%Y-%m-%d %H:%M:%S" ) + test_environment_dict[ 'time_tested' ] = now test_environment_dict[ 'tool_shed_database_version' ] = get_database_version( app ) test_environment_dict[ 'tool_shed_mercurial_version' ] = __version__.version test_environment_dict[ 'tool_shed_revision' ] = get_repository_current_revision( os.getcwd() ) diff -r 32205f911e7442fb6eed7155f8306df7aafb731e -r 3c7edca24e5aa0563e7078b56469730fd2bc27a1 lib/tool_shed/util/container_util.py --- a/lib/tool_shed/util/container_util.py +++ b/lib/tool_shed/util/container_util.py @@ -1052,17 +1052,22 @@ """Return a folder hierarchy containing tool dependencies.""" # This container is displayed only in the tool shed. if tool_test_results_dicts: - multiple_tool_test_results_dicts = len( tool_test_results_dicts ) > 1 test_results_dict_id = 0 folder_id += 1 tool_test_results_root_folder = Folder( id=folder_id, key='root', label='root', parent=None ) + multiple_tool_test_results_dicts = len( tool_test_results_dicts ) > 1 + if multiple_tool_test_results_dicts: + folder_id += 1 + test_runs_folder = Folder( id=folder_id, key='test_runs', label='Test runs', parent=tool_test_results_root_folder ) + tool_test_results_root_folder.folders.append( test_runs_folder ) for index, tool_test_results_dict in enumerate( tool_test_results_dicts ): test_environment_dict = tool_test_results_dict.get( 'test_environment', None ) if test_environment_dict is not None: time_tested = test_environment_dict.get( 'time_tested', 'unknown_%d' % index ) if multiple_tool_test_results_dicts: folder_id += 1 - containing_folder = Folder( id=folder_id, key='test_results', label=time_tested, parent=tool_test_results_root_folder ) + containing_folder = Folder( id=folder_id, key='test_results', label=time_tested, parent=test_runs_folder ) + test_runs_folder.folders.append( containing_folder ) else: containing_folder = tool_test_results_root_folder test_results_dict_id += 1 @@ -1150,6 +1155,7 @@ key='installation_errors', label='Installation errors', parent=containing_folder ) + containing_folder.installation_errors.append( installation_error_base_folder ) if current_repository_errors: folder_id += 1 subfolder = Folder( id=folder_id, @@ -1200,9 +1206,6 @@ error_message=tool_dependency_error_dict.get( 'error_message', '' ) ) subfolder.tool_dependency_installation_errors.append( tool_dependency_installation_error ) installation_error_base_folder.folders.append( subfolder ) - containing_folder.installation_errors.append( installation_error_base_folder ) - if multiple_tool_test_results_dicts: - tool_test_results_root_folder.folders.append( containing_folder ) else: tool_test_results_root_folder = None return folder_id, tool_test_results_root_folder diff -r 32205f911e7442fb6eed7155f8306df7aafb731e -r 3c7edca24e5aa0563e7078b56469730fd2bc27a1 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -1206,7 +1206,8 @@ response_dict = register_test_result( galaxy_tool_shed_url, tool_test_results_dicts, tool_test_results_dict, repository_dict, params ) log.debug( 'Revision %s of repository %s installed and passed functional tests.' % ( str( changeset_revision ), str( name ) ) ) else: - tool_test_results_dict[ 'failed_tests' ].append( extract_log_data( result, from_tool_test=True ) ) + # The extract_log_data() netod returns a list. + tool_test_results_dict[ 'failed_tests' ] = extract_log_data( result, from_tool_test=True ) results_dict[ 'repositories_failed' ].append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_dict ) params = dict( tools_functionally_correct=False, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: A bit of code cleanup.
by commits-noreply＠bitbucket.org 27 Nov '13

27 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/32205f911e74/ Changeset: 32205f911e74 User: greg Date: 2013-11-27 21:53:29 Summary: A bit of code cleanup. Affected #: 1 file diff -r 8b613225f1def637af4774b860578a77a7fb0fe4 -r 32205f911e7442fb6eed7155f8306df7aafb731e lib/tool_shed/util/container_util.py --- a/lib/tool_shed/util/container_util.py +++ b/lib/tool_shed/util/container_util.py @@ -1066,9 +1066,6 @@ else: containing_folder = tool_test_results_root_folder test_results_dict_id += 1 - #folder_id += 1 - #test_results_folder = Folder( id=folder_id, key='test_results', label='Automated test environment', parent=containing_folder ) - #containing_folder.folders.append( test_results_folder ) folder_id += 1 folder = Folder( id=folder_id, key='test_environment', label='Automated test environment', parent=containing_folder ) containing_folder.folders.append( folder ) @@ -1204,7 +1201,6 @@ subfolder.tool_dependency_installation_errors.append( tool_dependency_installation_error ) installation_error_base_folder.folders.append( subfolder ) containing_folder.installation_errors.append( installation_error_base_folder ) - #containing_folder.folders.append( containing_folder ) if multiple_tool_test_results_dicts: tool_test_results_root_folder.folders.append( containing_folder ) else: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Fix imports.
by commits-noreply＠bitbucket.org 27 Nov '13

27 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8b613225f1de/ Changeset: 8b613225f1de User: Dave Bouvier Date: 2013-11-27 21:37:02 Summary: Fix imports. Affected #: 1 file diff -r 11028b31d3f024ef920c3168d3f752bc13ba3ad8 -r 8b613225f1def637af4774b860578a77a7fb0fe4 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -15,6 +15,7 @@ import ConfigParser import galaxy.webapps.tool_shed.config as tool_shed_config import galaxy.webapps.tool_shed.model.mapping +import tool_shed.util.shed_util_common as suc import logging import shutil import tempfile @@ -31,8 +32,6 @@ from mercurial import __version__ from optparse import OptionParser from time import strftime -from tool_shed.util.shed_util_common import clone_repository -from tool_shed.util.shed_util_common import get_configured_ui log = logging.getLogger( 'check_repositories_for_functional_tests' ) assert sys.version_info[ :2 ] >= ( 2, 6 ) @@ -129,9 +128,9 @@ testable_revision = False # Clone the repository up to the changeset revision we're checking. repo_dir = repository.repo_path( app ) - repo = hg.repository( get_configured_ui(), repo_dir ) + repo = hg.repository( suc.get_configured_ui(), repo_dir ) work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-cafr" ) - cloned_ok, error_message = clone_repository( repo_dir, work_dir, changeset_revision ) + cloned_ok, error_message = suc.clone_repository( repo_dir, work_dir, changeset_revision ) if cloned_ok: # Iterate through all the directories in the cloned changeset revision and determine whether there's a # directory named test-data. If this directory is not present update the metadata record for the changeset Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fixes for the tool shed's install and test framework and enhancements to allow a configurable number of test runs to be stored and displayed.
by commits-noreply＠bitbucket.org 27 Nov '13

27 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/11028b31d3f0/ Changeset: 11028b31d3f0 User: greg Date: 2013-11-27 21:28:03 Summary: Fixes for the tool shed's install and test framework and enhancements to allow a configurable number of test runs to be stored and displayed. Affected #: 10 files diff -r ad38b77e96fafc96060f9ee8031094f4a15df278 -r 11028b31d3f024ef920c3168d3f752bc13ba3ad8 lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -453,5 +453,7 @@ value_mapper = { 'id' : trans.security.encode_id, 'repository_id' : trans.security.encode_id } if repository_metadata.time_last_tested is not None: - value_mapper[ 'time_last_tested' ] = time_ago + # For some reason the Dictifiable.to_dict() method in ~/galaxy/model/item_attrs.py requires + # a function rather than a mapped value, so just pass the time_ago function here. + value_mapper[ 'time_last_tested' ] = time_ago return value_mapper diff -r ad38b77e96fafc96060f9ee8031094f4a15df278 -r 11028b31d3f024ef920c3168d3f752bc13ba3ad8 lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -166,12 +166,11 @@ repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) flush_needed = False for key, new_value in payload.items(): - if hasattr( repository_metadata, key ): + if key == 'time_last_tested': + repository_metadata.time_last_tested = datetime.datetime.utcnow() + flush_needed = True + elif hasattr( repository_metadata, key ): setattr( repository_metadata, key, new_value ) - if key in [ 'tools_functionally_correct', 'time_last_tested' ]: - # Automatically update repository_metadata.time_last_tested. - repository_metadata.time_last_tested = datetime.datetime.utcnow() - flush_needed = True flush_needed = True if flush_needed: trans.sa_session.add( repository_metadata ) @@ -192,5 +191,7 @@ value_mapper = { 'id' : trans.security.encode_id, 'repository_id' : trans.security.encode_id } if repository_metadata.time_last_tested is not None: + # For some reason the Dictifiable.to_dict() method in ~/galaxy/model/item_attrs.py requires + # a function rather than a mapped value, so just pass the time_ago function here. value_mapper[ 'time_last_tested' ] = time_ago return value_mapper diff -r ad38b77e96fafc96060f9ee8031094f4a15df278 -r 11028b31d3f024ef920c3168d3f752bc13ba3ad8 lib/galaxy/webapps/tool_shed/config.py --- a/lib/galaxy/webapps/tool_shed/config.py +++ b/lib/galaxy/webapps/tool_shed/config.py @@ -53,6 +53,8 @@ self.tool_data_table_config_path = resolve_path( kwargs.get( 'tool_data_table_config_path', 'tool_data_table_conf.xml' ), self.root ) self.shed_tool_data_table_config = resolve_path( kwargs.get( 'shed_tool_data_table_config', 'shed_tool_data_table_conf.xml' ), self.root ) self.ftp_upload_dir = kwargs.get( 'ftp_upload_dir', None ) + # Install and test framework for testing tools contained in repositories. + self.num_tool_test_results_saved = kwargs.get( 'num_tool_test_results_saved', 5 ) # Location for dependencies if 'tool_dependency_dir' in kwargs: self.tool_dependency_dir = resolve_path( kwargs.get( "tool_dependency_dir" ), self.root ) diff -r ad38b77e96fafc96060f9ee8031094f4a15df278 -r 11028b31d3f024ef920c3168d3f752bc13ba3ad8 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1515,51 +1515,59 @@ tool_shed_url = web.url_for( '/', qualified=True ) functional_test_results = [] - for metadata_row in trans.sa_session.query( trans.model.RepositoryMetadata ) \ + for repository_metadata in trans.sa_session.query( trans.model.RepositoryMetadata ) \ .filter( metadata_filter ) \ .join( trans.model.Repository ) \ .filter( and_( trans.model.Repository.table.c.deleted == False, trans.model.Repository.table.c.private == False, trans.model.Repository.table.c.deprecated == False, trans.model.Repository.table.c.user_id == user.id ) ): - if not metadata_row.tool_test_results: - continue - if metadata_row.changeset_revision != metadata_row.repository.tip( trans.app ): - continue - current_repository_errors = [] - tool_dependency_errors = [] - repository_dependency_errors = [] - description_lines = [] - # Per the RSS 2.0 specification, all dates in RSS feeds must be formatted as specified in RFC 822 - # section 5.1, e.g. Sat, 07 Sep 2002 00:00:01 UT - time_tested = metadata_row.time_last_tested.strftime( '%a, %d %b %Y %H:%M:%S UT' ) - repository = metadata_row.repository - # Generate a citable URL for this repository with owner and changeset revision. - repository_citable_url = suc.url_join( tool_shed_url, 'view', user.username, repository.name, metadata_row.changeset_revision ) - passed_tests = len( metadata_row.tool_test_results.get( 'passed_tests', [] ) ) - failed_tests = len( metadata_row.tool_test_results.get( 'failed_tests', [] ) ) - missing_test_components = len( metadata_row.tool_test_results.get( 'missing_test_components', [] ) ) - installation_errors = metadata_row.tool_test_results.get( 'installation_errors', [] ) - if installation_errors: - tool_dependency_errors = installation_errors.get( 'tool_dependencies', [] ) - repository_dependency_errors = installation_errors.get( 'repository_dependencies', [] ) - current_repository_errors = installation_errors.get( 'current_repository', [] ) - description_lines.append( '%d tests passed, %d tests failed, %d tests missing test components.' % \ - ( passed_tests, failed_tests, missing_test_components ) ) - if current_repository_errors: - description_lines.append( '\nThis repository did not install correctly. ' ) - if tool_dependency_errors or repository_dependency_errors: - description_lines.append( '\n%d tool dependencies and %d repository dependencies failed to install. ' % \ - ( len( tool_dependency_errors ), len( repository_dependency_errors ) ) ) - title = 'Revision %s of %s' % ( metadata_row.changeset_revision, repository.name ) - # The guid attribute in an RSS feed's list of items allows a feed reader to choose not to show an item as updated - # if the guid is unchanged. For functional test results, the citable URL is sufficiently unique to enable - # that behavior. - functional_test_results.append( dict( title=title, - guid=repository_citable_url, - link=repository_citable_url, - description='\n'.join( description_lines ), - pubdate=time_tested ) ) + repository = repository_metadata.repository + repo_dir = repository.repo_path( trans.app ) + repo = hg.repository( suc.get_configured_ui(), repo_dir ) + latest_downloadable_changeset_revsion = suc.get_latest_downloadable_changeset_revision( trans, repository, repo ) + if repository_metadata.changeset_revision == latest_downloadable_changeset_revsion: + # We'll display only the test run for the latest installable revision in the rss feed. + tool_test_results = repository_metadata.tool_test_results + if tool_test_results is not None: + # The tool_test_results column used to contain a single dictionary, but was recently enhanced to contain + # a list of dictionaries, one for each install and test run. We'll display only the latest run in the rss + # feed for nwo. + if isinstance( tool_test_results, list ): + tool_test_results = tool_test_results[ 0 ] + current_repository_errors = [] + tool_dependency_errors = [] + repository_dependency_errors = [] + description_lines = [] + # Per the RSS 2.0 specification, all dates in RSS feeds must be formatted as specified in RFC 822 + # section 5.1, e.g. Sat, 07 Sep 2002 00:00:01 UT + time_tested = repository_metadata.time_last_tested.strftime( '%a, %d %b %Y %H:%M:%S UT' ) + # Generate a citable URL for this repository with owner and changeset revision. + repository_citable_url = suc.url_join( tool_shed_url, 'view', user.username, repository.name, repository_metadata.changeset_revision ) + passed_tests = len( tool_test_results.get( 'passed_tests', [] ) ) + failed_tests = len( tool_test_results.get( 'failed_tests', [] ) ) + missing_test_components = len( tool_test_results.get( 'missing_test_components', [] ) ) + installation_errors = tool_test_results.get( 'installation_errors', [] ) + if installation_errors: + tool_dependency_errors = installation_errors.get( 'tool_dependencies', [] ) + repository_dependency_errors = installation_errors.get( 'repository_dependencies', [] ) + current_repository_errors = installation_errors.get( 'current_repository', [] ) + description_lines.append( '%d tests passed, %d tests failed, %d tests missing test components.' % \ + ( passed_tests, failed_tests, missing_test_components ) ) + if current_repository_errors: + description_lines.append( '\nThis repository did not install correctly. ' ) + if tool_dependency_errors or repository_dependency_errors: + description_lines.append( '\n%d tool dependencies and %d repository dependencies failed to install. ' % \ + ( len( tool_dependency_errors ), len( repository_dependency_errors ) ) ) + title = 'Revision %s of %s' % ( repository_metadata.changeset_revision, repository.name ) + # The guid attribute in an RSS feed's list of items allows a feed reader to choose not to show an item as updated + # if the guid is unchanged. For functional test results, the citable URL is sufficiently unique to enable + # that behavior. + functional_test_results.append( dict( title=title, + guid=repository_citable_url, + link=repository_citable_url, + description='\n'.join( description_lines ), + pubdate=time_tested ) ) trans.response.set_content_type( 'application/rss+xml' ) return trans.fill_template( '/rss.mako', title='Tool functional test results', diff -r ad38b77e96fafc96060f9ee8031094f4a15df278 -r 11028b31d3f024ef920c3168d3f752bc13ba3ad8 lib/galaxy/webapps/tool_shed/model/__init__.py --- a/lib/galaxy/webapps/tool_shed/model/__init__.py +++ b/lib/galaxy/webapps/tool_shed/model/__init__.py @@ -229,11 +229,14 @@ class RepositoryMetadata( object, Dictifiable ): - dict_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'has_repository_dependencies', 'includes_datatypes', - 'includes_tools', 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows', 'time_last_tested' ) - dict_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'tools_functionally_correct', 'do_not_test', - 'test_install_error', 'time_last_tested', 'tool_test_results', 'has_repository_dependencies', 'includes_datatypes', - 'includes_tools', 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) + dict_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'missing_test_components', + 'tools_functionally_correct', 'do_not_test', 'test_install_error', 'has_repository_dependencies', + 'includes_datatypes', 'includes_tools', 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', + 'includes_workflows', 'time_last_tested' ) + dict_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'missing_test_components', + 'tools_functionally_correct', 'do_not_test', 'test_install_error', 'time_last_tested', 'tool_test_results', + 'has_repository_dependencies', 'includes_datatypes', 'includes_tools', 'includes_tool_dependencies', + 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) def __init__( self, id=None, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False, downloadable=False, missing_test_components=None, tools_functionally_correct=False, do_not_test=False, test_install_error=False, time_last_tested=None, @@ -242,8 +245,8 @@ self.id = id self.repository_id = repository_id self.changeset_revision = changeset_revision - self.metadata = metadata or dict() - self.tool_versions = tool_versions or dict() + self.metadata = metadata + self.tool_versions = tool_versions self.malicious = malicious self.downloadable = downloadable self.missing_test_components = missing_test_components @@ -251,7 +254,7 @@ self.do_not_test = do_not_test self.test_install_error = test_install_error self.time_last_tested = time_last_tested - self.tool_test_results = tool_test_results or dict() + self.tool_test_results = tool_test_results self.has_repository_dependencies = has_repository_dependencies # We don't consider the special case has_repository_dependencies_only_if_compiling_contained_td here. self.includes_datatypes = includes_datatypes diff -r ad38b77e96fafc96060f9ee8031094f4a15df278 -r 11028b31d3f024ef920c3168d3f752bc13ba3ad8 lib/tool_shed/scripts/api/common.py --- a/lib/tool_shed/scripts/api/common.py +++ b/lib/tool_shed/scripts/api/common.py @@ -158,7 +158,7 @@ print e.read( 1024 ) sys.exit( 1 ) else: - return 'Error. '+ str( e.read( 1024 ) ) + return 'Error. ' + str( e.read( 1024 ) ) if not return_formatted: return r print 'Response' diff -r ad38b77e96fafc96060f9ee8031094f4a15df278 -r 11028b31d3f024ef920c3168d3f752bc13ba3ad8 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -25,6 +25,7 @@ from galaxy.model.orm import and_ from galaxy.model.orm import not_ from galaxy.model.orm import select +from galaxy.util import listify from mercurial import hg from mercurial import ui from mercurial import __version__ @@ -41,16 +42,17 @@ """Application that enables updating repository_metadata table records in the Tool Shed.""" def __init__( self, config ): - if config.database_connection is False: - config.database_connection = "sqlite:///%s?isolation_level=IMMEDIATE" % str( config.database ) - log.debug( 'Using database connection: %s' % str( config.database_connection ) ) + self.config = config + if self.config.database_connection is False: + self.config.database_connection = "sqlite:///%s?isolation_level=IMMEDIATE" % str( config.database ) + log.debug( 'Using database connection: %s' % str( self.config.database_connection ) ) # Setup the database engine and ORM - self.model = galaxy.webapps.tool_shed.model.mapping.init( config.file_path, - config.database_connection, + self.model = galaxy.webapps.tool_shed.model.mapping.init( self.config.file_path, + self.config.database_connection, engine_options={}, create_tables=False ) self.hgweb_config_manager = self.model.hgweb_config_manager - self.hgweb_config_manager.hgweb_config_dir = config.hgweb_config_dir + self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir log.debug( 'Using hgweb.config file: %s' % str( self.hgweb_config_manager.hgweb_config ) ) @property @@ -72,19 +74,7 @@ "id": "tool_wrapper", "name": "Map with Tool Wrapper", "requirements": [], - "tests": [ - { - "inputs": [ [ "parameter", "value" ], [ "other_parameter", "other_value" ], ], - "name": "Test-1", - "outputs": [ - [ - "output_field_name", - "output_file_name.bed" - ] - ], - "required_files": [ '1.bed', '2.bed', '3.bed' ] - } - ], + "tests": [], "tool_config": "database/community_files/000/repo_1/tool_wrapper.xml", "tool_type": "default", "version": "1.2.3", @@ -120,11 +110,6 @@ missing_test_components = [] repository = repository_metadata.repository records_checked += 1 - # Create the tool_test_results_dict dictionary, using the dictionary from the previous test run if available. - if repository_metadata.tool_test_results: - tool_test_results_dict = repository_metadata.tool_test_results - else: - tool_test_results_dict = {} # Check the next repository revision. changeset_revision = str( repository_metadata.changeset_revision ) name = repository.name @@ -141,7 +126,7 @@ tool_dicts = metadata.get( 'tools', None ) if tool_dicts is not None: has_test_data = False - testable_revision_found = False + testable_revision = False # Clone the repository up to the changeset revision we're checking. repo_dir = repository.repo_path( app ) repo = hg.repository( get_configured_ui(), repo_dir ) @@ -236,7 +221,7 @@ if test_errors not in missing_test_components: missing_test_components.append( test_errors ) if tool_has_tests and has_test_files: - testable_revision_found = True + testable_revision = True # Remove the cloned repository path. This has to be done after the check for required test files, for obvious reasons. if os.path.exists( work_dir ): shutil.rmtree( work_dir ) @@ -253,6 +238,27 @@ if 'missing_components' in invalid_test: print '# %s' % invalid_test[ 'missing_components' ] if not info_only: + # Get or create the list of tool_test_results dictionaries. + if repository_metadata.tool_test_results is not None: + # We'll listify the column value in case it uses the old approach of storing the results of only a single test run. + tool_test_results_dicts = listify( repository_metadata.tool_test_results ) + else: + tool_test_results_dicts = [] + if tool_test_results_dicts: + # Inspect the tool_test_results_dict for the last test run in case it contains only a test_environment + # entry. This will occur with multiple runs of this script without running the associated + # install_and_test_tool_sed_repositories.sh script which will further populate the tool_test_results_dict. + tool_test_results_dict = tool_test_results_dicts[ 0 ] + if len( tool_test_results_dict ) <= 1: + # We can re-use the mostly empty tool_test_results_dict for this run, but we need to eliminate it from + # the list of tool_test_results_dicts since it will be re-inserted later. + tool_test_results_dict = tool_test_results_dicts.pop( 0 ) + else: + # The latest tool_test_results_dict has been populated with the results of a test run, so it cannot be used. + tool_test_results_dict = {} + else: + # Create a new dictionary for the most recent test run. + tool_test_results_dict = {} test_environment_dict = tool_test_results_dict.get( 'test_environment', {} ) test_environment_dict[ 'tool_shed_database_version' ] = get_database_version( app ) test_environment_dict[ 'tool_shed_mercurial_version' ] = __version__.version @@ -270,12 +276,24 @@ # changeset revision will be created, either of which will be automatically checked and flagged as appropriate. # In the install and test script, this behavior is slightly different, since we do want to always run functional # tests on the most recent downloadable changeset revision. - if should_set_do_not_test_flag( app, repository_metadata.repository, changeset_revision ) and not testable_revision_found: + if should_set_do_not_test_flag( app, repository, changeset_revision, testable_revision ): + print "# Setting do_not_test to True on revision %s of repository %s because it is missing test components" % \ + ( changeset_revision, name ) + print "# and it is not the latest downloadable revision." repository_metadata.do_not_test = True repository_metadata.tools_functionally_correct = False repository_metadata.missing_test_components = True tool_test_results_dict[ 'missing_test_components' ] = missing_test_components - repository_metadata.tool_test_results = tool_test_results_dict + # Store only the configured number of test runs. + num_tool_test_results_saved = int( app.config.num_tool_test_results_saved ) + if len( tool_test_results_dicts ) >= num_tool_test_results_saved: + test_results_index = num_tool_test_results_saved - 1 + new_tool_test_results_dicts = tool_test_results_dicts[ :test_results_index ] + else: + new_tool_test_results_dicts = [ d for d in tool_test_results_dicts ] + # Insert the new element into the first position in the list. + new_tool_test_results_dicts.insert( 0, tool_test_results_dict ) + repository_metadata.tool_test_results = new_tool_test_results_dicts app.sa_session.add( repository_metadata ) app.sa_session.flush() stop = time.time() @@ -313,19 +331,6 @@ changelog_tuples.append( ( ctx.rev(), str( ctx ) ) ) return changelog_tuples -def is_most_recent_downloadable_revision( app, repository, changeset_revision, downloadable_revisions ): - # Get a list of ( numeric revision, changeset hash ) tuples from the changelog. - changelog = get_repo_changelog_tuples( repository.repo_path( app ) ) - latest_downloadable_revision = None - for ctx_rev, changeset_hash in changelog: - if changeset_hash in downloadable_revisions: - # The last changeset hash in the changelog that is present in the list of downloadable revisions will always be the most - # recent downloadable revision, since the changelog tuples are ordered from earliest to most recent. - latest_downloadable_revision = changeset_hash - if latest_downloadable_revision == changeset_revision: - return True - return False - def main(): '''Script that checks repositories to see if the tools contained within them have functional tests defined.''' parser = OptionParser() @@ -345,7 +350,7 @@ config_parser.read( ini_file ) config_dict = {} for key, value in config_parser.items( "app:main" ): - config_dict[key] = value + config_dict[ key ] = value config = tool_shed_config.Configuration( **config_dict ) config_section = options.section now = strftime( "%Y-%m-%d %H:%M:%S" ) @@ -361,34 +366,27 @@ print "# Displaying extra information ( --verbosity = %d )" % options.verbosity check_and_update_repository_metadata( app, info_only=options.info_only, verbosity=options.verbosity ) -def should_set_do_not_test_flag( app, repository, changeset_revision ): - ''' - Returns True if: - a) There are multiple downloadable revisions, and the provided changeset revision is not the most recent downloadable revision. In this case, - the revision will never be updated with correct data, and re-testing it would be redundant. - b) There are one or more downloadable revisions, and the provided changeset revision is the most recent downloadable revision. In this case, if - the repository is updated with test data or functional tests, the downloadable changeset revision that was tested will either be replaced - with the new changeset revision, or a new downloadable changeset revision will be created, either of which will be automatically checked and - flagged as appropriate. In the install and test script, this behavior is slightly different, since we do want to always run functional tests - on the most recent downloadable changeset revision. - ''' - repository_revisions = app.sa_session.query( app.model.RepositoryMetadata ) \ - .filter( and_( app.model.RepositoryMetadata.table.c.downloadable == True, - app.model.RepositoryMetadata.table.c.repository_id == repository.id ) ) \ - .all() - downloadable_revisions = [ repository_metadata.changeset_revision for repository_metadata in repository_revisions ] - is_latest_revision = is_most_recent_downloadable_revision( app, repository, changeset_revision, downloadable_revisions ) - if len( downloadable_revisions ) == 1: - return True - elif len( downloadable_revisions ) > 1 and is_latest_revision: - return True - elif len( downloadable_revisions ) > 1 and not is_latest_revision: - return True - else: - return False +def should_set_do_not_test_flag( app, repository, changeset_revision, testable_revision ): + """ + The received testable_revision is True if the tool has defined tests and test files are in the repository + This method returns True if the received repository has multiple downloadable revisions and the received + changeset_revision is not the most recent downloadable revision and the received testable_revision is False. + In this case, the received changeset_revision will never be updated with correct data, and re-testing it + would be redundant. + """ + if not testable_revision: + repo_dir = repository.repo_path( app ) + repo = hg.repository( suc.get_configured_ui(), repo_dir ) + changeset_revisions = suc.get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ) + if len( changeset_revisions ) > 1: + latest_downloadable_revision = changeset_revisions[ -1 ] + if changeset_revision != latest_downloadable_revision: + return True + return False if __name__ == "__main__": - # The repository_metadata.tool_test_results json value should have the following structure: + # The repository_metadata.tool_test_results json value should have the following list structure: + # [ # { # "test_environment": # { @@ -467,6 +465,7 @@ # }, # ] # } + # ] # # Optionally, "traceback" may be included in a test_errors dict, if it is relevant. No script should overwrite anything other # than the list relevant to what it is testing. diff -r ad38b77e96fafc96060f9ee8031094f4a15df278 -r 11028b31d3f024ef920c3168d3f752bc13ba3ad8 lib/tool_shed/util/container_util.py --- a/lib/tool_shed/util/container_util.py +++ b/lib/tool_shed/util/container_util.py @@ -1,7 +1,7 @@ import logging import os import threading -from galaxy.util import asbool +import galaxy.util from galaxy.web.framework.helpers import time_ago from tool_shed.util import common_util from tool_shed.util import readme_util @@ -72,8 +72,8 @@ repository_name=name, repository_owner=owner, changeset_revision=changeset_revision, - prior_installation_required=asbool( prior_installation_required ), - only_if_compiling_contained_td=asbool( only_if_compiling_contained_td ) ) + prior_installation_required=galaxy.util.asbool( prior_installation_required ), + only_if_compiling_contained_td=galaxy.util.asbool( only_if_compiling_contained_td ) ) class DataManager( object ): @@ -497,8 +497,8 @@ repository_name=name, repository_owner=owner, changeset_revision=changeset_revision, - prior_installation_required=asbool( prior_installation_required ), - only_if_compiling_contained_td=asbool( only_if_compiling_contained_td ), + prior_installation_required=galaxy.util.asbool( prior_installation_required ), + only_if_compiling_contained_td=galaxy.util.asbool( only_if_compiling_contained_td ), error=error ) folder.invalid_repository_dependencies.append( ird ) invalid_repository_dependencies_folder.folders.append( folder ) @@ -831,16 +831,12 @@ label='Valid tools' ) containers_dict[ 'valid_tools' ] = valid_tools_root_folder # Tool test results container. - tool_test_results = repository_metadata.tool_test_results + tool_test_results = galaxy.util.listify( repository_metadata.tool_test_results ) # Only create and populate this folder if there are actual tool test results to display. if can_display_tool_test_results( tool_test_results, exclude=exclude ): - time_tested = repository_metadata.time_last_tested - if time_tested is not None: - time_tested = time_ago( time_tested ) folder_id, tool_test_results_root_folder = build_tool_test_results_folder( trans, folder_id, tool_test_results, - time_tested, label='Tool test results' ) containers_dict[ 'tool_test_results' ] = tool_test_results_root_folder # Workflows container. @@ -1052,153 +1048,165 @@ tool_dependencies_root_folder = None return folder_id, tool_dependencies_root_folder -def build_tool_test_results_folder( trans, folder_id, tool_test_results_dict, time_tested, label='Tool test results' ): +def build_tool_test_results_folder( trans, folder_id, tool_test_results_dicts, label='Tool test results' ): """Return a folder hierarchy containing tool dependencies.""" # This container is displayed only in the tool shed. - if tool_test_results_dict: + if tool_test_results_dicts: + multiple_tool_test_results_dicts = len( tool_test_results_dicts ) > 1 + test_results_dict_id = 0 folder_id += 1 tool_test_results_root_folder = Folder( id=folder_id, key='root', label='root', parent=None ) - test_environment_dict = tool_test_results_dict.get( 'test_environment', None ) - if test_environment_dict is not None: - folder_id += 1 - test_results_folder = Folder( id=folder_id, key='test_results', label=label, parent=tool_test_results_root_folder ) - tool_test_results_root_folder.folders.append( test_results_folder ) - folder_id += 1 - folder = Folder( id=folder_id, key='test_environment', label='Automated test environment', parent=test_results_folder ) - test_results_folder.folders.append( folder ) - architecture = test_environment_dict.get( 'architecture', '' ) - galaxy_database_version = test_environment_dict.get( 'galaxy_database_version', '' ) - galaxy_revision = test_environment_dict.get( 'galaxy_revision', '' ) - python_version = test_environment_dict.get( 'python_version', '' ) - system = test_environment_dict.get( 'system', '' ) - time_tested = time_tested - tool_shed_database_version = test_environment_dict.get( 'tool_shed_database_version', '' ) - tool_shed_mercurial_version = test_environment_dict.get( 'tool_shed_mercurial_version', '' ) - tool_shed_revision = test_environment_dict.get( 'tool_shed_revision', '' ) - test_environment = TestEnvironment( id=1, - architecture=architecture, - galaxy_database_version=galaxy_database_version, - galaxy_revision=galaxy_revision, - python_version=python_version, - system=system, - time_tested=time_tested, - tool_shed_database_version=tool_shed_database_version, - tool_shed_mercurial_version=tool_shed_mercurial_version, - tool_shed_revision=tool_shed_revision ) - folder.test_environments.append( test_environment ) - not_tested_dict = tool_test_results_dict.get( 'not_tested', {} ) - if not_tested_dict: - folder_id += 1 - folder = Folder( id=folder_id, key='not_tested', label='Not tested', parent=test_results_folder ) - test_results_folder.folders.append( folder ) - not_tested_id = 0 - not_tested = NotTested( id=not_tested_id, - reason=not_tested_dict.get( 'reason', '' ) ) - folder.not_tested.append( not_tested ) - passed_tests_dicts = tool_test_results_dict.get( 'passed_tests', [] ) - if passed_tests_dicts: - folder_id += 1 - folder = Folder( id=folder_id, key='passed_tests', label='Tests that passed successfully', parent=test_results_folder ) - test_results_folder.folders.append( folder ) - passed_test_id = 0 - for passed_tests_dict in passed_tests_dicts: - passed_test_id += 1 - passed_test = PassedTest( id=passed_test_id, - test_id=passed_tests_dict.get( 'test_id' '' ), - tool_id=passed_tests_dict.get( 'tool_id', '' ), - tool_version=passed_tests_dict.get( 'tool_version', '' ) ) - folder.passed_tests.append( passed_test ) - failed_tests_dicts = tool_test_results_dict.get( 'failed_tests', [] ) - if failed_tests_dicts: - folder_id += 1 - folder = Folder( id=folder_id, key='failed_tests', label='Tests that failed', parent=test_results_folder ) - test_results_folder.folders.append( folder ) - failed_test_id = 0 - for failed_tests_dict in failed_tests_dicts: - failed_test_id += 1 - failed_test = FailedTest( id=failed_test_id, - stderr=failed_tests_dict.get( 'stderr', '' ), - test_id=failed_tests_dict.get( 'test_id', '' ), - tool_id=failed_tests_dict.get( 'tool_id', '' ), - tool_version=failed_tests_dict.get( 'tool_version', '' ), - traceback=failed_tests_dict.get( 'traceback', '' ) ) - folder.failed_tests.append( failed_test ) - missing_test_components_dicts = tool_test_results_dict.get( 'missing_test_components', [] ) - if missing_test_components_dicts: - folder_id += 1 - folder = Folder( id=folder_id, key='missing_test_components', label='Tools missing tests or test data', parent=test_results_folder ) - test_results_folder.folders.append( folder ) - missing_test_component_id = 0 - for missing_test_components_dict in missing_test_components_dicts: - missing_test_component_id += 1 - missing_test_component = MissingTestComponent( id=missing_test_component_id, - missing_components=missing_test_components_dict.get( 'missing_components', '' ), - tool_guid=missing_test_components_dict.get( 'tool_guid', '' ), - tool_id=missing_test_components_dict.get( 'tool_id', '' ), - tool_version=missing_test_components_dict.get( 'tool_version', '' ) ) - folder.missing_test_components.append( missing_test_component ) - installation_error_dicts = tool_test_results_dict.get( 'installation_errors', {} ) - if installation_error_dicts: - current_repository_errors = installation_error_dicts.get( 'current_repository', [] ) - repository_dependency_errors = installation_error_dicts.get( 'repository_dependencies', [] ) - tool_dependency_errors = installation_error_dicts.get( 'tool_dependencies', [] ) - if current_repository_errors or repository_dependency_errors or tool_dependency_errors: + for index, tool_test_results_dict in enumerate( tool_test_results_dicts ): + test_environment_dict = tool_test_results_dict.get( 'test_environment', None ) + if test_environment_dict is not None: + time_tested = test_environment_dict.get( 'time_tested', 'unknown_%d' % index ) + if multiple_tool_test_results_dicts: + folder_id += 1 + containing_folder = Folder( id=folder_id, key='test_results', label=time_tested, parent=tool_test_results_root_folder ) + else: + containing_folder = tool_test_results_root_folder + test_results_dict_id += 1 + #folder_id += 1 + #test_results_folder = Folder( id=folder_id, key='test_results', label='Automated test environment', parent=containing_folder ) + #containing_folder.folders.append( test_results_folder ) folder_id += 1 - installation_error_base_folder = Folder( id=folder_id, - key='installation_errors', - label='Installation errors', - parent=test_results_folder ) - if current_repository_errors: + folder = Folder( id=folder_id, key='test_environment', label='Automated test environment', parent=containing_folder ) + containing_folder.folders.append( folder ) + architecture = test_environment_dict.get( 'architecture', '' ) + galaxy_database_version = test_environment_dict.get( 'galaxy_database_version', '' ) + galaxy_revision = test_environment_dict.get( 'galaxy_revision', '' ) + python_version = test_environment_dict.get( 'python_version', '' ) + system = test_environment_dict.get( 'system', '' ) + tool_shed_database_version = test_environment_dict.get( 'tool_shed_database_version', '' ) + tool_shed_mercurial_version = test_environment_dict.get( 'tool_shed_mercurial_version', '' ) + tool_shed_revision = test_environment_dict.get( 'tool_shed_revision', '' ) + test_environment = TestEnvironment( id=1, + architecture=architecture, + galaxy_database_version=galaxy_database_version, + galaxy_revision=galaxy_revision, + python_version=python_version, + system=system, + time_tested=time_tested, + tool_shed_database_version=tool_shed_database_version, + tool_shed_mercurial_version=tool_shed_mercurial_version, + tool_shed_revision=tool_shed_revision ) + folder.test_environments.append( test_environment ) + not_tested_dict = tool_test_results_dict.get( 'not_tested', {} ) + if not_tested_dict: + folder_id += 1 + folder = Folder( id=folder_id, key='not_tested', label='Not tested', parent=containing_folder ) + containing_folder.folders.append( folder ) + not_tested_id = 0 + not_tested = NotTested( id=not_tested_id, + reason=not_tested_dict.get( 'reason', '' ) ) + folder.not_tested.append( not_tested ) + passed_tests_dicts = tool_test_results_dict.get( 'passed_tests', [] ) + if passed_tests_dicts: + folder_id += 1 + folder = Folder( id=folder_id, key='passed_tests', label='Tests that passed successfully', parent=containing_folder ) + containing_folder.folders.append( folder ) + passed_test_id = 0 + for passed_tests_dict in passed_tests_dicts: + passed_test_id += 1 + passed_test = PassedTest( id=passed_test_id, + test_id=passed_tests_dict.get( 'test_id' '' ), + tool_id=passed_tests_dict.get( 'tool_id', '' ), + tool_version=passed_tests_dict.get( 'tool_version', '' ) ) + folder.passed_tests.append( passed_test ) + failed_tests_dicts = tool_test_results_dict.get( 'failed_tests', [] ) + if failed_tests_dicts: + folder_id += 1 + folder = Folder( id=folder_id, key='failed_tests', label='Tests that failed', parent=containing_folder ) + containing_folder.folders.append( folder ) + failed_test_id = 0 + for failed_tests_dict in failed_tests_dicts: + failed_test_id += 1 + failed_test = FailedTest( id=failed_test_id, + stderr=failed_tests_dict.get( 'stderr', '' ), + test_id=failed_tests_dict.get( 'test_id', '' ), + tool_id=failed_tests_dict.get( 'tool_id', '' ), + tool_version=failed_tests_dict.get( 'tool_version', '' ), + traceback=failed_tests_dict.get( 'traceback', '' ) ) + folder.failed_tests.append( failed_test ) + missing_test_components_dicts = tool_test_results_dict.get( 'missing_test_components', [] ) + if missing_test_components_dicts: + folder_id += 1 + folder = Folder( id=folder_id, key='missing_test_components', label='Tools missing tests or test data', parent=containing_folder ) + containing_folder.folders.append( folder ) + missing_test_component_id = 0 + for missing_test_components_dict in missing_test_components_dicts: + missing_test_component_id += 1 + missing_test_component = MissingTestComponent( id=missing_test_component_id, + missing_components=missing_test_components_dict.get( 'missing_components', '' ), + tool_guid=missing_test_components_dict.get( 'tool_guid', '' ), + tool_id=missing_test_components_dict.get( 'tool_id', '' ), + tool_version=missing_test_components_dict.get( 'tool_version', '' ) ) + folder.missing_test_components.append( missing_test_component ) + installation_error_dicts = tool_test_results_dict.get( 'installation_errors', {} ) + if installation_error_dicts: + current_repository_errors = installation_error_dicts.get( 'current_repository', [] ) + repository_dependency_errors = installation_error_dicts.get( 'repository_dependencies', [] ) + tool_dependency_errors = installation_error_dicts.get( 'tool_dependencies', [] ) + if current_repository_errors or repository_dependency_errors or tool_dependency_errors: folder_id += 1 - subfolder = Folder( id=folder_id, - key='current_repository_errors', - label='This repository', - parent=installation_error_base_folder ) - repository_error_id = 0 - for repository_error_dict in current_repository_errors: - repository_error_id += 1 - repository_installation_error = RepositoryInstallationError( id=repository_error_id, - tool_shed=repository_error_dict.get( 'tool_shed', '' ), - name=repository_error_dict.get( 'name', '' ), - owner=repository_error_dict.get( 'owner', '' ), - changeset_revision=repository_error_dict.get( 'changeset_revision', '' ), - error_message=repository_error_dict.get( 'error_message', '' ) ) - subfolder.current_repository_installation_errors.append( repository_installation_error ) - installation_error_base_folder.folders.append( subfolder ) - if repository_dependency_errors: - folder_id += 1 - subfolder = Folder( id=folder_id, - key='repository_dependency_errors', - label='Repository dependencies', - parent=installation_error_base_folder ) - repository_error_id = 0 - for repository_error_dict in repository_dependency_errors: - repository_error_id += 1 - repository_installation_error = RepositoryInstallationError( id=repository_error_id, - tool_shed=repository_error_dict.get( 'tool_shed', '' ), - name=repository_error_dict.get( 'name', '' ), - owner=repository_error_dict.get( 'owner', '' ), - changeset_revision=repository_error_dict.get( 'changeset_revision', '' ), - error_message=repository_error_dict.get( 'error_message', '' ) ) - subfolder.repository_installation_errors.append( repository_installation_error ) - installation_error_base_folder.folders.append( subfolder ) - if tool_dependency_errors: - folder_id += 1 - subfolder = Folder( id=folder_id, - key='tool_dependency_errors', - label='Tool dependencies', - parent=installation_error_base_folder ) - tool_dependency_error_id = 0 - for tool_dependency_error_dict in tool_dependency_errors: - tool_dependency_error_id += 1 - tool_dependency_installation_error = ToolDependencyInstallationError( id=tool_dependency_error_id, - type=tool_dependency_error_dict.get( 'type', '' ), - name=tool_dependency_error_dict.get( 'name', '' ), - version=tool_dependency_error_dict.get( 'version', '' ), - error_message=tool_dependency_error_dict.get( 'error_message', '' ) ) - subfolder.tool_dependency_installation_errors.append( tool_dependency_installation_error ) - installation_error_base_folder.folders.append( subfolder ) - test_results_folder.installation_errors.append( installation_error_base_folder ) + installation_error_base_folder = Folder( id=folder_id, + key='installation_errors', + label='Installation errors', + parent=containing_folder ) + if current_repository_errors: + folder_id += 1 + subfolder = Folder( id=folder_id, + key='current_repository_errors', + label='This repository', + parent=installation_error_base_folder ) + repository_error_id = 0 + for repository_error_dict in current_repository_errors: + repository_error_id += 1 + repository_installation_error = RepositoryInstallationError( id=repository_error_id, + tool_shed=repository_error_dict.get( 'tool_shed', '' ), + name=repository_error_dict.get( 'name', '' ), + owner=repository_error_dict.get( 'owner', '' ), + changeset_revision=repository_error_dict.get( 'changeset_revision', '' ), + error_message=repository_error_dict.get( 'error_message', '' ) ) + subfolder.current_repository_installation_errors.append( repository_installation_error ) + installation_error_base_folder.folders.append( subfolder ) + if repository_dependency_errors: + folder_id += 1 + subfolder = Folder( id=folder_id, + key='repository_dependency_errors', + label='Repository dependencies', + parent=installation_error_base_folder ) + repository_error_id = 0 + for repository_error_dict in repository_dependency_errors: + repository_error_id += 1 + repository_installation_error = RepositoryInstallationError( id=repository_error_id, + tool_shed=repository_error_dict.get( 'tool_shed', '' ), + name=repository_error_dict.get( 'name', '' ), + owner=repository_error_dict.get( 'owner', '' ), + changeset_revision=repository_error_dict.get( 'changeset_revision', '' ), + error_message=repository_error_dict.get( 'error_message', '' ) ) + subfolder.repository_installation_errors.append( repository_installation_error ) + installation_error_base_folder.folders.append( subfolder ) + if tool_dependency_errors: + folder_id += 1 + subfolder = Folder( id=folder_id, + key='tool_dependency_errors', + label='Tool dependencies', + parent=installation_error_base_folder ) + tool_dependency_error_id = 0 + for tool_dependency_error_dict in tool_dependency_errors: + tool_dependency_error_id += 1 + tool_dependency_installation_error = ToolDependencyInstallationError( id=tool_dependency_error_id, + type=tool_dependency_error_dict.get( 'type', '' ), + name=tool_dependency_error_dict.get( 'name', '' ), + version=tool_dependency_error_dict.get( 'version', '' ), + error_message=tool_dependency_error_dict.get( 'error_message', '' ) ) + subfolder.tool_dependency_installation_errors.append( tool_dependency_installation_error ) + installation_error_base_folder.folders.append( subfolder ) + containing_folder.installation_errors.append( installation_error_base_folder ) + #containing_folder.folders.append( containing_folder ) + if multiple_tool_test_results_dicts: + tool_test_results_root_folder.folders.append( containing_folder ) else: tool_test_results_root_folder = None return folder_id, tool_test_results_root_folder @@ -1247,13 +1255,13 @@ workflows_root_folder = None return folder_id, workflows_root_folder -def can_display_tool_test_results( tool_test_results_dict, exclude=None ): +def can_display_tool_test_results( tool_test_results_dicts, exclude=None ): # Only create and populate the tool_test_results container if there are actual tool test results to display. if exclude is None: exclude = [] if 'tool_test_results' in exclude: return False - if tool_test_results_dict: + for tool_test_results_dict in tool_test_results_dicts: # We check for more than a single entry in the tool_test_results dictionary because it may have # only the "test_environment" entry, but we want at least 1 of "passed_tests", "failed_tests", # "installation_errors", "missing_test_components" "skipped_tests", "not_tested" or any other @@ -1288,7 +1296,7 @@ if key_is_current_repositorys_key( repository_name, repository_owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, key ): label = 'Repository dependencies' else: - if asbool( prior_installation_required ): + if galaxy.util.asbool( prior_installation_required ): prior_installation_required_str = " <i>(prior install required)</i>" else: prior_installation_required_str = "" @@ -1432,8 +1440,8 @@ repository_name=repository_name, repository_owner=repository_owner, changeset_revision=changeset_revision, - prior_installation_required=asbool( prior_installation_required ), - only_if_compiling_contained_td=asbool( only_if_compiling_contained_td ), + prior_installation_required=galaxy.util.asbool( prior_installation_required ), + only_if_compiling_contained_td=galaxy.util.asbool( only_if_compiling_contained_td ), installation_status=installation_status, tool_shed_repository_id=tool_shed_repository_id ) # Insert the repository_dependency into the folder. diff -r ad38b77e96fafc96060f9ee8031094f4a15df278 -r 11028b31d3f024ef920c3168d3f752bc13ba3ad8 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -58,7 +58,6 @@ from mercurial import __version__ from nose.plugins import Plugin from paste import httpserver -from time import strftime from tool_shed.util import tool_dependency_util from tool_shed.util.xml_util import parse_xml @@ -80,6 +79,12 @@ default_galaxy_test_host = '127.0.0.1' default_galaxy_master_api_key = None +# This script can be run in such a way that no Tool Shed database records should be changed. +if '-info_only' in sys.argv or 'GALAXY_INSTALL_TEST_INFO_ONLY' in os.environ: + can_update_tool_shed = False +else: + can_update_tool_shed = True + # Should this serve static resources (scripts, images, styles, etc.)? STATIC_ENABLED = True @@ -157,7 +162,7 @@ galaxy_tool_shed_url = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_URL', None ) tool_shed_api_key = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY', None ) exclude_list_file = os.environ.get( 'GALAXY_INSTALL_TEST_EXCLUDE_REPOSITORIES', 'install_test_exclude.xml' ) - + if 'GALAXY_INSTALL_TEST_SECRET' not in os.environ: galaxy_encode_secret = 'changethisinproductiontoo' os.environ[ 'GALAXY_INSTALL_TEST_SECRET' ] = galaxy_encode_secret @@ -325,16 +330,16 @@ missing_tool_dependencies.extend( get_missing_tool_dependencies( repository_dependency ) ) return missing_tool_dependencies -def get_repository_info_from_api( url, repository_dict ): +def get_repository_dict( url, repository_dict ): error_message = '' parts = [ 'api', 'repositories', repository_dict[ 'repository_id' ] ] api_url = get_api_url( base=url, parts=parts ) extended_dict, error_message = json_from_url( api_url ) if error_message: return None, error_message - latest_changeset_revision, error_message = get_latest_downloadable_changeset_revision( url, - extended_dict[ 'name' ], - extended_dict[ 'owner' ] ) + name = str( extended_dict[ 'name' ] ) + owner = str( extended_dict[ 'owner' ] ) + latest_changeset_revision, error_message = get_latest_downloadable_changeset_revision( url, name, owner ) if error_message: return None, error_message extended_dict[ 'latest_revision' ] = str( latest_changeset_revision ) @@ -361,8 +366,9 @@ repository_dicts = [] params = urllib.urlencode( dict( do_not_test='false', downloadable='true', + includes_tools='true', malicious='false', - includes_tools='true', + missing_test_components='false', skip_tool_test='false' ) ) api_url = get_api_url( base=tool_shed_url, parts=[ 'repository_revisions' ], params=params ) baseline_repository_dicts, error_message = json_from_url( api_url ) @@ -372,7 +378,7 @@ for baseline_repository_dict in baseline_repository_dicts: # We need to get some details from the tool shed API, such as repository name and owner, to pass on to the # module that will generate the install methods. - repository_dict, error_message = get_repository_info_from_api( galaxy_tool_shed_url, baseline_repository_dict ) + repository_dict, error_message = get_repository_dict( galaxy_tool_shed_url, baseline_repository_dict ) if error_message: log.debug( 'Error getting additional details about repository %s from the API: %s' % ( str( name ), error_message ) ) else: @@ -447,16 +453,18 @@ tool_id = parts[ -2 ] return tool_id, tool_version -def get_tool_test_results_dict( tool_shed_url, encoded_repository_metadata_id ): +def get_tool_test_results_dicts( tool_shed_url, encoded_repository_metadata_id ): + """ + Return the list of dictionaries contained in the Tool Shed's repository_metadata.tool_test_results + column via the Tool Shed API. + """ error_message = '' api_path = [ 'api', 'repository_revisions', encoded_repository_metadata_id ] api_url = get_api_url( base=tool_shed_url, parts=api_path ) repository_metadata, error_message = json_from_url( api_url ) if error_message: return None, error_message - tool_test_results = repository_metadata.get( 'tool_test_results', {} ) - if tool_test_results is None: - return None, error_message + tool_test_results = repository_metadata.get( 'tool_test_results', [] ) return tool_test_results, error_message def get_webapp_global_conf(): @@ -466,7 +474,8 @@ global_conf.update( get_static_settings() ) return global_conf -def handle_missing_dependencies( app, repository, missing_tool_dependencies, repository_dict, tool_test_results_dict, results_dict ): +def handle_missing_dependencies( app, repository, missing_tool_dependencies, repository_dict, + tool_test_results_dicts, tool_test_results_dict, results_dict ): """Handle missing repository or tool dependencies for an installed repository.""" # If a tool dependency fails to install correctly, this should be considered an installation error, # and functional tests should be skipped, since the tool dependency needs to be correctly installed @@ -494,7 +503,8 @@ params = dict( tools_functionally_correct=False, do_not_test=False, test_install_error=True ) - register_test_result( galaxy_tool_shed_url, tool_test_results_dict, repository_dict, params ) + # TODO: do something usefule with response_dict + response_dict = register_test_result( galaxy_tool_shed_url, tool_test_results_dicts, tool_test_results_dict, repository_dict, params ) # Since this repository is missing components, we do not want to test it, so deactivate it or uninstall it. # The deactivate flag is set to True if the environment variable GALAXY_INSTALL_TEST_KEEP_TOOL_DEPENDENCIES # is set to 'true'. @@ -608,6 +618,13 @@ name = str( repository_dict[ 'name' ] ) owner = str( repository_dict[ 'owner' ] ) changeset_revision = str( repository_dict[ 'changeset_revision' ] ) + # Populate the tool_test_results_dict. + tool_test_results_dicts, error_message = get_tool_test_results_dicts( galaxy_tool_shed_url, encoded_repository_metadata_id ) + # The preparation script ~/tool_shed/scripts/check_repositories_for_functional_tests.py will have entered + # information in the 'test_environment' and possibly the 'missing_test_components' entries of the first + # tool_test_results_dict in the list of tool_test_results_dicts. We need to be careful to not lose this + # information. + tool_test_results_dict = tool_test_results_dicts.pop( 0 ) # See if this repository should be skipped for any reason. skip_this_repository = False skip_reason = None @@ -622,21 +639,20 @@ tool_test_results_dict[ 'not_tested' ] = dict( reason=skip_reason ) params = dict( tools_functionally_correct=False, do_not_test=False ) - register_test_result( galaxy_tool_shed_url, tool_test_results_dict, repository_dict, params ) + # TODO: do something usefule with response_dict + response_dict = register_test_result( galaxy_tool_shed_url, tool_test_results_dicts, tool_test_results_dict, repository_dict, params ) log.debug( "Not testing revision %s of repository %s owned by %s." % ( changeset_revision, name, owner ) ) else: - # Populate the tool_test_results_dict. - tool_test_results_dict, error_message = get_tool_test_results_dict( galaxy_tool_shed_url, encoded_repository_metadata_id ) if error_message: log.debug( error_message ) else: - # The preparation script ~/tool_shed/scripts/check_repositories_for_functional_tests.py will have entered - # information in the 'missing_test_components' entry of the tool_test_results_dict dictionary for repositories - # that are missing test components. We need to be careful to not lose this information. For all other repositories, - # no changes will have been made to this dictionary by the preparation script, and tool_test_results_dict will be None. - # Initialize the tool_test_results_dict dictionary with the information about the current test environment. test_environment_dict = tool_test_results_dict.get( 'test_environment', None ) test_environment_dict = get_test_environment( test_environment_dict ) + # Add the current time as the approximate time that this test run occurs. A similar value will also be + # set to the repository_metadata.time_last_tested column, but we also store it here because the Tool Shed + # may be configured to store multiple test run results, so each must be associated with a time stamp. + now = time.strftime( "%Y-%m-%d %H:%M:%S" ) + test_environment_dict[ 'time_tested' ] = now test_environment_dict[ 'galaxy_database_version' ] = get_database_version( app ) test_environment_dict[ 'galaxy_revision' ] = get_repository_current_revision( os.getcwd() ) tool_test_results_dict[ 'test_environment' ] = test_environment_dict @@ -662,8 +678,8 @@ params = dict( tools_functionally_correct=False, test_install_error=True, do_not_test=False ) - - register_test_result( galaxy_tool_shed_url, tool_test_results_dict, repository_dict, params ) + # TODO: do something usefule with response_dict + response_dict = register_test_result( galaxy_tool_shed_url, tool_test_results_dicts, tool_test_results_dict, repository_dict, params ) try: if deactivate: # We are deactivating this repository and all of its repository dependencies. @@ -697,6 +713,7 @@ repository, missing_tool_dependencies, repository_dict, + tool_test_results_dicts, tool_test_results_dict, results_dict ) else: @@ -705,7 +722,12 @@ file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( shed_tools_dict ) ) log.debug( 'Saved generated shed_tools_dict to %s\nContents: %s' % ( str( galaxy_shed_tools_dict ), str( shed_tools_dict ) ) ) try: - results_dict = test_repository_tools( app, repository, repository_dict, tool_test_results_dict, results_dict ) + results_dict = test_repository_tools( app, + repository, + repository_dict, + tool_test_results_dicts, + tool_test_results_dict, + results_dict ) except Exception, e: exception_message = 'Error executing tests for repository %s: %s' % ( name, str( e ) ) log.exception( exception_message ) @@ -714,7 +736,12 @@ params = dict( tools_functionally_correct=False, do_not_test=False, test_install_error=False ) - register_test_result( galaxy_tool_shed_url, tool_test_results_dict, repository_dict, params ) + # TODO: do something usefule with response_dict + response_dict = register_test_result( galaxy_tool_shed_url, + tool_test_results_dicts, + tool_test_results_dict, + repository_dict, + params ) results_dict[ 'repositories_failed' ].append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) total_repositories_tested += 1 results_dict[ 'total_repositories_tested' ] = total_repositories_tested @@ -937,7 +964,7 @@ tool_data_path=additional_tool_data_path, shed_tool_data_table_config=None, persist=False ) - now = strftime( "%Y-%m-%d %H:%M:%S" ) + now = time.strftime( "%Y-%m-%d %H:%M:%S" ) print "####################################################################################" print "# %s - running repository installation and testing script." % now print "####################################################################################" @@ -949,12 +976,12 @@ repositories_passed = results_dict[ 'repositories_passed' ] repositories_failed = results_dict[ 'repositories_failed' ] repositories_failed_install = results_dict[ 'repositories_failed_install' ] - now = strftime( "%Y-%m-%d %H:%M:%S" ) + now = time.strftime( "%Y-%m-%d %H:%M:%S" ) print "####################################################################################" print "# %s - repository installation and testing script completed." % now print "# Repository revisions tested: %s" % str( total_repositories_tested ) - if '-info_only' in sys.argv: - print "# -info_only set, not updating the tool shed." + if not can_update_tool_shed: + print "# This run will not update the Tool Shed database." if total_repositories_tested > 0: if repositories_passed: print '# ----------------------------------------------------------------------------------' @@ -1048,26 +1075,26 @@ log.debug( 'Repository %s owned by %s, all revisions.' % ( str( name ), str( owner ) ) ) return exclude_list -def register_test_result( url, test_results_dict, repository_dict, params ): +def register_test_result( url, tool_test_results_dicts, tool_test_results_dict, repository_dict, params ): """ Update the repository metadata tool_test_results and appropriate flags using the Tool SHed API. This method updates tool_test_results with the relevant data, sets the do_not_test and tools_functionally correct flags to the appropriate values and updates the time_last_tested field to the value of the received time_tested. """ - if '-info_only' in sys.argv or 'GALAXY_INSTALL_TEST_INFO_ONLY' in os.environ: - return {} - else: + if can_update_tool_shed: metadata_revision_id = repository_dict.get( 'id', None ) if metadata_revision_id is not None: + tool_test_results_dicts.insert( 0, tool_test_results_dict ) + params[ 'tool_test_results' ] = tool_test_results_dicts # Set the time_last_tested entry so that the repository_metadata.time_last_tested will be set in the tool shed. - time_tested = datetime.utcnow() - test_results_dict[ 'time_last_tested' ] = time_ago( time_tested ) - params[ 'tool_test_results' ] = test_results_dict + params[ 'time_last_tested' ] = 'This entry will result in this value being set via the Tool Shed API.' url = '%s' % ( suc.url_join( galaxy_tool_shed_url,'api', 'repository_revisions', str( metadata_revision_id ) ) ) try: return update( tool_shed_api_key, url, params, return_formatted=False ) except Exception, e: log.exception( 'Error attempting to register test results: %s' % str( e ) ) + return {} + else: return {} def remove_generated_tests( app ): @@ -1138,7 +1165,7 @@ for repository in repositories_by_owner[ owner ]: print "# %s owned by %s, changeset revision %s" % ( repository[ 'name' ], repository[ 'owner' ], repository[ 'changeset_revision' ] ) -def test_repository_tools( app, repository, repository_dict, tool_test_results_dict, results_dict ): +def test_repository_tools( app, repository, repository_dict, tool_test_results_dicts, tool_test_results_dict, results_dict ): """Test tools contained in the received repository.""" name = str( repository.name ) owner = str( repository.owner ) @@ -1175,7 +1202,8 @@ params = dict( tools_functionally_correct=True, do_not_test=False, test_install_error=False ) - register_test_result( galaxy_tool_shed_url, tool_test_results_dict, repository_dict, params ) + # TODO: do something usefule with response_dict + response_dict = register_test_result( galaxy_tool_shed_url, tool_test_results_dicts, tool_test_results_dict, repository_dict, params ) log.debug( 'Revision %s of repository %s installed and passed functional tests.' % ( str( changeset_revision ), str( name ) ) ) else: tool_test_results_dict[ 'failed_tests' ].append( extract_log_data( result, from_tool_test=True ) ) @@ -1184,7 +1212,8 @@ params = dict( tools_functionally_correct=False, test_install_error=False, do_not_test=str( set_do_not_test ) ) - register_test_result( galaxy_tool_shed_url, tool_test_results_dict, repository_dict, params ) + # TODO: do something usefule with response_dict + response_dict = register_test_result( galaxy_tool_shed_url, tool_test_results_dicts, tool_test_results_dict, repository_dict, params ) log.debug( 'Revision %s of repository %s installed successfully but did not pass functional tests.' % \ ( str( changeset_revision ), str( name ) ) ) # Run the uninstall method. This removes tool functional test methods from the test_toolbox module and uninstalls the diff -r ad38b77e96fafc96060f9ee8031094f4a15df278 -r 11028b31d3f024ef920c3168d3f752bc13ba3ad8 tool_shed_wsgi.ini.sample --- a/tool_shed_wsgi.ini.sample +++ b/tool_shed_wsgi.ini.sample @@ -74,6 +74,9 @@ # path to sendmail sendmail_path = /usr/sbin/sendmail +# Number of saved tool test results produced by the install and test framework for each repository. +#num_tool_test_results_saved = 5 + # For use by email messages sent from the tool shed #smtp_server = smtp.your_tool_shed_server #email_from = your_tool_shed_email@server Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: guerler: Pack scripts
by commits-noreply＠bitbucket.org 27 Nov '13

27 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ad38b77e96fa/ Changeset: ad38b77e96fa User: guerler Date: 2013-11-27 20:06:06 Summary: Pack scripts Affected #: 1 file diff -r 9be62566dbb6f0a0bfc20f6a8027832399838861 -r ad38b77e96fafc96060f9ee8031094f4a15df278 static/scripts/packed/galaxy.menu.js --- a/static/scripts/packed/galaxy.menu.js +++ b/static/scripts/packed/galaxy.menu.js @@ -1,1 +1,1 @@ -define(["galaxy.masthead"],function(b){var a=Backbone.Model.extend({options:null,masthead:null,initialize:function(c){this.options=c.config;this.masthead=c.masthead;this.create()},create:function(){var d=new b.GalaxyMastheadTab({title:"Analyze Data",content:"root/index"});this.masthead.append(d);var c=new b.GalaxyMastheadTab({title:"Workflow",content:"workflow"});this.masthead.append(c);var g=new b.GalaxyMastheadTab({title:"Shared Data",content:"library/index"});g.addMenu({title:"Data Libraries",content:"library/index",divider:true});g.addMenu({title:"Published Histories",content:"history/list_published"});g.addMenu({title:"Published Workflows",content:"workflow/list_published"});g.addMenu({title:"Published Visualizations",content:"visualization/list_published"});g.addMenu({title:"Published Pages",content:"page/list_published"});this.masthead.append(g);if(this.options.user.requests){var h=new b.GalaxyMastheadTab({title:"Lab"});h.addMenu({title:"Sequencing Requests",content:"requests/index"});h.addMenu({title:"Find Samples",content:"requests/find_samples_index"});h.addMenu({title:"Help",content:this.options.lims_doc_url});this.masthead.append(h)}var k=new b.GalaxyMastheadTab({title:"Visualization",content:"visualization/list"});k.addMenu({title:"New Track Browser",content:"visualization/trackster",target:"_frame"});k.addMenu({title:"Saved Visualizations",content:"visualization/list",target:"_frame"});this.masthead.append(k);if(this.options.enable_cloud_launch){var e=new b.GalaxyMastheadTab({title:"Cloud",content:"cloudlaunch/index"});e.addMenu({title:"New Cloud Cluster",content:"cloudlaunch/index"});this.masthead.append(e)}if(this.options.is_admin_user){var f=new b.GalaxyMastheadTab({title:"Admin",content:"admin/index",extra_class:"admin-only"});this.masthead.append(f)}var j=new b.GalaxyMastheadTab({title:"Help"});if(this.options.biostar_url){j.addMenu({title:"Galaxy Q&A Site",content:this.options.biostar_url_redirect,target:"_blank"});j.addMenu({title:"Ask a question",content:"biostar/biostar_question_redirect",target:"_blank"})}j.addMenu({title:"Support",content:this.options.support_url,target:"_blank"});j.addMenu({title:"Search",content:this.options.search_url,target:"_blank"});j.addMenu({title:"Mailing Lists",content:this.options.mailing_lists,target:"_blank"});j.addMenu({title:"Videos",content:this.options.screencasts_url,target:"_blank"});j.addMenu({title:"Wiki",content:this.options.wiki_url,target:"_blank"});j.addMenu({title:"How to Cite Galaxy",content:this.options.citation_url,target:"_blank"});if(!this.options.terms_url){j.addMenu({title:"Terms and Conditions",content:this.options.terms_url,target:"_blank"})}this.masthead.append(j);if(!this.options.user.valid){var i=new b.GalaxyMastheadTab({title:"User",extra_class:"loggedout-only"});if(this.options.allow_user_creation){i.addMenu({title:"Login",content:"user/login",target:"galaxy_main"})}if(this.options.allow_user_creation){i.addMenu({title:"Register",content:"user/create",target:"galaxy_main"})}this.masthead.append(i)}else{var i=new b.GalaxyMastheadTab({title:"User",extra_class:"loggedin-only"});i.addMenu({title:"Logged in as "+this.options.user.email});if(this.options.use_remote_user&&this.options.remote_user_logout_href){i.addMenu({title:"Logout",content:this.options.remote_user_logout_href,target:"_top"})}else{i.addMenu({title:"Preferences",content:"user?cntrller=user",target:"galaxy_main"});i.addMenu({title:"Custom Builds",content:"user/dbkeys",target:"galaxy_main"});i.addMenu({title:"Logout",content:"user/logout",target:"_top",divider:true})}i.addMenu({title:"Saved Histories",content:"history/list",target:"galaxy_main"});i.addMenu({title:"Saved Datasets",content:"dataset/list",target:"galaxy_main"});i.addMenu({title:"Saved Pages",content:"page/list",target:"_top"});i.addMenu({title:"API Keys",content:"user/api_keys?cntrller=user",target:"galaxy_main"});if(this.options.use_remote_user){i.addMenu({title:"Public Name",content:"user/edit_username?cntrller=user",target:"galaxy_main"})}this.masthead.append(i)}}});return{GalaxyMenu:a}}); \ No newline at end of file +define(["galaxy.masthead"],function(b){var a=Backbone.Model.extend({options:null,masthead:null,initialize:function(c){this.options=c.config;this.masthead=c.masthead;this.create()},create:function(){var d=new b.GalaxyMastheadTab({title:"Analyze Data",content:"root/index"});this.masthead.append(d);var c=new b.GalaxyMastheadTab({title:"Workflow",content:"workflow"});this.masthead.append(c);var g=new b.GalaxyMastheadTab({title:"Shared Data",content:"library/index"});g.addMenu({title:"Data Libraries",content:"library/index",divider:true});g.addMenu({title:"Published Histories",content:"history/list_published"});g.addMenu({title:"Published Workflows",content:"workflow/list_published"});g.addMenu({title:"Published Visualizations",content:"visualization/list_published"});g.addMenu({title:"Published Pages",content:"page/list_published"});this.masthead.append(g);if(this.options.user.requests){var h=new b.GalaxyMastheadTab({title:"Lab"});h.addMenu({title:"Sequencing Requests",content:"requests/index"});h.addMenu({title:"Find Samples",content:"requests/find_samples_index"});h.addMenu({title:"Help",content:this.options.lims_doc_url});this.masthead.append(h)}var k=new b.GalaxyMastheadTab({title:"Visualization",content:"visualization/list"});k.addMenu({title:"New Track Browser",content:"visualization/trackster",target:"_frame"});k.addMenu({title:"Saved Visualizations",content:"visualization/list",target:"_frame"});this.masthead.append(k);if(this.options.enable_cloud_launch){var e=new b.GalaxyMastheadTab({title:"Cloud",content:"cloudlaunch/index"});e.addMenu({title:"New Cloud Cluster",content:"cloudlaunch/index"});this.masthead.append(e)}if(this.options.is_admin_user){var f=new b.GalaxyMastheadTab({title:"Admin",content:"admin/index",extra_class:"admin-only"});this.masthead.append(f)}var j=new b.GalaxyMastheadTab({title:"Help"});if(this.options.biostar_url){j.addMenu({title:"Galaxy Q&A Site",content:this.options.biostar_url_redirect,target:"_blank"});j.addMenu({title:"Ask a question",content:"biostar/biostar_question_redirect",target:"_blank"})}j.addMenu({title:"Support",content:this.options.support_url,target:"_blank"});j.addMenu({title:"Search",content:this.options.search_url,target:"_blank"});j.addMenu({title:"Mailing Lists",content:this.options.mailing_lists,target:"_blank"});j.addMenu({title:"Videos",content:this.options.screencasts_url,target:"_blank"});j.addMenu({title:"Wiki",content:this.options.wiki_url,target:"_blank"});j.addMenu({title:"How to Cite Galaxy",content:this.options.citation_url,target:"_blank"});if(!this.options.terms_url){j.addMenu({title:"Terms and Conditions",content:this.options.terms_url,target:"_blank"})}this.masthead.append(j);if(!this.options.user.valid){var i=new b.GalaxyMastheadTab({title:"User",extra_class:"loggedout-only"});i.addMenu({title:"Login",content:"user/login",target:"galaxy_main"});if(this.options.allow_user_creation){i.addMenu({title:"Register",content:"user/create",target:"galaxy_main"})}this.masthead.append(i)}else{var i=new b.GalaxyMastheadTab({title:"User",extra_class:"loggedin-only"});i.addMenu({title:"Logged in as "+this.options.user.email});if(this.options.use_remote_user&&this.options.remote_user_logout_href){i.addMenu({title:"Logout",content:this.options.remote_user_logout_href,target:"_top"})}else{i.addMenu({title:"Preferences",content:"user?cntrller=user",target:"galaxy_main"});i.addMenu({title:"Custom Builds",content:"user/dbkeys",target:"galaxy_main"});i.addMenu({title:"Logout",content:"user/logout",target:"_top",divider:true})}i.addMenu({title:"Saved Histories",content:"history/list",target:"galaxy_main"});i.addMenu({title:"Saved Datasets",content:"dataset/list",target:"galaxy_main"});i.addMenu({title:"Saved Pages",content:"page/list",target:"_top"});i.addMenu({title:"API Keys",content:"user/api_keys?cntrller=user",target:"galaxy_main"});if(this.options.use_remote_user){i.addMenu({title:"Public Name",content:"user/edit_username?cntrller=user",target:"galaxy_main"})}this.masthead.append(i)}}});return{GalaxyMenu:a}}); \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: guerler: Masthead: Fix login
by commits-noreply＠bitbucket.org 27 Nov '13

27 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/9be62566dbb6/ Changeset: 9be62566dbb6 User: guerler Date: 2013-11-27 19:44:25 Summary: Masthead: Fix login Affected #: 1 file diff -r 97940f59772dd8567c68cb8f895d00d99487fdb9 -r 9be62566dbb6f0a0bfc20f6a8027832399838861 static/scripts/galaxy.menu.js --- a/static/scripts/galaxy.menu.js +++ b/static/scripts/galaxy.menu.js @@ -223,14 +223,11 @@ }); // login - if (this.options.allow_user_creation) - { - tab_user.addMenu({ - title : "Login", - content : "user/login", - target : "galaxy_main" - }); - } + tab_user.addMenu({ + title : "Login", + content : "user/login", + target : "galaxy_main" + }); // register if (this.options.allow_user_creation) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Mimic Galaxy functional test handling of GALAXY_TEST_DB_TEMPLATE for tool shed functional tests.
by commits-noreply＠bitbucket.org 27 Nov '13

27 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/97940f59772d/ Changeset: 97940f59772d User: jmchilton Date: 2013-11-27 15:37:30 Summary: Mimic Galaxy functional test handling of GALAXY_TEST_DB_TEMPLATE for tool shed functional tests. See 83a95ea606d9fcd15f7554b71804625e203eb1ed for full details (essentially GALAXY_TEST_DB_TEMPLATE can be sqlite file path or URL - will be copied over so original left in pristine state and auto upgraded as needed. Affected #: 1 file diff -r 0712180dc77d6df0f19870046562cc34317eb3cc -r 97940f59772dd8567c68cb8f895d00d99487fdb9 test/tool_shed/functional_tests.py --- a/test/tool_shed/functional_tests.py +++ b/test/tool_shed/functional_tests.py @@ -32,6 +32,7 @@ import twill, unittest, time import sys, threading, random import httplib, socket +import urllib from paste import httpserver # This is for the tool shed application. import galaxy.webapps.tool_shed.app @@ -186,10 +187,20 @@ toolshed_database_connection = os.environ[ 'TOOL_SHED_TEST_DBURI' ] else: toolshed_database_connection = 'sqlite:///' + os.path.join( shed_db_path, 'community_test.sqlite' ) + galaxy_database_auto_migrate = False if 'GALAXY_TEST_DBURI' in os.environ: galaxy_database_connection = os.environ[ 'GALAXY_TEST_DBURI' ] else: - galaxy_database_connection = 'sqlite:///' + os.path.join( galaxy_db_path, 'universe_test.sqlite' ) + db_path = os.path.join( galaxy_db_path, 'universe.sqlite' ) + if 'GALAXY_TEST_DB_TEMPLATE' in os.environ: + # Middle ground between recreating a completely new + # database and pointing at existing database with + # GALAXY_TEST_DBURI. The former requires a lot of setup + # time, the latter results in test failures in certain + # cases (namely tool shed tests expecting clean database). + __copy_database_template(os.environ['GALAXY_TEST_DB_TEMPLATE'], db_path) + galaxy_database_auto_migrate = True + galaxy_database_connection = 'sqlite:///%s' % db_path tool_shed_global_conf = get_webapp_global_conf() tool_shed_global_conf[ '__file__' ] = 'tool_shed_wsgi.ini.sample' kwargs = dict( admin_users = 'test(a)bx.psu.edu', @@ -311,6 +322,7 @@ admin_users = 'test(a)bx.psu.edu', allow_library_path_paste = True, database_connection = galaxy_database_connection, + database_auto_migrate = galaxy_database_auto_migrate, datatype_converters_config_file = "datatype_converters_conf.xml.sample", enable_tool_shed_check = True, file_path = galaxy_file_path, @@ -449,6 +461,22 @@ else: return 1 + +def __copy_database_template( source, db_path ): + """ + Copy a 'clean' sqlite template database (from file or URL) to specified + database path. + """ + os.makedirs( os.path.dirname( db_path ) ) + if os.path.exists( source ): + shutil.copy( source, db_path ) + assert os.path.exists( db_path ) + elif source.startswith("http"): + urllib.urlretrieve( source, db_path ) + else: + raise Exception( "Failed to copy database template from source %s" % source ) + + if __name__ == "__main__": try: sys.exit( main() ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: UI: temporary workaround for popupmenus v. XSS security
by commits-noreply＠bitbucket.org 27 Nov '13

27 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0712180dc77d/ Changeset: 0712180dc77d User: carlfeberhard Date: 2013-11-27 00:08:42 Summary: UI: temporary workaround for popupmenus v. XSS security Affected #: 2 files diff -r 451af16e671a1be729d0cae6e7ecb1c838af48bd -r 0712180dc77d6df0f19870046562cc34317eb3cc static/scripts/mvc/ui.js --- a/static/scripts/mvc/ui.js +++ b/static/scripts/mvc/ui.js @@ -297,18 +297,26 @@ function closePopup( event ){ $( document ).off( 'click.close_popup' ); if( window.parent !== window ){ - $( window.parent.document ).off( "click.close_popup" ); + try { + $( window.parent.document ).off( "click.close_popup" ); + } catch( err ){} } else { - $( 'iframe#galaxy_main' ).contents().off( "click.close_popup" ); + try { + $( 'iframe#galaxy_main' ).contents().off( "click.close_popup" ); + } catch( err ){} } menu.remove(); } $( 'html' ).one( "click.close_popup", closePopup ); if( window.parent !== window ){ - $( window.parent.document ).find( 'html' ).one( "click.close_popup", closePopup ); + try { + $( window.parent.document ).find( 'html' ).one( "click.close_popup", closePopup ); + } catch( err ){} } else { - $( 'iframe#galaxy_main' ).contents().one( "click.close_popup", closePopup ); + try { + $( 'iframe#galaxy_main' ).contents().one( "click.close_popup", closePopup ); + } catch( err ){} } }, diff -r 451af16e671a1be729d0cae6e7ecb1c838af48bd -r 0712180dc77d6df0f19870046562cc34317eb3cc static/scripts/packed/mvc/ui.js --- a/static/scripts/packed/mvc/ui.js +++ b/static/scripts/packed/mvc/ui.js @@ -1,1 +1,1 @@ -var IconButton=Backbone.Model.extend({defaults:{title:"",icon_class:"",on_click:null,menu_options:null,is_menu_button:true,id:null,href:null,target:null,enabled:true,visible:true,tooltip_config:{}}});var IconButtonView=Backbone.View.extend({initialize:function(){this.model.attributes.tooltip_config={placement:"bottom"};this.model.bind("change",this.render,this)},render:function(){this.$el.tooltip("hide");var a=this.template(this.model.toJSON());a.tooltip(this.model.get("tooltip_config"));this.$el.replaceWith(a);this.setElement(a);return this},events:{click:"click"},click:function(a){if(_.isFunction(this.model.get("on_click"))){this.model.get("on_click")(a);return false}return true},template:function(b){var a='title="'+b.title+'" class="icon-button';if(b.is_menu_button){a+=" menu-button"}a+=" "+b.icon_class;if(!b.enabled){a+="_disabled"}a+='"';if(b.id){a+=' id="'+b.id+'"'}a+=' href="'+b.href+'"';if(b.target){a+=' target="'+b.target+'"'}if(!b.visible){a+=' style="display: none;"'}if(b.enabled){a="<a "+a+"/>"}else{a="<span "+a+"/>"}return $(a)}});var IconButtonCollection=Backbone.Collection.extend({model:IconButton});var IconButtonMenuView=Backbone.View.extend({tagName:"div",initialize:function(){this.render()},render:function(){var a=this;this.collection.each(function(d){var b=$("<a/>").attr("href","javascript:void(0)").attr("title",d.attributes.title).addClass("icon-button menu-button").addClass(d.attributes.icon_class).appendTo(a.$el).click(d.attributes.on_click);if(d.attributes.tooltip_config){b.tooltip(d.attributes.tooltip_config)}var c=d.get("options");if(c){make_popupmenu(b,c)}});return this}});var create_icon_buttons_menu=function(b,a){if(!a){a={}}var c=new IconButtonCollection(_.map(b,function(d){return new IconButton(_.extend(d,a))}));return new IconButtonMenuView({collection:c})};var Grid=Backbone.Collection.extend({});var GridView=Backbone.View.extend({});var PopupMenu=Backbone.View.extend({initialize:function(b,a){this.$button=b||$("<div/>");this.options=a||[];var c=this;this.$button.click(function(d){$(".popmenu-wrapper").remove();c._renderAndShow(d);return false})},_renderAndShow:function(a){this.render();this.$el.appendTo("body").css(this._getShownPosition(a)).show();this._setUpCloseBehavior()},render:function(){this.$el.addClass("popmenu-wrapper").hide().css({position:"absolute"}).html(this.template(this.$button.attr("id"),this.options));if(this.options.length){var a=this;this.$el.find("li").each(function(c,b){var d=a.options[c];if(d.func){$(this).children("a.popupmenu-option").click(function(e){d.func.call(a,e,d)})}})}return this},template:function(b,a){return['<ul id="',b,'-menu" class="dropdown-menu">',this._templateOptions(a),"</ul>"].join("")},_templateOptions:function(a){if(!a.length){return"<li>(no options)</li>"}return _.map(a,function(d){if(d.divider){return'<li class="divider"></li>'}else{if(d.header){return['<li class="head"><a href="javascript:void(0);">',d.html,"</a></li>"].join("")}}var c=d.href||"javascript:void(0);",e=(d.target)?(' target="'+d.target+'"'):(""),b=(d.checked)?('<span class="fa fa-check"></span>'):("");return['<li><a class="popupmenu-option" href="',c,'"',e,">",b,d.html,"</a></li>"].join("")}).join("")},_getShownPosition:function(b){var c=this.$el.width();var a=b.pageX-c/2;a=Math.min(a,$(document).scrollLeft()+$(window).width()-c-5);a=Math.max(a,$(document).scrollLeft()+5);return{top:b.pageY,left:a}},_setUpCloseBehavior:function(){var b=this;function a(c){$(document).off("click.close_popup");if(window.parent!==window){$(window.parent.document).off("click.close_popup")}else{$("iframe#galaxy_main").contents().off("click.close_popup")}b.remove()}$("html").one("click.close_popup",a);if(window.parent!==window){$(window.parent.document).find("html").one("click.close_popup",a)}else{$("iframe#galaxy_main").contents().one("click.close_popup",a)}},addItem:function(b,a){a=(a>=0)?a:this.options.length;this.options.splice(a,0,b);return this},removeItem:function(a){if(a>=0){this.options.splice(a,1)}return this},findIndexByHtml:function(b){for(var a=0;a<this.options.length;a++){if(_.has(this.options[a],"html")&&(this.options[a].html===b)){return a}}return null},findItemByHtml:function(a){return this.options[(this.findIndexByHtml(a))]},toString:function(){return"PopupMenu"}});PopupMenu.make_popupmenu=function(b,c){var a=[];_.each(c,function(f,d){var e={html:d};if(f===null){e.header=true}else{if(jQuery.type(f)==="function"){e.func=f}}a.push(e)});return new PopupMenu($(b),a)};PopupMenu.convertLinksToOptions=function(c,a){c=$(c);a=a||"a";var b=[];c.find(a).each(function(g,e){var f={},d=$(g);f.html=d.text();if(d.attr("href")){var j=d.attr("href"),k=d.attr("target"),h=d.attr("confirm");f.func=function(){if((h)&&(!confirm(h))){return}switch(k){case"_parent":window.parent.location=j;break;case"_top":window.top.location=j;break;default:window.location=j}}}b.push(f)});return b};PopupMenu.fromExistingDom=function(d,c,a){d=$(d);c=$(c);var b=PopupMenu.convertLinksToOptions(c,a);c.remove();return new PopupMenu(d,b)};PopupMenu.make_popup_menus=function(c,b,d){c=c||document;b=b||"div[popupmenu]";d=d||function(e,f){return"#"+e.attr("popupmenu")};var a=[];$(c).find(b).each(function(){var e=$(this),f=$(c).find(d(e,c));a.push(PopupMenu.fromDom(f,e));f.addClass("popup")});return a};var faIconButton=function(a){a=a||{};a.tooltipConfig=a.tooltipConfig||{placement:"bottom"};a.classes=["icon-btn"].concat(a.classes||[]);if(a.disabled){a.classes.push("disabled")}var b=['<a class="',a.classes.join(" "),'"',((a.title)?(' title="'+a.title+'"'):("")),((a.target)?(' target="'+a.target+'"'):("")),' href="',((a.href)?(a.href):("javascript:void(0);")),'">','<span class="fa ',a.faIcon,'"></span>',"</a>"].join("");var c=$(b).tooltip(a.tooltipConfig);if(_.isFunction(a.onclick)){c.click(a.onclick)}return c};var searchInput=function(k){var a=27,h=13,i=$("<div/>"),b={initialVal:"",name:"search",placeholder:"search",classes:"",onclear:function(){},onsearch:function(l){},minSearchLen:0,escWillClear:true,oninit:function(){}};if(jQuery.type(k)==="object"){k=jQuery.extend(true,b,k)}function d(l){var m=$(this).parent().children("input");m.val("");m.trigger("clear:searchInput");k.onclear()}function j(m,l){$(this).trigger("search:searchInput",l);k.onsearch(l)}function c(){return['<input type="text" name="',k.name,'" placeholder="',k.placeholder,'" ','class="search-query ',k.classes,'" ',"/>"].join("")}function g(){return $(c()).css({width:"100%","padding-right":"24px"}).focus(function(l){$(this).select()}).keyup(function(m){if(m.which===a&&k.escWillClear){d.call(this,m)}else{var l=$(this).val();if((m.which===h)||(k.minSearchLen&&l.length>=k.minSearchLen)){j.call(this,m,l)}else{if(!l.length){d.call(this,m)}}}}).val(k.initialVal)}function f(){return'<span class="search-clear fa fa-times-circle"></span>'}function e(){return $(f()).css({position:"absolute",right:"15px","font-size":"1.4em","line-height":"23px",color:"grey"}).click(function(l){d.call(this,l)})}return i.append([g(),e()])};function LoadingIndicator(a,c){var b=this;c=jQuery.extend({cover:false},c||{});function d(){var e=['<div class="loading-indicator">','<div class="loading-indicator-text">','<span class="fa fa-spinner fa-spin fa-lg"></span>','<span class="loading-indicator-message">loading...</span>',"</div>","</div>"].join("\n");var g=$(e).hide().css(c.css||{position:"fixed"}),f=g.children(".loading-indicator-text");if(c.cover){g.css({"z-index":2,top:a.css("top"),bottom:a.css("bottom"),left:a.css("left"),right:a.css("right"),opacity:0.5,"background-color":"white","text-align":"center"});f=g.children(".loading-indicator-text").css({"margin-top":"20px"})}else{f=g.children(".loading-indicator-text").css({margin:"12px 0px 0px 10px",opacity:"0.85",color:"grey"});f.children(".loading-indicator-message").css({margin:"0px 8px 0px 0px","font-style":"italic"})}return g}b.show=function(f,e,g){f=f||"loading...";e=e||"fast";b.$indicator=d().insertBefore(a);b.message(f);b.$indicator.fadeIn(e,g);return b};b.message=function(e){b.$indicator.find("i").text(e)};b.hide=function(e,f){e=e||"fast";if(b.$indicator&&b.$indicator.size()){b.$indicator.fadeOut(e,function(){b.$indicator.remove();if(f){f()}})}else{if(f){f()}}return b};return b}; \ No newline at end of file +var IconButton=Backbone.Model.extend({defaults:{title:"",icon_class:"",on_click:null,menu_options:null,is_menu_button:true,id:null,href:null,target:null,enabled:true,visible:true,tooltip_config:{}}});var IconButtonView=Backbone.View.extend({initialize:function(){this.model.attributes.tooltip_config={placement:"bottom"};this.model.bind("change",this.render,this)},render:function(){this.$el.tooltip("hide");var a=this.template(this.model.toJSON());a.tooltip(this.model.get("tooltip_config"));this.$el.replaceWith(a);this.setElement(a);return this},events:{click:"click"},click:function(a){if(_.isFunction(this.model.get("on_click"))){this.model.get("on_click")(a);return false}return true},template:function(b){var a='title="'+b.title+'" class="icon-button';if(b.is_menu_button){a+=" menu-button"}a+=" "+b.icon_class;if(!b.enabled){a+="_disabled"}a+='"';if(b.id){a+=' id="'+b.id+'"'}a+=' href="'+b.href+'"';if(b.target){a+=' target="'+b.target+'"'}if(!b.visible){a+=' style="display: none;"'}if(b.enabled){a="<a "+a+"/>"}else{a="<span "+a+"/>"}return $(a)}});var IconButtonCollection=Backbone.Collection.extend({model:IconButton});var IconButtonMenuView=Backbone.View.extend({tagName:"div",initialize:function(){this.render()},render:function(){var a=this;this.collection.each(function(d){var b=$("<a/>").attr("href","javascript:void(0)").attr("title",d.attributes.title).addClass("icon-button menu-button").addClass(d.attributes.icon_class).appendTo(a.$el).click(d.attributes.on_click);if(d.attributes.tooltip_config){b.tooltip(d.attributes.tooltip_config)}var c=d.get("options");if(c){make_popupmenu(b,c)}});return this}});var create_icon_buttons_menu=function(b,a){if(!a){a={}}var c=new IconButtonCollection(_.map(b,function(d){return new IconButton(_.extend(d,a))}));return new IconButtonMenuView({collection:c})};var Grid=Backbone.Collection.extend({});var GridView=Backbone.View.extend({});var PopupMenu=Backbone.View.extend({initialize:function(b,a){this.$button=b||$("<div/>");this.options=a||[];var c=this;this.$button.click(function(d){$(".popmenu-wrapper").remove();c._renderAndShow(d);return false})},_renderAndShow:function(a){this.render();this.$el.appendTo("body").css(this._getShownPosition(a)).show();this._setUpCloseBehavior()},render:function(){this.$el.addClass("popmenu-wrapper").hide().css({position:"absolute"}).html(this.template(this.$button.attr("id"),this.options));if(this.options.length){var a=this;this.$el.find("li").each(function(c,b){var d=a.options[c];if(d.func){$(this).children("a.popupmenu-option").click(function(e){d.func.call(a,e,d)})}})}return this},template:function(b,a){return['<ul id="',b,'-menu" class="dropdown-menu">',this._templateOptions(a),"</ul>"].join("")},_templateOptions:function(a){if(!a.length){return"<li>(no options)</li>"}return _.map(a,function(d){if(d.divider){return'<li class="divider"></li>'}else{if(d.header){return['<li class="head"><a href="javascript:void(0);">',d.html,"</a></li>"].join("")}}var c=d.href||"javascript:void(0);",e=(d.target)?(' target="'+d.target+'"'):(""),b=(d.checked)?('<span class="fa fa-check"></span>'):("");return['<li><a class="popupmenu-option" href="',c,'"',e,">",b,d.html,"</a></li>"].join("")}).join("")},_getShownPosition:function(b){var c=this.$el.width();var a=b.pageX-c/2;a=Math.min(a,$(document).scrollLeft()+$(window).width()-c-5);a=Math.max(a,$(document).scrollLeft()+5);return{top:b.pageY,left:a}},_setUpCloseBehavior:function(){var c=this;function a(e){$(document).off("click.close_popup");if(window.parent!==window){try{$(window.parent.document).off("click.close_popup")}catch(d){}}else{try{$("iframe#galaxy_main").contents().off("click.close_popup")}catch(d){}}c.remove()}$("html").one("click.close_popup",a);if(window.parent!==window){try{$(window.parent.document).find("html").one("click.close_popup",a)}catch(b){}}else{try{$("iframe#galaxy_main").contents().one("click.close_popup",a)}catch(b){}}},addItem:function(b,a){a=(a>=0)?a:this.options.length;this.options.splice(a,0,b);return this},removeItem:function(a){if(a>=0){this.options.splice(a,1)}return this},findIndexByHtml:function(b){for(var a=0;a<this.options.length;a++){if(_.has(this.options[a],"html")&&(this.options[a].html===b)){return a}}return null},findItemByHtml:function(a){return this.options[(this.findIndexByHtml(a))]},toString:function(){return"PopupMenu"}});PopupMenu.make_popupmenu=function(b,c){var a=[];_.each(c,function(f,d){var e={html:d};if(f===null){e.header=true}else{if(jQuery.type(f)==="function"){e.func=f}}a.push(e)});return new PopupMenu($(b),a)};PopupMenu.convertLinksToOptions=function(c,a){c=$(c);a=a||"a";var b=[];c.find(a).each(function(g,e){var f={},d=$(g);f.html=d.text();if(d.attr("href")){var j=d.attr("href"),k=d.attr("target"),h=d.attr("confirm");f.func=function(){if((h)&&(!confirm(h))){return}switch(k){case"_parent":window.parent.location=j;break;case"_top":window.top.location=j;break;default:window.location=j}}}b.push(f)});return b};PopupMenu.fromExistingDom=function(d,c,a){d=$(d);c=$(c);var b=PopupMenu.convertLinksToOptions(c,a);c.remove();return new PopupMenu(d,b)};PopupMenu.make_popup_menus=function(c,b,d){c=c||document;b=b||"div[popupmenu]";d=d||function(e,f){return"#"+e.attr("popupmenu")};var a=[];$(c).find(b).each(function(){var e=$(this),f=$(c).find(d(e,c));a.push(PopupMenu.fromDom(f,e));f.addClass("popup")});return a};var faIconButton=function(a){a=a||{};a.tooltipConfig=a.tooltipConfig||{placement:"bottom"};a.classes=["icon-btn"].concat(a.classes||[]);if(a.disabled){a.classes.push("disabled")}var b=['<a class="',a.classes.join(" "),'"',((a.title)?(' title="'+a.title+'"'):("")),((a.target)?(' target="'+a.target+'"'):("")),' href="',((a.href)?(a.href):("javascript:void(0);")),'">','<span class="fa ',a.faIcon,'"></span>',"</a>"].join("");var c=$(b).tooltip(a.tooltipConfig);if(_.isFunction(a.onclick)){c.click(a.onclick)}return c};var searchInput=function(k){var a=27,h=13,i=$("<div/>"),b={initialVal:"",name:"search",placeholder:"search",classes:"",onclear:function(){},onsearch:function(l){},minSearchLen:0,escWillClear:true,oninit:function(){}};if(jQuery.type(k)==="object"){k=jQuery.extend(true,b,k)}function d(l){var m=$(this).parent().children("input");m.val("");m.trigger("clear:searchInput");k.onclear()}function j(m,l){$(this).trigger("search:searchInput",l);k.onsearch(l)}function c(){return['<input type="text" name="',k.name,'" placeholder="',k.placeholder,'" ','class="search-query ',k.classes,'" ',"/>"].join("")}function g(){return $(c()).css({width:"100%","padding-right":"24px"}).focus(function(l){$(this).select()}).keyup(function(m){if(m.which===a&&k.escWillClear){d.call(this,m)}else{var l=$(this).val();if((m.which===h)||(k.minSearchLen&&l.length>=k.minSearchLen)){j.call(this,m,l)}else{if(!l.length){d.call(this,m)}}}}).val(k.initialVal)}function f(){return'<span class="search-clear fa fa-times-circle"></span>'}function e(){return $(f()).css({position:"absolute",right:"15px","font-size":"1.4em","line-height":"23px",color:"grey"}).click(function(l){d.call(this,l)})}return i.append([g(),e()])};function LoadingIndicator(a,c){var b=this;c=jQuery.extend({cover:false},c||{});function d(){var e=['<div class="loading-indicator">','<div class="loading-indicator-text">','<span class="fa fa-spinner fa-spin fa-lg"></span>','<span class="loading-indicator-message">loading...</span>',"</div>","</div>"].join("\n");var g=$(e).hide().css(c.css||{position:"fixed"}),f=g.children(".loading-indicator-text");if(c.cover){g.css({"z-index":2,top:a.css("top"),bottom:a.css("bottom"),left:a.css("left"),right:a.css("right"),opacity:0.5,"background-color":"white","text-align":"center"});f=g.children(".loading-indicator-text").css({"margin-top":"20px"})}else{f=g.children(".loading-indicator-text").css({margin:"12px 0px 0px 10px",opacity:"0.85",color:"grey"});f.children(".loading-indicator-message").css({margin:"0px 8px 0px 0px","font-style":"italic"})}return g}b.show=function(f,e,g){f=f||"loading...";e=e||"fast";b.$indicator=d().insertBefore(a);b.message(f);b.$indicator.fadeIn(e,g);return b};b.message=function(e){b.$indicator.find("i").text(e)};b.hide=function(e,f){e=e||"fast";if(b.$indicator&&b.$indicator.size()){b.$indicator.fadeOut(e,function(){b.$indicator.remove();if(f){f()}})}else{if(f){f()}}return b};return b}; \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Add backbone dependency to trackster module; this is needed for Trackster router to function. Pack script.
by commits-noreply＠bitbucket.org 26 Nov '13

26 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/451af16e671a/ Changeset: 451af16e671a User: jgoecks Date: 2013-11-26 21:46:10 Summary: Add backbone dependency to trackster module; this is needed for Trackster router to function. Pack script. Affected #: 2 files diff -r f4832c4ace99f24bc42dcf275b55c0b2671fb6be -r 451af16e671a1be729d0cae6e7ecb1c838af48bd static/scripts/packed/viz/trackster.js --- a/static/scripts/packed/viz/trackster.js +++ b/static/scripts/packed/viz/trackster.js @@ -1,1 +1,1 @@ -var ui=null;var view=null;var browser_router=null;require(["utils/galaxy.utils","libs/jquery/jstorage","libs/jquery/jquery.event.drag","libs/jquery/jquery.event.hover","libs/jquery/jquery.mousewheel","libs/jquery/jquery-ui","libs/jquery/select2","libs/farbtastic","libs/jquery/jquery.form","libs/jquery/jquery.rating","mvc/ui"],function(a){a.cssLoadFile("static/style/jquery.rating.css");a.cssLoadFile("static/style/autocomplete_tagging.css");a.cssLoadFile("static/style/jquery-ui/smoothness/jquery-ui.css");a.cssLoadFile("static/style/library.css");a.cssLoadFile("static/style/trackster.css")});define(["viz/visualization","viz/trackster_ui"],function(a,b){var c=Backbone.View.extend({initialize:function(){ui=new b.TracksterUI(galaxy_config.root);ui.createButtonMenu();ui.buttonMenu.$el.attr("style","float: right");$("#center .unified-panel-header-inner").append(ui.buttonMenu.$el);$("#right .unified-panel-title").append("Bookmarks");$("#right .unified-panel-icons").append("<a id='add-bookmark-button' class='icon-button menu-button plus-button' href='javascript:void(0);' title='Add bookmark'></a>");$("#right-border").click(function(){view.resize_window()});force_right_panel("hide");if(galaxy_config.app.id){this.view_existing()}else{this.view_new()}},set_up_router:function(d){browser_router=new a.TrackBrowserRouter(d);Backbone.history.start()},view_existing:function(){var d=galaxy_config.app.viz_config;view=ui.create_visualization({container:$("#center .unified-panel-body"),name:d.title,vis_id:d.vis_id,dbkey:d.dbkey},d.viewport,d.tracks,d.bookmarks,true);this.init_editor()},view_new:function(){var d=this;$.ajax({url:galaxy_config.root+"api/genomes?chrom_info=True",data:{},error:function(){alert("Couldn't create new browser.")},success:function(e){Galaxy.modal.show({title:"New Visualization",body:d.template_view_new(e),buttons:{Cancel:function(){window.location=galaxy_config.root+"visualization/list"},Create:function(){d.create_browser($("#new-title").val(),$("#new-dbkey").val());Galaxy.modal.hide()}}});if(galaxy_config.app.default_dbkey){$("#new-dbkey").val(galaxy_config.app.default_dbkey)}$("#new-title").focus();$("select[name='dbkey']").select2();$("#overlay").css("overflow","auto")}})},template_view_new:function(d){var f='<form id="new-browser-form" action="javascript:void(0);" method="post" onsubmit="return false;"><div class="form-row"><label for="new-title">Browser name:</label><div class="form-row-input"><input type="text" name="title" id="new-title" value="Unnamed"></input></div><div style="clear: both;"></div></div><div class="form-row"><label for="new-dbkey">Reference genome build (dbkey): </label><div class="form-row-input"><select name="dbkey" id="new-dbkey">';for(var e=0;e<d.length;e++){f+='<option value="'+d[e][1]+'">'+d[e][0]+"</option>"}f+='</select></div><div style="clear: both;"></div></div><div class="form-row">Is the build not listed here? <a href="'+galaxy_config.root+'user/dbkeys?use_panels=True">Add a Custom Build</a></div></form>';return f},create_browser:function(e,d){$(document).trigger("convert_to_values");view=ui.create_visualization({container:$("#center .unified-panel-body"),name:e,dbkey:d},galaxy_config.app.gene_region);this.init_editor();view.editor=true},init_editor:function(){$("#center .unified-panel-title").text(view.prefs.name+" ("+view.dbkey+")");if(galaxy_config.app.add_dataset){$.ajax({url:galaxy_config.root+"api/datasets/"+galaxy_config.app.add_dataset,data:{hda_ldda:"hda",data_type:"track_config"},dataType:"json",success:function(d){view.add_drawable(b.object_from_template(d,view,view))}})}$("#add-bookmark-button").click(function(){var e=view.chrom+":"+view.low+"-"+view.high,d="Bookmark description";return ui.add_bookmark(e,d,true)});ui.init_keyboard_nav(view);this.set_up_router({view:view})}});return{GalaxyApp:c}}); \ No newline at end of file +var ui=null;var view=null;var browser_router=null;require(["utils/galaxy.utils","libs/jquery/jstorage","libs/jquery/jquery.event.drag","libs/jquery/jquery.event.hover","libs/jquery/jquery.mousewheel","libs/jquery/jquery-ui","libs/jquery/select2","libs/farbtastic","libs/jquery/jquery.form","libs/jquery/jquery.rating","mvc/ui"],function(a){a.cssLoadFile("static/style/jquery.rating.css");a.cssLoadFile("static/style/autocomplete_tagging.css");a.cssLoadFile("static/style/jquery-ui/smoothness/jquery-ui.css");a.cssLoadFile("static/style/library.css");a.cssLoadFile("static/style/trackster.css")});define(["libs/backbone/backbone","viz/visualization","viz/trackster_ui"],function(c,a,b){var d=Backbone.View.extend({initialize:function(){ui=new b.TracksterUI(galaxy_config.root);ui.createButtonMenu();ui.buttonMenu.$el.attr("style","float: right");$("#center .unified-panel-header-inner").append(ui.buttonMenu.$el);$("#right .unified-panel-title").append("Bookmarks");$("#right .unified-panel-icons").append("<a id='add-bookmark-button' class='icon-button menu-button plus-button' href='javascript:void(0);' title='Add bookmark'></a>");$("#right-border").click(function(){view.resize_window()});force_right_panel("hide");if(galaxy_config.app.id){this.view_existing()}else{this.view_new()}},set_up_router:function(e){browser_router=new a.TrackBrowserRouter(e);Backbone.history.start()},view_existing:function(){var e=galaxy_config.app.viz_config;view=ui.create_visualization({container:$("#center .unified-panel-body"),name:e.title,vis_id:e.vis_id,dbkey:e.dbkey},e.viewport,e.tracks,e.bookmarks,true);this.init_editor()},view_new:function(){var e=this;$.ajax({url:galaxy_config.root+"api/genomes?chrom_info=True",data:{},error:function(){alert("Couldn't create new browser.")},success:function(f){Galaxy.modal.show({title:"New Visualization",body:e.template_view_new(f),buttons:{Cancel:function(){window.location=galaxy_config.root+"visualization/list"},Create:function(){e.create_browser($("#new-title").val(),$("#new-dbkey").val());Galaxy.modal.hide()}}});if(galaxy_config.app.default_dbkey){$("#new-dbkey").val(galaxy_config.app.default_dbkey)}$("#new-title").focus();$("select[name='dbkey']").select2();$("#overlay").css("overflow","auto")}})},template_view_new:function(e){var g='<form id="new-browser-form" action="javascript:void(0);" method="post" onsubmit="return false;"><div class="form-row"><label for="new-title">Browser name:</label><div class="form-row-input"><input type="text" name="title" id="new-title" value="Unnamed"></input></div><div style="clear: both;"></div></div><div class="form-row"><label for="new-dbkey">Reference genome build (dbkey): </label><div class="form-row-input"><select name="dbkey" id="new-dbkey">';for(var f=0;f<e.length;f++){g+='<option value="'+e[f][1]+'">'+e[f][0]+"</option>"}g+='</select></div><div style="clear: both;"></div></div><div class="form-row">Is the build not listed here? <a href="'+galaxy_config.root+'user/dbkeys?use_panels=True">Add a Custom Build</a></div></form>';return g},create_browser:function(f,e){$(document).trigger("convert_to_values");view=ui.create_visualization({container:$("#center .unified-panel-body"),name:f,dbkey:e},galaxy_config.app.gene_region);this.init_editor();view.editor=true},init_editor:function(){$("#center .unified-panel-title").text(view.prefs.name+" ("+view.dbkey+")");if(galaxy_config.app.add_dataset){$.ajax({url:galaxy_config.root+"api/datasets/"+galaxy_config.app.add_dataset,data:{hda_ldda:"hda",data_type:"track_config"},dataType:"json",success:function(e){view.add_drawable(b.object_from_template(e,view,view))}})}$("#add-bookmark-button").click(function(){var f=view.chrom+":"+view.low+"-"+view.high,e="Bookmark description";return ui.add_bookmark(f,e,true)});ui.init_keyboard_nav(view);this.set_up_router({view:view})}});return{GalaxyApp:d}}); \ No newline at end of file diff -r f4832c4ace99f24bc42dcf275b55c0b2671fb6be -r 451af16e671a1be729d0cae6e7ecb1c838af48bd static/scripts/viz/trackster.js --- a/static/scripts/viz/trackster.js +++ b/static/scripts/viz/trackster.js @@ -29,8 +29,8 @@ }); // trackster viewer -define( ["viz/visualization", "viz/trackster_ui"], - function(visualization, trackster_ui) +define( ["libs/backbone/backbone", "viz/visualization", "viz/trackster_ui"], + function(backbone, visualization, trackster_ui) { var TracksterView = Backbone.View.extend( Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Replace Backbone-relational usage with manually model chaining and remove backbone-relational scripts. Fix default colors bug in Trackster. Pack scripts.
by commits-noreply＠bitbucket.org 26 Nov '13

26 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f4832c4ace99/ Changeset: f4832c4ace99 User: jgoecks Date: 2013-11-26 20:50:41 Summary: Replace Backbone-relational usage with manually model chaining and remove backbone-relational scripts. Fix default colors bug in Trackster. Pack scripts. Affected #: 33 files diff -r a58f6ca10843c91727807e7150b549ecd9b13ec5 -r f4832c4ace99f24bc42dcf275b55c0b2671fb6be static/scripts/galaxy.frame.js --- a/static/scripts/galaxy.frame.js +++ b/static/scripts/galaxy.frame.js @@ -3,7 +3,7 @@ */ // dependencies -define(["galaxy.masthead", "libs/backbone/backbone-relational"], function(mod_masthead) { +define(["galaxy.masthead"], function(mod_masthead) { // frame manager var GalaxyFrame = Backbone.View.extend( diff -r a58f6ca10843c91727807e7150b549ecd9b13ec5 -r f4832c4ace99f24bc42dcf275b55c0b2671fb6be static/scripts/galaxy.masthead.js --- a/static/scripts/galaxy.masthead.js +++ b/static/scripts/galaxy.masthead.js @@ -3,7 +3,7 @@ */ // dependencies -define(["utils/galaxy.utils", "libs/backbone/backbone-relational"], function(mod_utils) { +define(["utils/galaxy.utils"], function(mod_utils) { // masthead var GalaxyMasthead = Backbone.View.extend( diff -r a58f6ca10843c91727807e7150b549ecd9b13ec5 -r f4832c4ace99f24bc42dcf275b55c0b2671fb6be static/scripts/galaxy.menu.js --- a/static/scripts/galaxy.menu.js +++ b/static/scripts/galaxy.menu.js @@ -3,7 +3,7 @@ */ // dependencies -define(["galaxy.masthead", "libs/backbone/backbone-relational"], function(mod_masthead) { +define(["galaxy.masthead"], function(mod_masthead) { // frame manager var GalaxyMenu = Backbone.Model.extend( diff -r a58f6ca10843c91727807e7150b549ecd9b13ec5 -r f4832c4ace99f24bc42dcf275b55c0b2671fb6be static/scripts/galaxy.modal.js --- a/static/scripts/galaxy.modal.js +++ b/static/scripts/galaxy.modal.js @@ -3,7 +3,7 @@ */ // dependencies -define(["libs/backbone/backbone-relational"], function() { +define([], function() { // frame manager var GalaxyModal = Backbone.View.extend( diff -r a58f6ca10843c91727807e7150b549ecd9b13ec5 -r f4832c4ace99f24bc42dcf275b55c0b2671fb6be static/scripts/galaxy.upload.js --- a/static/scripts/galaxy.upload.js +++ b/static/scripts/galaxy.upload.js @@ -3,7 +3,7 @@ */ // dependencies -define(["galaxy.modal", "galaxy.masthead", "utils/galaxy.utils", "utils/galaxy.uploadbox", "libs/backbone/backbone-relational"], function(mod_modal, mod_masthead, mod_utils) { +define(["galaxy.modal", "galaxy.masthead", "utils/galaxy.utils", "utils/galaxy.uploadbox"], function(mod_modal, mod_masthead, mod_utils) { // galaxy upload var GalaxyUpload = Backbone.View.extend( diff -r a58f6ca10843c91727807e7150b549ecd9b13ec5 -r f4832c4ace99f24bc42dcf275b55c0b2671fb6be static/scripts/libs/backbone/backbone-relational.js --- a/static/scripts/libs/backbone/backbone-relational.js +++ /dev/null @@ -1,1919 +0,0 @@ -/* vim: set tabstop=4 softtabstop=4 shiftwidth=4 noexpandtab: */ -/** - * Backbone-relational.js 0.8.5 - * (c) 2011-2013 Paul Uithol and contributors (https://github.com/PaulUithol/Backbone-relational/graphs/contributors) - * - * Backbone-relational may be freely distributed under the MIT license; see the accompanying LICENSE.txt. - * For details and documentation: https://github.com/PaulUithol/Backbone-relational. - * Depends on Backbone (and thus on Underscore as well): https://github.com/documentcloud/backbone. - */ -( function( undefined ) { - "use strict"; - - /** - * CommonJS shim - **/ - var _, Backbone, exports; - if ( typeof window === 'undefined' ) { - _ = require( 'underscore' ); - Backbone = require( 'backbone' ); - exports = Backbone; - typeof module === 'undefined' || ( module.exports = exports ); - } - else { - _ = window._; - Backbone = window.Backbone; - exports = window; - } - - Backbone.Relational = { - showWarnings: true - }; - - /** - * Semaphore mixin; can be used as both binary and counting. - **/ - Backbone.Semaphore = { - _permitsAvailable: null, - _permitsUsed: 0, - - acquire: function() { - if ( this._permitsAvailable && this._permitsUsed >= this._permitsAvailable ) { - throw new Error( 'Max permits acquired' ); - } - else { - this._permitsUsed++; - } - }, - - release: function() { - if ( this._permitsUsed === 0 ) { - throw new Error( 'All permits released' ); - } - else { - this._permitsUsed--; - } - }, - - isLocked: function() { - return this._permitsUsed > 0; - }, - - setAvailablePermits: function( amount ) { - if ( this._permitsUsed > amount ) { - throw new Error( 'Available permits cannot be less than used permits' ); - } - this._permitsAvailable = amount; - } - }; - - /** - * A BlockingQueue that accumulates items while blocked (via 'block'), - * and processes them when unblocked (via 'unblock'). - * Process can also be called manually (via 'process'). - */ - Backbone.BlockingQueue = function() { - this._queue = []; - }; - _.extend( Backbone.BlockingQueue.prototype, Backbone.Semaphore, { - _queue: null, - - add: function( func ) { - if ( this.isBlocked() ) { - this._queue.push( func ); - } - else { - func(); - } - }, - - // Some of the queued events may trigger other blocking events. By - // copying the queue here it allows queued events to process closer to - // the natural order. - // - // queue events [ 'A', 'B', 'C' ] - // A handler of 'B' triggers 'D' and 'E' - // By copying `this._queue` this executes: - // [ 'A', 'B', 'D', 'E', 'C' ] - // The same order the would have executed if they didn't have to be - // delayed and queued. - process: function() { - var queue = this._queue; - this._queue = []; - while ( queue && queue.length ) { - queue.shift()(); - } - }, - - block: function() { - this.acquire(); - }, - - unblock: function() { - this.release(); - if ( !this.isBlocked() ) { - this.process(); - } - }, - - isBlocked: function() { - return this.isLocked(); - } - }); - /** - * Global event queue. Accumulates external events ('add:<key>', 'remove:<key>' and 'change:<key>') - * until the top-level object is fully initialized (see 'Backbone.RelationalModel'). - */ - Backbone.Relational.eventQueue = new Backbone.BlockingQueue(); - - /** - * Backbone.Store keeps track of all created (and destruction of) Backbone.RelationalModel. - * Handles lookup for relations. - */ - Backbone.Store = function() { - this._collections = []; - this._reverseRelations = []; - this._orphanRelations = []; - this._subModels = []; - this._modelScopes = [ exports ]; - }; - _.extend( Backbone.Store.prototype, Backbone.Events, { - /** - * Create a new `Relation`. - * @param {Backbone.RelationalModel} [model] - * @param {Object} relation - * @param {Object} [options] - */ - initializeRelation: function( model, relation, options ) { - var type = !_.isString( relation.type ) ? relation.type : Backbone[ relation.type ] || this.getObjectByName( relation.type ); - if ( type && type.prototype instanceof Backbone.Relation ) { - new type( model, relation, options ); // Also pushes the new Relation into `model._relations` - } - else { - Backbone.Relational.showWarnings && typeof console !== 'undefined' && console.warn( 'Relation=%o; missing or invalid relation type!', relation ); - } - }, - - /** - * Add a scope for `getObjectByName` to look for model types by name. - * @param {Object} scope - */ - addModelScope: function( scope ) { - this._modelScopes.push( scope ); - }, - - /** - * Remove a scope. - * @param {Object} scope - */ - removeModelScope: function( scope ) { - this._modelScopes = _.without( this._modelScopes, scope ); - }, - - /** - * Add a set of subModelTypes to the store, that can be used to resolve the '_superModel' - * for a model later in 'setupSuperModel'. - * - * @param {Backbone.RelationalModel} subModelTypes - * @param {Backbone.RelationalModel} superModelType - */ - addSubModels: function( subModelTypes, superModelType ) { - this._subModels.push({ - 'superModelType': superModelType, - 'subModels': subModelTypes - }); - }, - - /** - * Check if the given modelType is registered as another model's subModel. If so, add it to the super model's - * '_subModels', and set the modelType's '_superModel', '_subModelTypeName', and '_subModelTypeAttribute'. - * - * @param {Backbone.RelationalModel} modelType - */ - setupSuperModel: function( modelType ) { - _.find( this._subModels, function( subModelDef ) { - return _.find( subModelDef.subModels || [], function( subModelTypeName, typeValue ) { - var subModelType = this.getObjectByName( subModelTypeName ); - - if ( modelType === subModelType ) { - // Set 'modelType' as a child of the found superModel - subModelDef.superModelType._subModels[ typeValue ] = modelType; - - // Set '_superModel', '_subModelTypeValue', and '_subModelTypeAttribute' on 'modelType'. - modelType._superModel = subModelDef.superModelType; - modelType._subModelTypeValue = typeValue; - modelType._subModelTypeAttribute = subModelDef.superModelType.prototype.subModelTypeAttribute; - return true; - } - }, this ); - }, this ); - }, - - /** - * Add a reverse relation. Is added to the 'relations' property on model's prototype, and to - * existing instances of 'model' in the store as well. - * @param {Object} relation - * @param {Backbone.RelationalModel} relation.model - * @param {String} relation.type - * @param {String} relation.key - * @param {String|Object} relation.relatedModel - */ - addReverseRelation: function( relation ) { - var exists = _.any( this._reverseRelations, function( rel ) { - return _.all( relation || [], function( val, key ) { - return val === rel[ key ]; - }); - }); - - if ( !exists && relation.model && relation.type ) { - this._reverseRelations.push( relation ); - this._addRelation( relation.model, relation ); - this.retroFitRelation( relation ); - } - }, - - /** - * Deposit a `relation` for which the `relatedModel` can't be resolved at the moment. - * - * @param {Object} relation - */ - addOrphanRelation: function( relation ) { - var exists = _.any( this._orphanRelations, function( rel ) { - return _.all( relation || [], function( val, key ) { - return val === rel[ key ]; - }); - }); - - if ( !exists && relation.model && relation.type ) { - this._orphanRelations.push( relation ); - } - }, - - /** - * Try to initialize any `_orphanRelation`s - */ - processOrphanRelations: function() { - // Make sure to operate on a copy since we're removing while iterating - _.each( this._orphanRelations.slice( 0 ), function( rel ) { - var relatedModel = Backbone.Relational.store.getObjectByName( rel.relatedModel ); - if ( relatedModel ) { - this.initializeRelation( null, rel ); - this._orphanRelations = _.without( this._orphanRelations, rel ); - } - }, this ); - }, - - /** - * - * @param {Backbone.RelationalModel.constructor} type - * @param {Object} relation - * @private - */ - _addRelation: function( type, relation ) { - if ( !type.prototype.relations ) { - type.prototype.relations = []; - } - type.prototype.relations.push( relation ); - - _.each( type._subModels || [], function( subModel ) { - this._addRelation( subModel, relation ); - }, this ); - }, - - /** - * Add a 'relation' to all existing instances of 'relation.model' in the store - * @param {Object} relation - */ - retroFitRelation: function( relation ) { - var coll = this.getCollection( relation.model, false ); - coll && coll.each( function( model ) { - if ( !( model instanceof relation.model ) ) { - return; - } - - new relation.type( model, relation ); - }, this ); - }, - - /** - * Find the Store's collection for a certain type of model. - * @param {Backbone.RelationalModel} type - * @param {Boolean} [create=true] Should a collection be created if none is found? - * @return {Backbone.Collection} A collection if found (or applicable for 'model'), or null - */ - getCollection: function( type, create ) { - if ( type instanceof Backbone.RelationalModel ) { - type = type.constructor; - } - - var rootModel = type; - while ( rootModel._superModel ) { - rootModel = rootModel._superModel; - } - - var coll = _.find( this._collections, function(item) { - return item.model === rootModel; - }); - - if ( !coll && create !== false ) { - coll = this._createCollection( rootModel ); - } - - return coll; - }, - - /** - * Find a model type on one of the modelScopes by name. Names are split on dots. - * @param {String} name - * @return {Object} - */ - getObjectByName: function( name ) { - var parts = name.split( '.' ), - type = null; - - _.find( this._modelScopes, function( scope ) { - type = _.reduce( parts || [], function( memo, val ) { - return memo ? memo[ val ] : undefined; - }, scope ); - - if ( type && type !== scope ) { - return true; - } - }, this ); - - return type; - }, - - _createCollection: function( type ) { - var coll; - - // If 'type' is an instance, take its constructor - if ( type instanceof Backbone.RelationalModel ) { - type = type.constructor; - } - - // Type should inherit from Backbone.RelationalModel. - if ( type.prototype instanceof Backbone.RelationalModel ) { - coll = new Backbone.Collection(); - coll.model = type; - - this._collections.push( coll ); - } - - return coll; - }, - - /** - * Find the attribute that is to be used as the `id` on a given object - * @param type - * @param {String|Number|Object|Backbone.RelationalModel} item - * @return {String|Number} - */ - resolveIdForItem: function( type, item ) { - var id = _.isString( item ) || _.isNumber( item ) ? item : null; - - if ( id === null ) { - if ( item instanceof Backbone.RelationalModel ) { - id = item.id; - } - else if ( _.isObject( item ) ) { - id = item[ type.prototype.idAttribute ]; - } - } - - // Make all falsy values `null` (except for 0, which could be an id.. see '/issues/179') - if ( !id && id !== 0 ) { - id = null; - } - - return id; - }, - - /** - * Find a specific model of a certain `type` in the store - * @param type - * @param {String|Number|Object|Backbone.RelationalModel} item - */ - find: function( type, item ) { - var id = this.resolveIdForItem( type, item ); - var coll = this.getCollection( type ); - - // Because the found object could be of any of the type's superModel - // types, only return it if it's actually of the type asked for. - if ( coll ) { - var obj = coll.get( id ); - - if ( obj instanceof type ) { - return obj; - } - } - - return null; - }, - - /** - * Add a 'model' to its appropriate collection. Retain the original contents of 'model.collection'. - * @param {Backbone.RelationalModel} model - */ - register: function( model ) { - var coll = this.getCollection( model ); - - if ( coll ) { - var modelColl = model.collection; - coll.add( model ); - this.listenTo( model, 'destroy', this.unregister, this ); - this.listenTo( model, 'relational:unregister', this.unregister, this ); - model.collection = modelColl; - } - }, - - /** - * Check if the given model may use the given `id` - * @param model - * @param [id] - */ - checkId: function( model, id ) { - var coll = this.getCollection( model ), - duplicate = coll && coll.get( id ); - - if ( duplicate && model !== duplicate ) { - if ( Backbone.Relational.showWarnings && typeof console !== 'undefined' ) { - console.warn( 'Duplicate id! Old RelationalModel=%o, new RelationalModel=%o', duplicate, model ); - } - - throw new Error( "Cannot instantiate more than one Backbone.RelationalModel with the same id per type!" ); - } - }, - - /** - * Explicitly update a model's id in its store collection - * @param {Backbone.RelationalModel} model - */ - update: function( model ) { - var coll = this.getCollection( model ); - // This triggers updating the lookup indices kept in a collection - coll._onModelEvent( 'change:' + model.idAttribute, model, coll ); - - // Trigger an event on model so related models (having the model's new id in their keyContents) can add it. - model.trigger( 'relational:change:id', model, coll ); - }, - - /** - * Remove a 'model' from the store. - * @param {Backbone.RelationalModel} model - */ - unregister: function( model, collection, options ) { - this.stopListening( model ); - var coll = this.getCollection( model ); - coll && coll.remove( model, options ); - }, - - /** - * Reset the `store` to it's original state. The `reverseRelations` are kept though, since attempting to - * re-initialize these on models would lead to a large amount of warnings. - */ - reset: function() { - this.stopListening(); - this._collections = []; - this._subModels = []; - this._modelScopes = [ exports ]; - } - }); - Backbone.Relational.store = new Backbone.Store(); - - /** - * The main Relation class, from which 'HasOne' and 'HasMany' inherit. Internally, 'relational:<key>' events - * are used to regulate addition and removal of models from relations. - * - * @param {Backbone.RelationalModel} [instance] Model that this relation is created for. If no model is supplied, - * Relation just tries to instantiate it's `reverseRelation` if specified, and bails out after that. - * @param {Object} options - * @param {string} options.key - * @param {Backbone.RelationalModel.constructor} options.relatedModel - * @param {Boolean|String} [options.includeInJSON=true] Serialize the given attribute for related model(s)' in toJSON, or just their ids. - * @param {Boolean} [options.createModels=true] Create objects from the contents of keys if the object is not found in Backbone.store. - * @param {Object} [options.reverseRelation] Specify a bi-directional relation. If provided, Relation will reciprocate - * the relation to the 'relatedModel'. Required and optional properties match 'options', except that it also needs - * {Backbone.Relation|String} type ('HasOne' or 'HasMany'). - * @param {Object} opts - */ - Backbone.Relation = function( instance, options, opts ) { - this.instance = instance; - // Make sure 'options' is sane, and fill with defaults from subclasses and this object's prototype - options = _.isObject( options ) ? options : {}; - this.reverseRelation = _.defaults( options.reverseRelation || {}, this.options.reverseRelation ); - this.options = _.defaults( options, this.options, Backbone.Relation.prototype.options ); - - this.reverseRelation.type = !_.isString( this.reverseRelation.type ) ? this.reverseRelation.type : - Backbone[ this.reverseRelation.type ] || Backbone.Relational.store.getObjectByName( this.reverseRelation.type ); - - this.key = this.options.key; - this.keySource = this.options.keySource || this.key; - this.keyDestination = this.options.keyDestination || this.keySource || this.key; - - this.model = this.options.model || this.instance.constructor; - this.relatedModel = this.options.relatedModel; - if ( _.isString( this.relatedModel ) ) { - this.relatedModel = Backbone.Relational.store.getObjectByName( this.relatedModel ); - } - - if ( !this.checkPreconditions() ) { - return; - } - - // Add the reverse relation on 'relatedModel' to the store's reverseRelations - if ( !this.options.isAutoRelation && this.reverseRelation.type && this.reverseRelation.key ) { - Backbone.Relational.store.addReverseRelation( _.defaults( { - isAutoRelation: true, - model: this.relatedModel, - relatedModel: this.model, - reverseRelation: this.options // current relation is the 'reverseRelation' for its own reverseRelation - }, - this.reverseRelation // Take further properties from this.reverseRelation (type, key, etc.) - ) ); - } - - if ( instance ) { - var contentKey = this.keySource; - if ( contentKey !== this.key && typeof this.instance.get( this.key ) === 'object' ) { - contentKey = this.key; - } - - this.setKeyContents( this.instance.get( contentKey ) ); - this.relatedCollection = Backbone.Relational.store.getCollection( this.relatedModel ); - - // Explicitly clear 'keySource', to prevent a leaky abstraction if 'keySource' differs from 'key'. - if ( this.keySource !== this.key ) { - delete this.instance.attributes[ this.keySource ]; - } - - // Add this Relation to instance._relations - this.instance._relations[ this.key ] = this; - - this.initialize( opts ); - - if ( this.options.autoFetch ) { - this.instance.fetchRelated( this.key, _.isObject( this.options.autoFetch ) ? this.options.autoFetch : {} ); - } - - // When 'relatedModel' are created or destroyed, check if it affects this relation. - this.listenTo( this.instance, 'destroy', this.destroy ) - .listenTo( this.relatedCollection, 'relational:add relational:change:id', this.tryAddRelated ) - .listenTo( this.relatedCollection, 'relational:remove', this.removeRelated ) - } - }; - // Fix inheritance :\ - Backbone.Relation.extend = Backbone.Model.extend; - // Set up all inheritable **Backbone.Relation** properties and methods. - _.extend( Backbone.Relation.prototype, Backbone.Events, Backbone.Semaphore, { - options: { - createModels: true, - includeInJSON: true, - isAutoRelation: false, - autoFetch: false, - parse: false - }, - - instance: null, - key: null, - keyContents: null, - relatedModel: null, - relatedCollection: null, - reverseRelation: null, - related: null, - - /** - * Check several pre-conditions. - * @return {Boolean} True if pre-conditions are satisfied, false if they're not. - */ - checkPreconditions: function() { - var i = this.instance, - k = this.key, - m = this.model, - rm = this.relatedModel, - warn = Backbone.Relational.showWarnings && typeof console !== 'undefined'; - - if ( !m || !k || !rm ) { - warn && console.warn( 'Relation=%o: missing model, key or relatedModel (%o, %o, %o).', this, m, k, rm ); - return false; - } - // Check if the type in 'model' inherits from Backbone.RelationalModel - if ( !( m.prototype instanceof Backbone.RelationalModel ) ) { - warn && console.warn( 'Relation=%o: model does not inherit from Backbone.RelationalModel (%o).', this, i ); - return false; - } - // Check if the type in 'relatedModel' inherits from Backbone.RelationalModel - if ( !( rm.prototype instanceof Backbone.RelationalModel ) ) { - warn && console.warn( 'Relation=%o: relatedModel does not inherit from Backbone.RelationalModel (%o).', this, rm ); - return false; - } - // Check if this is not a HasMany, and the reverse relation is HasMany as well - if ( this instanceof Backbone.HasMany && this.reverseRelation.type === Backbone.HasMany ) { - warn && console.warn( 'Relation=%o: relation is a HasMany, and the reverseRelation is HasMany as well.', this ); - return false; - } - // Check if we're not attempting to create a relationship on a `key` that's already used. - if ( i && _.keys( i._relations ).length ) { - var existing = _.find( i._relations, function( rel ) { - return rel.key === k; - }, this ); - - if ( existing ) { - warn && console.warn( 'Cannot create relation=%o on %o for model=%o: already taken by relation=%o.', - this, k, i, existing ); - return false; - } - } - - return true; - }, - - /** - * Set the related model(s) for this relation - * @param {Backbone.Model|Backbone.Collection} related - */ - setRelated: function( related ) { - this.related = related; - - this.instance.acquire(); - this.instance.attributes[ this.key ] = related; - this.instance.release(); - }, - - /** - * Determine if a relation (on a different RelationalModel) is the reverse - * relation of the current one. - * @param {Backbone.Relation} relation - * @return {Boolean} - */ - _isReverseRelation: function( relation ) { - return relation.instance instanceof this.relatedModel && this.reverseRelation.key === relation.key && - this.key === relation.reverseRelation.key; - }, - - /** - * Get the reverse relations (pointing back to 'this.key' on 'this.instance') for the currently related model(s). - * @param {Backbone.RelationalModel} [model] Get the reverse relations for a specific model. - * If not specified, 'this.related' is used. - * @return {Backbone.Relation[]} - */ - getReverseRelations: function( model ) { - var reverseRelations = []; - // Iterate over 'model', 'this.related.models' (if this.related is a Backbone.Collection), or wrap 'this.related' in an array. - var models = !_.isUndefined( model ) ? [ model ] : this.related && ( this.related.models || [ this.related ] ); - _.each( models || [], function( related ) { - _.each( related.getRelations() || [], function( relation ) { - if ( this._isReverseRelation( relation ) ) { - reverseRelations.push( relation ); - } - }, this ); - }, this ); - - return reverseRelations; - }, - - /** - * When `this.instance` is destroyed, cleanup our relations. - * Get reverse relation, call removeRelated on each. - */ - destroy: function() { - this.stopListening(); - - if ( this instanceof Backbone.HasOne ) { - this.setRelated( null ); - } - else if ( this instanceof Backbone.HasMany ) { - this.setRelated( this._prepareCollection() ); - } - - _.each( this.getReverseRelations(), function( relation ) { - relation.removeRelated( this.instance ); - }, this ); - } - }); - - Backbone.HasOne = Backbone.Relation.extend({ - options: { - reverseRelation: { type: 'HasMany' } - }, - - initialize: function( opts ) { - this.listenTo( this.instance, 'relational:change:' + this.key, this.onChange ); - - var related = this.findRelated( opts ); - this.setRelated( related ); - - // Notify new 'related' object of the new relation. - _.each( this.getReverseRelations(), function( relation ) { - relation.addRelated( this.instance, opts ); - }, this ); - }, - - /** - * Find related Models. - * @param {Object} [options] - * @return {Backbone.Model} - */ - findRelated: function( options ) { - var related = null; - - options = _.defaults( { parse: this.options.parse }, options ); - - if ( this.keyContents instanceof this.relatedModel ) { - related = this.keyContents; - } - else if ( this.keyContents || this.keyContents === 0 ) { // since 0 can be a valid `id` as well - var opts = _.defaults( { create: this.options.createModels }, options ); - related = this.relatedModel.findOrCreate( this.keyContents, opts ); - } - - // Nullify `keyId` if we have a related model; in case it was already part of the relation - if ( this.related ) { - this.keyId = null; - } - - return related; - }, - - /** - * Normalize and reduce `keyContents` to an `id`, for easier comparison - * @param {String|Number|Backbone.Model} keyContents - */ - setKeyContents: function( keyContents ) { - this.keyContents = keyContents; - this.keyId = Backbone.Relational.store.resolveIdForItem( this.relatedModel, this.keyContents ); - }, - - /** - * Event handler for `change:<key>`. - * If the key is changed, notify old & new reverse relations and initialize the new relation. - */ - onChange: function( model, attr, options ) { - // Don't accept recursive calls to onChange (like onChange->findRelated->findOrCreate->initializeRelations->addRelated->onChange) - if ( this.isLocked() ) { - return; - } - this.acquire(); - options = options ? _.clone( options ) : {}; - - // 'options.__related' is set by 'addRelated'/'removeRelated'. If it is set, the change - // is the result of a call from a relation. If it's not, the change is the result of - // a 'set' call on this.instance. - var changed = _.isUndefined( options.__related ), - oldRelated = changed ? this.related : options.__related; - - if ( changed ) { - this.setKeyContents( attr ); - var related = this.findRelated( options ); - this.setRelated( related ); - } - - // Notify old 'related' object of the terminated relation - if ( oldRelated && this.related !== oldRelated ) { - _.each( this.getReverseRelations( oldRelated ), function( relation ) { - relation.removeRelated( this.instance, null, options ); - }, this ); - } - - // Notify new 'related' object of the new relation. Note we do re-apply even if this.related is oldRelated; - // that can be necessary for bi-directional relations if 'this.instance' was created after 'this.related'. - // In that case, 'this.instance' will already know 'this.related', but the reverse might not exist yet. - _.each( this.getReverseRelations(), function( relation ) { - relation.addRelated( this.instance, options ); - }, this ); - - // Fire the 'change:<key>' event if 'related' was updated - if ( !options.silent && this.related !== oldRelated ) { - var dit = this; - this.changed = true; - Backbone.Relational.eventQueue.add( function() { - dit.instance.trigger( 'change:' + dit.key, dit.instance, dit.related, options, true ); - dit.changed = false; - }); - } - this.release(); - }, - - /** - * If a new 'this.relatedModel' appears in the 'store', try to match it to the last set 'keyContents' - */ - tryAddRelated: function( model, coll, options ) { - if ( ( this.keyId || this.keyId === 0 ) && model.id === this.keyId ) { // since 0 can be a valid `id` as well - this.addRelated( model, options ); - this.keyId = null; - } - }, - - addRelated: function( model, options ) { - // Allow 'model' to set up its relations before proceeding. - // (which can result in a call to 'addRelated' from a relation of 'model') - var dit = this; - model.queue( function() { - if ( model !== dit.related ) { - var oldRelated = dit.related || null; - dit.setRelated( model ); - dit.onChange( dit.instance, model, _.defaults( { __related: oldRelated }, options ) ); - } - }); - }, - - removeRelated: function( model, coll, options ) { - if ( !this.related ) { - return; - } - - if ( model === this.related ) { - var oldRelated = this.related || null; - this.setRelated( null ); - this.onChange( this.instance, model, _.defaults( { __related: oldRelated }, options ) ); - } - } - }); - - Backbone.HasMany = Backbone.Relation.extend({ - collectionType: null, - - options: { - reverseRelation: { type: 'HasOne' }, - collectionType: Backbone.Collection, - collectionKey: true, - collectionOptions: {} - }, - - initialize: function( opts ) { - this.listenTo( this.instance, 'relational:change:' + this.key, this.onChange ); - - // Handle a custom 'collectionType' - this.collectionType = this.options.collectionType; - if ( _.isString( this.collectionType ) ) { - this.collectionType = Backbone.Relational.store.getObjectByName( this.collectionType ); - } - if ( this.collectionType !== Backbone.Collection && !( this.collectionType.prototype instanceof Backbone.Collection ) ) { - throw new Error( '`collectionType` must inherit from Backbone.Collection' ); - } - - var related = this.findRelated( opts ); - this.setRelated( related ); - }, - - /** - * Bind events and setup collectionKeys for a collection that is to be used as the backing store for a HasMany. - * If no 'collection' is supplied, a new collection will be created of the specified 'collectionType' option. - * @param {Backbone.Collection} [collection] - * @return {Backbone.Collection} - */ - _prepareCollection: function( collection ) { - if ( this.related ) { - this.stopListening( this.related ); - } - - if ( !collection || !( collection instanceof Backbone.Collection ) ) { - var options = _.isFunction( this.options.collectionOptions ) ? - this.options.collectionOptions( this.instance ) : this.options.collectionOptions; - - collection = new this.collectionType( null, options ); - } - - collection.model = this.relatedModel; - - if ( this.options.collectionKey ) { - var key = this.options.collectionKey === true ? this.options.reverseRelation.key : this.options.collectionKey; - - if ( collection[ key ] && collection[ key ] !== this.instance ) { - if ( Backbone.Relational.showWarnings && typeof console !== 'undefined' ) { - console.warn( 'Relation=%o; collectionKey=%s already exists on collection=%o', this, key, this.options.collectionKey ); - } - } - else if ( key ) { - collection[ key ] = this.instance; - } - } - - this.listenTo( collection, 'relational:add', this.handleAddition ) - .listenTo( collection, 'relational:remove', this.handleRemoval ) - .listenTo( collection, 'relational:reset', this.handleReset ); - - return collection; - }, - - /** - * Find related Models. - * @param {Object} [options] - * @return {Backbone.Collection} - */ - findRelated: function( options ) { - var related = null; - - options = _.defaults( { parse: this.options.parse }, options ); - - // Replace 'this.related' by 'this.keyContents' if it is a Backbone.Collection - if ( this.keyContents instanceof Backbone.Collection ) { - this._prepareCollection( this.keyContents ); - related = this.keyContents; - } - // Otherwise, 'this.keyContents' should be an array of related object ids. - // Re-use the current 'this.related' if it is a Backbone.Collection; otherwise, create a new collection. - else { - var toAdd = []; - - _.each( this.keyContents, function( attributes ) { - if ( attributes instanceof this.relatedModel ) { - var model = attributes; - } - else { - // If `merge` is true, update models here, instead of during update. - model = this.relatedModel.findOrCreate( attributes, - _.extend( { merge: true }, options, { create: this.options.createModels } ) - ); - } - - model && toAdd.push( model ); - }, this ); - - if ( this.related instanceof Backbone.Collection ) { - related = this.related; - } - else { - related = this._prepareCollection(); - } - - // By now, both `merge` and `parse` will already have been executed for models if they were specified. - // Disable them to prevent additional calls. - related.set( toAdd, _.defaults( { merge: false, parse: false }, options ) ); - } - - // Remove entries from `keyIds` that were already part of the relation (and are thus 'unchanged') - this.keyIds = _.difference( this.keyIds, _.pluck( related.models, 'id' ) ); - - return related; - }, - - /** - * Normalize and reduce `keyContents` to a list of `ids`, for easier comparison - * @param {String|Number|String[]|Number[]|Backbone.Collection} keyContents - */ - setKeyContents: function( keyContents ) { - this.keyContents = keyContents instanceof Backbone.Collection ? keyContents : null; - this.keyIds = []; - - if ( !this.keyContents && ( keyContents || keyContents === 0 ) ) { // since 0 can be a valid `id` as well - // Handle cases the an API/user supplies just an Object/id instead of an Array - this.keyContents = _.isArray( keyContents ) ? keyContents : [ keyContents ]; - - _.each( this.keyContents, function( item ) { - var itemId = Backbone.Relational.store.resolveIdForItem( this.relatedModel, item ); - if ( itemId || itemId === 0 ) { - this.keyIds.push( itemId ); - } - }, this ); - } - }, - - /** - * Event handler for `change:<key>`. - * If the contents of the key are changed, notify old & new reverse relations and initialize the new relation. - */ - onChange: function( model, attr, options ) { - options = options ? _.clone( options ) : {}; - this.setKeyContents( attr ); - this.changed = false; - - var related = this.findRelated( options ); - this.setRelated( related ); - - if ( !options.silent ) { - var dit = this; - Backbone.Relational.eventQueue.add( function() { - // The `changed` flag can be set in `handleAddition` or `handleRemoval` - if ( dit.changed ) { - dit.instance.trigger( 'change:' + dit.key, dit.instance, dit.related, options, true ); - dit.changed = false; - } - }); - } - }, - - /** - * When a model is added to a 'HasMany', trigger 'add' on 'this.instance' and notify reverse relations. - * (should be 'HasOne', must set 'this.instance' as their related). - */ - handleAddition: function( model, coll, options ) { - //console.debug('handleAddition called; args=%o', arguments); - options = options ? _.clone( options ) : {}; - this.changed = true; - - _.each( this.getReverseRelations( model ), function( relation ) { - relation.addRelated( this.instance, options ); - }, this ); - - // Only trigger 'add' once the newly added model is initialized (so, has its relations set up) - var dit = this; - !options.silent && Backbone.Relational.eventQueue.add( function() { - dit.instance.trigger( 'add:' + dit.key, model, dit.related, options ); - }); - }, - - /** - * When a model is removed from a 'HasMany', trigger 'remove' on 'this.instance' and notify reverse relations. - * (should be 'HasOne', which should be nullified) - */ - handleRemoval: function( model, coll, options ) { - //console.debug('handleRemoval called; args=%o', arguments); - options = options ? _.clone( options ) : {}; - this.changed = true; - - _.each( this.getReverseRelations( model ), function( relation ) { - relation.removeRelated( this.instance, null, options ); - }, this ); - - var dit = this; - !options.silent && Backbone.Relational.eventQueue.add( function() { - dit.instance.trigger( 'remove:' + dit.key, model, dit.related, options ); - }); - }, - - handleReset: function( coll, options ) { - var dit = this; - options = options ? _.clone( options ) : {}; - !options.silent && Backbone.Relational.eventQueue.add( function() { - dit.instance.trigger( 'reset:' + dit.key, dit.related, options ); - }); - }, - - tryAddRelated: function( model, coll, options ) { - var item = _.contains( this.keyIds, model.id ); - - if ( item ) { - this.addRelated( model, options ); - this.keyIds = _.without( this.keyIds, model.id ); - } - }, - - addRelated: function( model, options ) { - // Allow 'model' to set up its relations before proceeding. - // (which can result in a call to 'addRelated' from a relation of 'model') - var dit = this; - model.queue( function() { - if ( dit.related && !dit.related.get( model ) ) { - dit.related.add( model, _.defaults( { parse: false }, options ) ); - } - }); - }, - - removeRelated: function( model, coll, options ) { - if ( this.related.get( model ) ) { - this.related.remove( model, options ); - } - } - }); - - /** - * A type of Backbone.Model that also maintains relations to other models and collections. - * New events when compared to the original: - * - 'add:<key>' (model, related collection, options) - * - 'remove:<key>' (model, related collection, options) - * - 'change:<key>' (model, related model or collection, options) - */ - Backbone.RelationalModel = Backbone.Model.extend({ - relations: null, // Relation descriptions on the prototype - _relations: null, // Relation instances - _isInitialized: false, - _deferProcessing: false, - _queue: null, - _attributeChangeFired: false, // Keeps track of `change` event firing under some conditions (like nested `set`s) - - subModelTypeAttribute: 'type', - subModelTypes: null, - - constructor: function( attributes, options ) { - // Nasty hack, for cases like 'model.get( <HasMany key> ).add( item )'. - // Defer 'processQueue', so that when 'Relation.createModels' is used we trigger 'HasMany' - // collection events only after the model is really fully set up. - // Example: event for "p.on( 'add:jobs' )" -> "p.get('jobs').add( { company: c.id, person: p.id } )". - if ( options && options.collection ) { - var dit = this, - collection = this.collection = options.collection; - - // Prevent `collection` from cascading down to nested models; they shouldn't go into this `if` clause. - delete options.collection; - - this._deferProcessing = true; - - var processQueue = function( model ) { - if ( model === dit ) { - dit._deferProcessing = false; - dit.processQueue(); - collection.off( 'relational:add', processQueue ); - } - }; - collection.on( 'relational:add', processQueue ); - - // So we do process the queue eventually, regardless of whether this model actually gets added to 'options.collection'. - _.defer( function() { - processQueue( dit ); - }); - } - - Backbone.Relational.store.processOrphanRelations(); - - this._queue = new Backbone.BlockingQueue(); - this._queue.block(); - Backbone.Relational.eventQueue.block(); - - try { - Backbone.Model.apply( this, arguments ); - } - finally { - // Try to run the global queue holding external events - Backbone.Relational.eventQueue.unblock(); - } - }, - - /** - * Override 'trigger' to queue 'change' and 'change:*' events - */ - trigger: function( eventName ) { - if ( eventName.length > 5 && eventName.indexOf( 'change' ) === 0 ) { - var dit = this, - args = arguments; - - Backbone.Relational.eventQueue.add( function() { - if ( !dit._isInitialized ) { - return; - } - - // Determine if the `change` event is still valid, now that all relations are populated - var changed = true; - if ( eventName === 'change' ) { - // `hasChanged` may have gotten reset by nested calls to `set`. - changed = dit.hasChanged() || dit._attributeChangeFired; - dit._attributeChangeFired = false; - } - else { - var attr = eventName.slice( 7 ), - rel = dit.getRelation( attr ); - - if ( rel ) { - // If `attr` is a relation, `change:attr` get triggered from `Relation.onChange`. - // These take precedence over `change:attr` events triggered by `Model.set`. - // The relation sets a fourth attribute to `true`. If this attribute is present, - // continue triggering this event; otherwise, it's from `Model.set` and should be stopped. - changed = ( args[ 4 ] === true ); - - // If this event was triggered by a relation, set the right value in `this.changed` - // (a Collection or Model instead of raw data). - if ( changed ) { - dit.changed[ attr ] = args[ 2 ]; - } - // Otherwise, this event is from `Model.set`. If the relation doesn't report a change, - // remove attr from `dit.changed` so `hasChanged` doesn't take it into account. - else if ( !rel.changed ) { - delete dit.changed[ attr ]; - } - } - else if ( changed ) { - dit._attributeChangeFired = true; - } - } - - changed && Backbone.Model.prototype.trigger.apply( dit, args ); - }); - } - else { - Backbone.Model.prototype.trigger.apply( this, arguments ); - } - - return this; - }, - - /** - * Initialize Relations present in this.relations; determine the type (HasOne/HasMany), then creates a new instance. - * Invoked in the first call so 'set' (which is made from the Backbone.Model constructor). - */ - initializeRelations: function( options ) { - this.acquire(); // Setting up relations often also involve calls to 'set', and we only want to enter this function once - this._relations = {}; - - _.each( _.result(this, 'relations') || [], function( rel ) { - Backbone.Relational.store.initializeRelation( this, rel, options ); - }, this ); - - this._isInitialized = true; - this.release(); - this.processQueue(); - }, - - /** - * When new values are set, notify this model's relations (also if options.silent is set). - * (Relation.setRelated locks this model before calling 'set' on it to prevent loops) - */ - updateRelations: function( options ) { - if ( this._isInitialized && !this.isLocked() ) { - _.each( this._relations, function( rel ) { - // Update from data in `rel.keySource` if data got set in there, or `rel.key` otherwise - var val = this.attributes[ rel.keySource ] || this.attributes[ rel.key ]; - if ( rel.related !== val ) { - this.trigger( 'relational:change:' + rel.key, this, val, options || {} ); - } - - // Explicitly clear 'keySource', to prevent a leaky abstraction if 'keySource' differs from 'key'. - if ( rel.keySource !== rel.key ) { - delete rel.instance.attributes[ rel.keySource ]; - } - }, this ); - } - }, - - /** - * Either add to the queue (if we're not initialized yet), or execute right away. - */ - queue: function( func ) { - this._queue.add( func ); - }, - - /** - * Process _queue - */ - processQueue: function() { - if ( this._isInitialized && !this._deferProcessing && this._queue.isBlocked() ) { - this._queue.unblock(); - } - }, - - /** - * Get a specific relation. - * @param key {string} The relation key to look for. - * @return {Backbone.Relation} An instance of 'Backbone.Relation', if a relation was found for 'key', or null. - */ - getRelation: function( key ) { - return this._relations[ key ]; - }, - - /** - * Get all of the created relations. - * @return {Backbone.Relation[]} - */ - getRelations: function() { - return _.values( this._relations ); - }, - - /** - * Retrieve related objects. - * @param key {string} The relation key to fetch models for. - * @param [options] {Object} Options for 'Backbone.Model.fetch' and 'Backbone.sync'. - * @param [refresh=false] {boolean} Fetch existing models from the server as well (in order to update them). - * @return {jQuery.when[]} An array of request objects - */ - fetchRelated: function( key, options, refresh ) { - // Set default `options` for fetch - options = _.extend( { update: true, remove: false }, options ); - - var setUrl, - requests = [], - rel = this.getRelation( key ), - idsToFetch = rel && ( ( rel.keyIds && rel.keyIds.slice( 0 ) ) || ( ( rel.keyId || rel.keyId === 0 ) ? [ rel.keyId ] : [] ) ); - - // On `refresh`, add the ids for current models in the relation to `idsToFetch` - if ( refresh ) { - var models = rel.related instanceof Backbone.Collection ? rel.related.models : [ rel.related ]; - _.each( models, function( model ) { - if ( model.id || model.id === 0 ) { - idsToFetch.push( model.id ); - } - }); - } - - if ( idsToFetch && idsToFetch.length ) { - // Find (or create) a model for each one that is to be fetched - var created = [], - models = _.map( idsToFetch, function( id ) { - var model = Backbone.Relational.store.find( rel.relatedModel, id ); - - if ( !model ) { - var attrs = {}; - attrs[ rel.relatedModel.prototype.idAttribute ] = id; - model = rel.relatedModel.findOrCreate( attrs, options ); - created.push( model ); - } - - return model; - }, this ); - - // Try if the 'collection' can provide a url to fetch a set of models in one request. - if ( rel.related instanceof Backbone.Collection && _.isFunction( rel.related.url ) ) { - setUrl = rel.related.url( models ); - } - - // An assumption is that when 'Backbone.Collection.url' is a function, it can handle building of set urls. - // To make sure it can, test if the url we got by supplying a list of models to fetch is different from - // the one supplied for the default fetch action (without args to 'url'). - if ( setUrl && setUrl !== rel.related.url() ) { - var opts = _.defaults( - { - error: function() { - var args = arguments; - _.each( created, function( model ) { - model.trigger( 'destroy', model, model.collection, options ); - options.error && options.error.apply( model, args ); - }); - }, - url: setUrl - }, - options - ); - - requests = [ rel.related.fetch( opts ) ]; - } - else { - requests = _.map( models, function( model ) { - var opts = _.defaults( - { - error: function() { - if ( _.contains( created, model ) ) { - model.trigger( 'destroy', model, model.collection, options ); - options.error && options.error.apply( model, arguments ); - } - } - }, - options - ); - return model.fetch( opts ); - }, this ); - } - } - - return requests; - }, - - get: function( attr ) { - var originalResult = Backbone.Model.prototype.get.call( this, attr ); - - // Use `originalResult` get if dotNotation not enabled or not required because no dot is in `attr` - if ( !this.dotNotation || attr.indexOf( '.' ) === -1 ) { - return originalResult; - } - - // Go through all splits and return the final result - var splits = attr.split( '.' ); - var result = _.reduce(splits, function( model, split ) { - if ( _.isNull(model) || _.isUndefined( model ) ) { - // Return undefined if the path cannot be expanded - return undefined; - } - if ( !( model instanceof Backbone.Model ) ) { - throw new Error( 'Attribute must be an instanceof Backbone.Model. Is: ' + model + ', currentSplit: ' + split ); - } - - return Backbone.Model.prototype.get.call( model, split ); - }, this ); - - if ( originalResult !== undefined && result !== undefined ) { - throw new Error( "Ambiguous result for '" + attr + "'. direct result: " + originalResult + ", dotNotation: " + result ); - } - - return originalResult || result; - }, - - set: function( key, value, options ) { - Backbone.Relational.eventQueue.block(); - - // Duplicate backbone's behavior to allow separate key/value parameters, instead of a single 'attributes' object - var attributes; - if ( _.isObject( key ) || key == null ) { - attributes = key; - options = value; - } - else { - attributes = {}; - attributes[ key ] = value; - } - - try { - var id = this.id, - newId = attributes && this.idAttribute in attributes && attributes[ this.idAttribute ]; - - // Check if we're not setting a duplicate id before actually calling `set`. - Backbone.Relational.store.checkId( this, newId ); - - var result = Backbone.Model.prototype.set.apply( this, arguments ); - - // Ideal place to set up relations, if this is the first time we're here for this model - if ( !this._isInitialized && !this.isLocked() ) { - this.constructor.initializeModelHierarchy(); - Backbone.Relational.store.register( this ); - this.initializeRelations( options ); - } - // The store should know about an `id` update asap - else if ( newId && newId !== id ) { - Backbone.Relational.store.update( this ); - } - - if ( attributes ) { - this.updateRelations( options ); - } - } - finally { - // Try to run the global queue holding external events - Backbone.Relational.eventQueue.unblock(); - } - - return result; - }, - - clone: function() { - var attributes = _.clone( this.attributes ); - if ( !_.isUndefined( attributes[ this.idAttribute ] ) ) { - attributes[ this.idAttribute ] = null; - } - - _.each( this.getRelations(), function( rel ) { - delete attributes[ rel.key ]; - }); - - return new this.constructor( attributes ); - }, - - /** - * Convert relations to JSON, omits them when required - */ - toJSON: function( options ) { - // If this Model has already been fully serialized in this branch once, return to avoid loops - if ( this.isLocked() ) { - return this.id; - } - - this.acquire(); - var json = Backbone.Model.prototype.toJSON.call( this, options ); - - if ( this.constructor._superModel && !( this.constructor._subModelTypeAttribute in json ) ) { - json[ this.constructor._subModelTypeAttribute ] = this.constructor._subModelTypeValue; - } - - _.each( this._relations, function( rel ) { - var related = json[ rel.key ], - includeInJSON = rel.options.includeInJSON, - value = null; - - if ( includeInJSON === true ) { - if ( related && _.isFunction( related.toJSON ) ) { - value = related.toJSON( options ); - } - } - else if ( _.isString( includeInJSON ) ) { - if ( related instanceof Backbone.Collection ) { - value = related.pluck( includeInJSON ); - } - else if ( related instanceof Backbone.Model ) { - value = related.get( includeInJSON ); - } - - // Add ids for 'unfound' models if includeInJSON is equal to (only) the relatedModel's `idAttribute` - if ( includeInJSON === rel.relatedModel.prototype.idAttribute ) { - if ( rel instanceof Backbone.HasMany ) { - value = value.concat( rel.keyIds ); - } - else if ( rel instanceof Backbone.HasOne ) { - value = value || rel.keyId; - } - } - } - else if ( _.isArray( includeInJSON ) ) { - if ( related instanceof Backbone.Collection ) { - value = []; - related.each( function( model ) { - var curJson = {}; - _.each( includeInJSON, function( key ) { - curJson[ key ] = model.get( key ); - }); - value.push( curJson ); - }); - } - else if ( related instanceof Backbone.Model ) { - value = {}; - _.each( includeInJSON, function( key ) { - value[ key ] = related.get( key ); - }); - } - } - else { - delete json[ rel.key ]; - } - - if ( includeInJSON ) { - json[ rel.keyDestination ] = value; - } - - if ( rel.keyDestination !== rel.key ) { - delete json[ rel.key ]; - } - }); - - this.release(); - return json; - } - }, - { - /** - * - * @param superModel - * @returns {Backbone.RelationalModel.constructor} - */ - setup: function( superModel ) { - // We don't want to share a relations array with a parent, as this will cause problems with - // reverse relations. - this.prototype.relations = ( this.prototype.relations || [] ).slice( 0 ); - - this._subModels = {}; - this._superModel = null; - - // If this model has 'subModelTypes' itself, remember them in the store - if ( this.prototype.hasOwnProperty( 'subModelTypes' ) ) { - Backbone.Relational.store.addSubModels( this.prototype.subModelTypes, this ); - } - // The 'subModelTypes' property should not be inherited, so reset it. - else { - this.prototype.subModelTypes = null; - } - - // Initialize all reverseRelations that belong to this new model. - _.each( this.prototype.relations || [], function( rel ) { - if ( !rel.model ) { - rel.model = this; - } - - if ( rel.reverseRelation && rel.model === this ) { - var preInitialize = true; - if ( _.isString( rel.relatedModel ) ) { - /** - * The related model might not be defined for two reasons - * 1. it is related to itself - * 2. it never gets defined, e.g. a typo - * 3. the model hasn't been defined yet, but will be later - * In neither of these cases do we need to pre-initialize reverse relations. - * However, for 3. (which is, to us, indistinguishable from 2.), we do need to attempt - * setting up this relation again later, in case the related model is defined later. - */ - var relatedModel = Backbone.Relational.store.getObjectByName( rel.relatedModel ); - preInitialize = relatedModel && ( relatedModel.prototype instanceof Backbone.RelationalModel ); - } - - if ( preInitialize ) { - Backbone.Relational.store.initializeRelation( null, rel ); - } - else if ( _.isString( rel.relatedModel ) ) { - Backbone.Relational.store.addOrphanRelation( rel ); - } - } - }, this ); - - return this; - }, - - /** - * Create a 'Backbone.Model' instance based on 'attributes'. - * @param {Object} attributes - * @param {Object} [options] - * @return {Backbone.Model} - */ - build: function( attributes, options ) { - // 'build' is a possible entrypoint; it's possible no model hierarchy has been determined yet. - this.initializeModelHierarchy(); - - // Determine what type of (sub)model should be built if applicable. - var model = this._findSubModelType(this, attributes) || this; - - return new model( attributes, options ); - }, - - /** - * Determines what type of (sub)model should be built if applicable. - * Looks up the proper subModelType in 'this._subModels', recursing into - * types until a match is found. Returns the applicable 'Backbone.Model' - * or null if no match is found. - * @param {Backbone.Model} type - * @param {Object} attributes - * @return {Backbone.Model} - */ - _findSubModelType: function (type, attributes) { - if ( type._subModels && type.prototype.subModelTypeAttribute in attributes ) { - var subModelTypeAttribute = attributes[type.prototype.subModelTypeAttribute]; - var subModelType = type._subModels[subModelTypeAttribute]; - if ( subModelType ) { - return subModelType; - } else { - // Recurse into subModelTypes to find a match - for ( subModelTypeAttribute in type._subModels ) { - subModelType = this._findSubModelType(type._subModels[subModelTypeAttribute], attributes); - if ( subModelType ) { - return subModelType; - } - } - } - } - return null; - }, - - /** - * - */ - initializeModelHierarchy: function() { - // Inherit any relations that have been defined in the parent model. - this.inheritRelations(); - - // If we came here through 'build' for a model that has 'subModelTypes' then try to initialize the ones that - // haven't been resolved yet. - if ( this.prototype.subModelTypes ) { - var resolvedSubModels = _.keys(this._subModels); - var unresolvedSubModels = _.omit(this.prototype.subModelTypes, resolvedSubModels); - _.each( unresolvedSubModels, function( subModelTypeName ) { - var subModelType = Backbone.Relational.store.getObjectByName( subModelTypeName ); - subModelType && subModelType.initializeModelHierarchy(); - }); - } - }, - - inheritRelations: function() { - // Bail out if we've been here before. - if (!_.isUndefined( this._superModel ) && !_.isNull( this._superModel )) { - return; - } - // Try to initialize the _superModel. - Backbone.Relational.store.setupSuperModel( this ); - - // If a superModel has been found, copy relations from the _superModel if they haven't been inherited automatically - // (due to a redefinition of 'relations'). - if ( this._superModel ) { - // The _superModel needs a chance to initialize its own inherited relations before we attempt to inherit relations - // from the _superModel. You don't want to call 'initializeModelHierarchy' because that could cause sub-models of - // this class to inherit their relations before this class has had chance to inherit it's relations. - this._superModel.inheritRelations(); - if ( this._superModel.prototype.relations ) { - // Find relations that exist on the '_superModel', but not yet on this model. - var inheritedRelations = _.select( this._superModel.prototype.relations || [], function( superRel ) { - return !_.any( this.prototype.relations || [], function( rel ) { - return superRel.relatedModel === rel.relatedModel && superRel.key === rel.key; - }, this ); - }, this ); - - this.prototype.relations = inheritedRelations.concat( this.prototype.relations ); - } - } - // Otherwise, make sure we don't get here again for this type by making '_superModel' false so we fail the - // isUndefined/isNull check next time. - else { - this._superModel = false; - } - }, - - /** - * Find an instance of `this` type in 'Backbone.Relational.store'. - * - If `attributes` is a string or a number, `findOrCreate` will just query the `store` and return a model if found. - * - If `attributes` is an object and is found in the store, the model will be updated with `attributes` unless `options.update` is `false`. - * Otherwise, a new model is created with `attributes` (unless `options.create` is explicitly set to `false`). - * @param {Object|String|Number} attributes Either a model's id, or the attributes used to create or update a model. - * @param {Object} [options] - * @param {Boolean} [options.create=true] - * @param {Boolean} [options.merge=true] - * @param {Boolean} [options.parse=false] - * @return {Backbone.RelationalModel} - */ - findOrCreate: function( attributes, options ) { - options || ( options = {} ); - var parsedAttributes = ( _.isObject( attributes ) && options.parse && this.prototype.parse ) ? - this.prototype.parse( _.clone( attributes ) ) : attributes; - - // Try to find an instance of 'this' model type in the store - var model = Backbone.Relational.store.find( this, parsedAttributes ); - - // If we found an instance, update it with the data in 'item' (unless 'options.merge' is false). - // If not, create an instance (unless 'options.create' is false). - if ( _.isObject( attributes ) ) { - if ( model && options.merge !== false ) { - // Make sure `options.collection` and `options.url` doesn't cascade to nested models - delete options.collection; - delete options.url; - - model.set( parsedAttributes, options ); - } - else if ( !model && options.create !== false ) { - model = this.build( attributes, options ); - } - } - - return model; - }, - - /** - * Find an instance of `this` type in 'Backbone.Relational.store'. - * - If `attributes` is a string or a number, `find` will just query the `store` and return a model if found. - * - If `attributes` is an object and is found in the store, the model will be updated with `attributes` unless `options.update` is `false`. - * @param {Object|String|Number} attributes Either a model's id, or the attributes used to create or update a model. - * @param {Object} [options] - * @param {Boolean} [options.merge=true] - * @param {Boolean} [options.parse=false] - * @return {Backbone.RelationalModel} - */ - find: function( attributes, options ) { - options || ( options = {} ); - options.create = false; - return this.findOrCreate( attributes, options ); - } - }); - _.extend( Backbone.RelationalModel.prototype, Backbone.Semaphore ); - - /** - * Override Backbone.Collection._prepareModel, so objects will be built using the correct type - * if the collection.model has subModels. - * Attempts to find a model for `attrs` in Backbone.store through `findOrCreate` - * (which sets the new properties on it if found), or instantiates a new model. - */ - Backbone.Collection.prototype.__prepareModel = Backbone.Collection.prototype._prepareModel; - Backbone.Collection.prototype._prepareModel = function ( attrs, options ) { - var model; - - if ( attrs instanceof Backbone.Model ) { - if ( !attrs.collection ) { - attrs.collection = this; - } - model = attrs; - } - else { - options || ( options = {} ); - options.collection = this; - - if ( typeof this.model.findOrCreate !== 'undefined' ) { - model = this.model.findOrCreate( attrs, options ); - } - else { - model = new this.model( attrs, options ); - } - - if ( model && model.isNew() && !model._validate( attrs, options ) ) { - this.trigger( 'invalid', this, attrs, options ); - model = false; - } - } - - return model; - }; - - - /** - * Override Backbone.Collection.set, so we'll create objects from attributes where required, - * and update the existing models. Also, trigger 'relational:add'. - */ - var set = Backbone.Collection.prototype.__set = Backbone.Collection.prototype.set; - Backbone.Collection.prototype.set = function( models, options ) { - // Short-circuit if this Collection doesn't hold RelationalModels - if ( !( this.model.prototype instanceof Backbone.RelationalModel ) ) { - return set.apply( this, arguments ); - } - - if ( options && options.parse ) { - models = this.parse( models, options ); - } - - if ( !_.isArray( models ) ) { - models = models ? [ models ] : []; - } - - var newModels = [], - toAdd = []; - - //console.debug( 'calling add on coll=%o; model=%o, options=%o', this, models, options ); - _.each( models, function( model ) { - if ( !( model instanceof Backbone.Model ) ) { - model = Backbone.Collection.prototype._prepareModel.call( this, model, options ); - } - - if ( model ) { - toAdd.push( model ); - - if ( !( this.get( model ) || this.get( model.cid ) ) ) { - newModels.push( model ); - } - // If we arrive in `add` while performing a `set` (after a create, so the model gains an `id`), - // we may get here before `_onModelEvent` has had the chance to update `_byId`. - else if ( model.id != null ) { - this._byId[ model.id ] = model; - } - } - }, this ); - - // Add 'models' in a single batch, so the original add will only be called once (and thus 'sort', etc). - // If `parse` was specified, the collection and contained models have been parsed now. - set.call( this, toAdd, _.defaults( { parse: false }, options ) ); - - _.each( newModels, function( model ) { - // Fire a `relational:add` event for any model in `newModels` that has actually been added to the collection. - if ( this.get( model ) || this.get( model.cid ) ) { - this.trigger( 'relational:add', model, this, options ); - } - }, this ); - - return this; - }; - - /** - * Override 'Backbone.Collection.remove' to trigger 'relational:remove'. - */ - var remove = Backbone.Collection.prototype.__remove = Backbone.Collection.prototype.remove; - Backbone.Collection.prototype.remove = function( models, options ) { - // Short-circuit if this Collection doesn't hold RelationalModels - if ( !( this.model.prototype instanceof Backbone.RelationalModel ) ) { - return remove.apply( this, arguments ); - } - - models = _.isArray( models ) ? models.slice( 0 ) : [ models ]; - options || ( options = {} ); - - var toRemove = []; - - //console.debug('calling remove on coll=%o; models=%o, options=%o', this, models, options ); - _.each( models, function( model ) { - model = this.get( model ) || ( model && this.get( model.cid ) ); - model && toRemove.push( model ); - }, this ); - - if ( toRemove.length ) { - remove.call( this, toRemove, options ); - - _.each( toRemove, function( model ) { - this.trigger('relational:remove', model, this, options); - }, this ); - } - - return this; - }; - - /** - * Override 'Backbone.Collection.reset' to trigger 'relational:reset'. - */ - var reset = Backbone.Collection.prototype.__reset = Backbone.Collection.prototype.reset; - Backbone.Collection.prototype.reset = function( models, options ) { - options = _.extend( { merge: true }, options ); - reset.call( this, models, options ); - - if ( this.model.prototype instanceof Backbone.RelationalModel ) { - this.trigger( 'relational:reset', this, options ); - } - - return this; - }; - - /** - * Override 'Backbone.Collection.sort' to trigger 'relational:reset'. - */ - var sort = Backbone.Collection.prototype.__sort = Backbone.Collection.prototype.sort; - Backbone.Collection.prototype.sort = function( options ) { - sort.call( this, options ); - - if ( this.model.prototype instanceof Backbone.RelationalModel ) { - this.trigger( 'relational:reset', this, options ); - } - - return this; - }; - - /** - * Override 'Backbone.Collection.trigger' so 'add', 'remove' and 'reset' events are queued until relations - * are ready. - */ - var trigger = Backbone.Collection.prototype.__trigger = Backbone.Collection.prototype.trigger; - Backbone.Collection.prototype.trigger = function( eventName ) { - // Short-circuit if this Collection doesn't hold RelationalModels - if ( !( this.model.prototype instanceof Backbone.RelationalModel ) ) { - return trigger.apply( this, arguments ); - } - - if ( eventName === 'add' || eventName === 'remove' || eventName === 'reset' || eventName === 'sort' ) { - var dit = this, - args = arguments; - - if ( _.isObject( args[ 3 ] ) ) { - args = _.toArray( args ); - // the fourth argument is the option object. - // we need to clone it, as it could be modified while we wait on the eventQueue to be unblocked - args[ 3 ] = _.clone( args[ 3 ] ); - } - - Backbone.Relational.eventQueue.add( function() { - trigger.apply( dit, args ); - }); - } - else { - trigger.apply( this, arguments ); - } - - return this; - }; - - // Override .extend() to automatically call .setup() - Backbone.RelationalModel.extend = function( protoProps, classProps ) { - var child = Backbone.Model.extend.apply( this, arguments ); - - child.setup( this ); - - return child; - }; -})(); diff -r a58f6ca10843c91727807e7150b549ecd9b13ec5 -r f4832c4ace99f24bc42dcf275b55c0b2671fb6be static/scripts/mvc/base-mvc.js --- a/static/scripts/mvc/base-mvc.js +++ b/static/scripts/mvc/base-mvc.js @@ -2,7 +2,7 @@ * Simple base model for any visible element. Includes useful attributes and ability * to set and track visibility. */ -var BaseModel = Backbone.RelationalModel.extend({ +var BaseModel = Backbone.Model.extend({ defaults: { name: null, hidden: false This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: guerler: Upload: Backup id of current history
by commits-noreply＠bitbucket.org 26 Nov '13

26 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a58f6ca10843/ Changeset: a58f6ca10843 User: guerler Date: 2013-11-26 18:52:51 Summary: Upload: Backup id of current history Affected #: 1 file diff -r 9f31145b33ceb83e539bcdd571b48ff440b4c1eb -r a58f6ca10843c91727807e7150b549ecd9b13ec5 static/scripts/galaxy.upload.js --- a/static/scripts/galaxy.upload.js +++ b/static/scripts/galaxy.upload.js @@ -17,6 +17,9 @@ // upload mod uploadbox: null, + // current history + current_history: null, + // extension types select_extension :[['Auto-detect', 'auto']], @@ -227,7 +230,6 @@ sy.addClass(this.state.running); // get configuration - var current_history = Galaxy.currHistoryPanel.model.get('id'); var file_type = it.find('#extension').val(); var genome = it.find('#genome').val(); var url_paste = it.find('#text-content').val(); @@ -251,7 +253,7 @@ // setup data data = {}; - data['history_id'] = current_history; + data['history_id'] = this.current_history; data['tool_id'] = 'upload1'; data['inputs'] = JSON.stringify(tool_input); @@ -375,6 +377,9 @@ } }); + // backup current history + this.current_history = Galaxy.currHistoryPanel.model.get('id'); + // update running this.counter.running = this.counter.announce; this.update_screen(); Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: HDA API: allow index filtering by deleted and/or visible
by commits-noreply＠bitbucket.org 26 Nov '13

26 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/9f31145b33ce/ Changeset: 9f31145b33ce User: carlfeberhard Date: 2013-11-26 18:28:17 Summary: HDA API: allow index filtering by deleted and/or visible Affected #: 1 file diff -r f8f21ec94e5f9baa5718cbff313b085588ad7c5a -r 9f31145b33ceb83e539bcdd571b48ff440b4c1eb lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -50,6 +50,7 @@ # otherwise, check permissions for the history first else: history = self.get_history( trans, history_id, check_ownership=True, check_accessible=True ) + # if ids, return _FULL_ data (as show) for each id passed if ids: ids = ids.split( ',' ) @@ -58,19 +59,39 @@ if encoded_hda_id in ids: #TODO: share code with show rval.append( self._detailed_hda_dict( trans, hda ) ) + # if no ids passed, return a _SUMMARY_ of _all_ datasets in the history else: + # details param allows a mixed set of summary and detailed hdas + #TODO: this is getting convoluted due to backwards compat details = kwd.get( 'details', None ) or [] if details and details != 'all': details = util.listify( details ) + # by default return all datasets - even if deleted or hidden (defaulting the next switches to None) + # if specified return those datasets that match the setting + # backwards compat + return_deleted = util.string_as_bool_or_none( kwd.get( 'deleted', None ) ) + return_visible = util.string_as_bool_or_none( kwd.get( 'visible', None ) ) + for hda in history.datasets: + # if either return_ setting has been requested (!= None), skip hdas that don't match the request + if return_deleted is not None: + if( ( return_deleted and not hda.deleted ) + or ( not return_deleted and hda.deleted ) ): + continue + if return_visible is not None: + if( ( return_visible and not hda.visible ) + or ( not return_visible and hda.visible ) ): + continue + encoded_hda_id = trans.security.encode_id( hda.id ) if( ( encoded_hda_id in details ) or ( details == 'all' ) ): rval.append( self._detailed_hda_dict( trans, hda ) ) else: rval.append( self._summary_hda_dict( trans, history_id, hda ) ) + except Exception, e: # for errors that are not specific to one hda (history lookup or summary list) rval = "Error in history API at listing contents: " + str( e ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: UI: remove existing popupmenu if another is clicked
by commits-noreply＠bitbucket.org 26 Nov '13

26 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f8f21ec94e5f/ Changeset: f8f21ec94e5f User: carlfeberhard Date: 2013-11-26 18:12:00 Summary: UI: remove existing popupmenu if another is clicked Affected #: 2 files diff -r 6fc4a11418458903a35aac3fd4878ab6c4265483 -r f8f21ec94e5f9baa5718cbff313b085588ad7c5a static/scripts/mvc/ui.js --- a/static/scripts/mvc/ui.js +++ b/static/scripts/mvc/ui.js @@ -190,6 +190,7 @@ * view for a popup menu */ var PopupMenu = Backbone.View.extend({ +//TODO: maybe better as singleton off the Galaxy obj /** Cache the desired button element and options, set up the button click handler * NOTE: attaches this view as HTML/jQ data on the button for later use. */ @@ -201,6 +202,8 @@ // set up button click -> open menu behavior var menu = this; this.$button.click( function( event ){ + // if there's already a menu open, remove it + $( '.popmenu-wrapper' ).remove(); menu._renderAndShow( event ); return false; }); @@ -209,10 +212,8 @@ // render the menu, append to the page body at the click position, and set up the 'click-away' handlers, show _renderAndShow: function( clickEvent ){ this.render(); - this.$el.appendTo( 'body' ); - this.$el.css( this._getShownPosition( clickEvent )); + this.$el.appendTo( 'body' ).css( this._getShownPosition( clickEvent )).show(); this._setUpCloseBehavior(); - this.$el.show(); }, // render the menu @@ -289,20 +290,26 @@ // bind an event handler to all available frames so that when anything is clicked // the menu is removed from the DOM and the event handler unbinds itself _setUpCloseBehavior: function(){ + var menu = this; +//TODO: alternately: focus hack, blocking overlay, jquery.blockui + // function to close popup and unbind itself - var menu = this; - var closePopupWhenClicked = function( $elClicked ){ - $elClicked.one( "click.close_popup", function(){ - menu.remove(); - }); - }; + function closePopup( event ){ + $( document ).off( 'click.close_popup' ); + if( window.parent !== window ){ + $( window.parent.document ).off( "click.close_popup" ); + } else { + $( 'iframe#galaxy_main' ).contents().off( "click.close_popup" ); + } + menu.remove(); + } - // bind to current, parent, and sibling frames - closePopupWhenClicked( $( window.document )); - closePopupWhenClicked( $( window.top.document )); - _.each( window.top.frames, function( siblingFrame ){ - closePopupWhenClicked( $( siblingFrame.document )); - }); + $( 'html' ).one( "click.close_popup", closePopup ); + if( window.parent !== window ){ + $( window.parent.document ).find( 'html' ).one( "click.close_popup", closePopup ); + } else { + $( 'iframe#galaxy_main' ).contents().one( "click.close_popup", closePopup ); + } }, // add a menu option/item at the given index diff -r 6fc4a11418458903a35aac3fd4878ab6c4265483 -r f8f21ec94e5f9baa5718cbff313b085588ad7c5a static/scripts/packed/mvc/ui.js --- a/static/scripts/packed/mvc/ui.js +++ b/static/scripts/packed/mvc/ui.js @@ -1,1 +1,1 @@ -var IconButton=Backbone.Model.extend({defaults:{title:"",icon_class:"",on_click:null,menu_options:null,is_menu_button:true,id:null,href:null,target:null,enabled:true,visible:true,tooltip_config:{}}});var IconButtonView=Backbone.View.extend({initialize:function(){this.model.attributes.tooltip_config={placement:"bottom"};this.model.bind("change",this.render,this)},render:function(){this.$el.tooltip("hide");var a=this.template(this.model.toJSON());a.tooltip(this.model.get("tooltip_config"));this.$el.replaceWith(a);this.setElement(a);return this},events:{click:"click"},click:function(a){if(_.isFunction(this.model.get("on_click"))){this.model.get("on_click")(a);return false}return true},template:function(b){var a='title="'+b.title+'" class="icon-button';if(b.is_menu_button){a+=" menu-button"}a+=" "+b.icon_class;if(!b.enabled){a+="_disabled"}a+='"';if(b.id){a+=' id="'+b.id+'"'}a+=' href="'+b.href+'"';if(b.target){a+=' target="'+b.target+'"'}if(!b.visible){a+=' style="display: none;"'}if(b.enabled){a="<a "+a+"/>"}else{a="<span "+a+"/>"}return $(a)}});var IconButtonCollection=Backbone.Collection.extend({model:IconButton});var IconButtonMenuView=Backbone.View.extend({tagName:"div",initialize:function(){this.render()},render:function(){var a=this;this.collection.each(function(d){var b=$("<a/>").attr("href","javascript:void(0)").attr("title",d.attributes.title).addClass("icon-button menu-button").addClass(d.attributes.icon_class).appendTo(a.$el).click(d.attributes.on_click);if(d.attributes.tooltip_config){b.tooltip(d.attributes.tooltip_config)}var c=d.get("options");if(c){make_popupmenu(b,c)}});return this}});var create_icon_buttons_menu=function(b,a){if(!a){a={}}var c=new IconButtonCollection(_.map(b,function(d){return new IconButton(_.extend(d,a))}));return new IconButtonMenuView({collection:c})};var Grid=Backbone.Collection.extend({});var GridView=Backbone.View.extend({});var PopupMenu=Backbone.View.extend({initialize:function(b,a){this.$button=b||$("<div/>");this.options=a||[];var c=this;this.$button.click(function(d){c._renderAndShow(d);return false})},_renderAndShow:function(a){this.render();this.$el.appendTo("body");this.$el.css(this._getShownPosition(a));this._setUpCloseBehavior();this.$el.show()},render:function(){this.$el.addClass("popmenu-wrapper").hide().css({position:"absolute"}).html(this.template(this.$button.attr("id"),this.options));if(this.options.length){var a=this;this.$el.find("li").each(function(c,b){var d=a.options[c];if(d.func){$(this).children("a.popupmenu-option").click(function(e){d.func.call(a,e,d)})}})}return this},template:function(b,a){return['<ul id="',b,'-menu" class="dropdown-menu">',this._templateOptions(a),"</ul>"].join("")},_templateOptions:function(a){if(!a.length){return"<li>(no options)</li>"}return _.map(a,function(d){if(d.divider){return'<li class="divider"></li>'}else{if(d.header){return['<li class="head"><a href="javascript:void(0);">',d.html,"</a></li>"].join("")}}var c=d.href||"javascript:void(0);",e=(d.target)?(' target="'+d.target+'"'):(""),b=(d.checked)?('<span class="fa fa-check"></span>'):("");return['<li><a class="popupmenu-option" href="',c,'"',e,">",b,d.html,"</a></li>"].join("")}).join("")},_getShownPosition:function(b){var c=this.$el.width();var a=b.pageX-c/2;a=Math.min(a,$(document).scrollLeft()+$(window).width()-c-5);a=Math.max(a,$(document).scrollLeft()+5);return{top:b.pageY,left:a}},_setUpCloseBehavior:function(){var b=this;var a=function(c){c.one("click.close_popup",function(){b.remove()})};a($(window.document));a($(window.top.document));_.each(window.top.frames,function(c){a($(c.document))})},addItem:function(b,a){a=(a>=0)?a:this.options.length;this.options.splice(a,0,b);return this},removeItem:function(a){if(a>=0){this.options.splice(a,1)}return this},findIndexByHtml:function(b){for(var a=0;a<this.options.length;a++){if(_.has(this.options[a],"html")&&(this.options[a].html===b)){return a}}return null},findItemByHtml:function(a){return this.options[(this.findIndexByHtml(a))]},toString:function(){return"PopupMenu"}});PopupMenu.make_popupmenu=function(b,c){var a=[];_.each(c,function(f,d){var e={html:d};if(f===null){e.header=true}else{if(jQuery.type(f)==="function"){e.func=f}}a.push(e)});return new PopupMenu($(b),a)};PopupMenu.convertLinksToOptions=function(c,a){c=$(c);a=a||"a";var b=[];c.find(a).each(function(g,e){var f={},d=$(g);f.html=d.text();if(d.attr("href")){var j=d.attr("href"),k=d.attr("target"),h=d.attr("confirm");f.func=function(){if((h)&&(!confirm(h))){return}switch(k){case"_parent":window.parent.location=j;break;case"_top":window.top.location=j;break;default:window.location=j}}}b.push(f)});return b};PopupMenu.fromExistingDom=function(d,c,a){d=$(d);c=$(c);var b=PopupMenu.convertLinksToOptions(c,a);c.remove();return new PopupMenu(d,b)};PopupMenu.make_popup_menus=function(c,b,d){c=c||document;b=b||"div[popupmenu]";d=d||function(e,f){return"#"+e.attr("popupmenu")};var a=[];$(c).find(b).each(function(){var e=$(this),f=$(c).find(d(e,c));a.push(PopupMenu.fromDom(f,e));f.addClass("popup")});return a};var faIconButton=function(a){a=a||{};a.tooltipConfig=a.tooltipConfig||{placement:"bottom"};a.classes=["icon-btn"].concat(a.classes||[]);if(a.disabled){a.classes.push("disabled")}var b=['<a class="',a.classes.join(" "),'"',((a.title)?(' title="'+a.title+'"'):("")),((a.target)?(' target="'+a.target+'"'):("")),' href="',((a.href)?(a.href):("javascript:void(0);")),'">','<span class="fa ',a.faIcon,'"></span>',"</a>"].join("");var c=$(b).tooltip(a.tooltipConfig);if(_.isFunction(a.onclick)){c.click(a.onclick)}return c};var searchInput=function(k){var a=27,h=13,i=$("<div/>"),b={initialVal:"",name:"search",placeholder:"search",classes:"",onclear:function(){},onsearch:function(l){},minSearchLen:0,escWillClear:true,oninit:function(){}};if(jQuery.type(k)==="object"){k=jQuery.extend(true,b,k)}function d(l){var m=$(this).parent().children("input");m.val("");m.trigger("clear:searchInput");k.onclear()}function j(m,l){$(this).trigger("search:searchInput",l);k.onsearch(l)}function c(){return['<input type="text" name="',k.name,'" placeholder="',k.placeholder,'" ','class="search-query ',k.classes,'" ',"/>"].join("")}function g(){return $(c()).css({width:"100%","padding-right":"24px"}).focus(function(l){$(this).select()}).keyup(function(m){if(m.which===a&&k.escWillClear){d.call(this,m)}else{var l=$(this).val();if((m.which===h)||(k.minSearchLen&&l.length>=k.minSearchLen)){j.call(this,m,l)}else{if(!l.length){d.call(this,m)}}}}).val(k.initialVal)}function f(){return'<span class="search-clear fa fa-times-circle"></span>'}function e(){return $(f()).css({position:"absolute",right:"15px","font-size":"1.4em","line-height":"23px",color:"grey"}).click(function(l){d.call(this,l)})}return i.append([g(),e()])};function LoadingIndicator(a,c){var b=this;c=jQuery.extend({cover:false},c||{});function d(){var e=['<div class="loading-indicator">','<div class="loading-indicator-text">','<span class="fa fa-spinner fa-spin fa-lg"></span>','<span class="loading-indicator-message">loading...</span>',"</div>","</div>"].join("\n");var g=$(e).hide().css(c.css||{position:"fixed"}),f=g.children(".loading-indicator-text");if(c.cover){g.css({"z-index":2,top:a.css("top"),bottom:a.css("bottom"),left:a.css("left"),right:a.css("right"),opacity:0.5,"background-color":"white","text-align":"center"});f=g.children(".loading-indicator-text").css({"margin-top":"20px"})}else{f=g.children(".loading-indicator-text").css({margin:"12px 0px 0px 10px",opacity:"0.85",color:"grey"});f.children(".loading-indicator-message").css({margin:"0px 8px 0px 0px","font-style":"italic"})}return g}b.show=function(f,e,g){f=f||"loading...";e=e||"fast";b.$indicator=d().insertBefore(a);b.message(f);b.$indicator.fadeIn(e,g);return b};b.message=function(e){b.$indicator.find("i").text(e)};b.hide=function(e,f){e=e||"fast";if(b.$indicator&&b.$indicator.size()){b.$indicator.fadeOut(e,function(){b.$indicator.remove();if(f){f()}})}else{if(f){f()}}return b};return b}; \ No newline at end of file +var IconButton=Backbone.Model.extend({defaults:{title:"",icon_class:"",on_click:null,menu_options:null,is_menu_button:true,id:null,href:null,target:null,enabled:true,visible:true,tooltip_config:{}}});var IconButtonView=Backbone.View.extend({initialize:function(){this.model.attributes.tooltip_config={placement:"bottom"};this.model.bind("change",this.render,this)},render:function(){this.$el.tooltip("hide");var a=this.template(this.model.toJSON());a.tooltip(this.model.get("tooltip_config"));this.$el.replaceWith(a);this.setElement(a);return this},events:{click:"click"},click:function(a){if(_.isFunction(this.model.get("on_click"))){this.model.get("on_click")(a);return false}return true},template:function(b){var a='title="'+b.title+'" class="icon-button';if(b.is_menu_button){a+=" menu-button"}a+=" "+b.icon_class;if(!b.enabled){a+="_disabled"}a+='"';if(b.id){a+=' id="'+b.id+'"'}a+=' href="'+b.href+'"';if(b.target){a+=' target="'+b.target+'"'}if(!b.visible){a+=' style="display: none;"'}if(b.enabled){a="<a "+a+"/>"}else{a="<span "+a+"/>"}return $(a)}});var IconButtonCollection=Backbone.Collection.extend({model:IconButton});var IconButtonMenuView=Backbone.View.extend({tagName:"div",initialize:function(){this.render()},render:function(){var a=this;this.collection.each(function(d){var b=$("<a/>").attr("href","javascript:void(0)").attr("title",d.attributes.title).addClass("icon-button menu-button").addClass(d.attributes.icon_class).appendTo(a.$el).click(d.attributes.on_click);if(d.attributes.tooltip_config){b.tooltip(d.attributes.tooltip_config)}var c=d.get("options");if(c){make_popupmenu(b,c)}});return this}});var create_icon_buttons_menu=function(b,a){if(!a){a={}}var c=new IconButtonCollection(_.map(b,function(d){return new IconButton(_.extend(d,a))}));return new IconButtonMenuView({collection:c})};var Grid=Backbone.Collection.extend({});var GridView=Backbone.View.extend({});var PopupMenu=Backbone.View.extend({initialize:function(b,a){this.$button=b||$("<div/>");this.options=a||[];var c=this;this.$button.click(function(d){$(".popmenu-wrapper").remove();c._renderAndShow(d);return false})},_renderAndShow:function(a){this.render();this.$el.appendTo("body").css(this._getShownPosition(a)).show();this._setUpCloseBehavior()},render:function(){this.$el.addClass("popmenu-wrapper").hide().css({position:"absolute"}).html(this.template(this.$button.attr("id"),this.options));if(this.options.length){var a=this;this.$el.find("li").each(function(c,b){var d=a.options[c];if(d.func){$(this).children("a.popupmenu-option").click(function(e){d.func.call(a,e,d)})}})}return this},template:function(b,a){return['<ul id="',b,'-menu" class="dropdown-menu">',this._templateOptions(a),"</ul>"].join("")},_templateOptions:function(a){if(!a.length){return"<li>(no options)</li>"}return _.map(a,function(d){if(d.divider){return'<li class="divider"></li>'}else{if(d.header){return['<li class="head"><a href="javascript:void(0);">',d.html,"</a></li>"].join("")}}var c=d.href||"javascript:void(0);",e=(d.target)?(' target="'+d.target+'"'):(""),b=(d.checked)?('<span class="fa fa-check"></span>'):("");return['<li><a class="popupmenu-option" href="',c,'"',e,">",b,d.html,"</a></li>"].join("")}).join("")},_getShownPosition:function(b){var c=this.$el.width();var a=b.pageX-c/2;a=Math.min(a,$(document).scrollLeft()+$(window).width()-c-5);a=Math.max(a,$(document).scrollLeft()+5);return{top:b.pageY,left:a}},_setUpCloseBehavior:function(){var b=this;function a(c){$(document).off("click.close_popup");if(window.parent!==window){$(window.parent.document).off("click.close_popup")}else{$("iframe#galaxy_main").contents().off("click.close_popup")}b.remove()}$("html").one("click.close_popup",a);if(window.parent!==window){$(window.parent.document).find("html").one("click.close_popup",a)}else{$("iframe#galaxy_main").contents().one("click.close_popup",a)}},addItem:function(b,a){a=(a>=0)?a:this.options.length;this.options.splice(a,0,b);return this},removeItem:function(a){if(a>=0){this.options.splice(a,1)}return this},findIndexByHtml:function(b){for(var a=0;a<this.options.length;a++){if(_.has(this.options[a],"html")&&(this.options[a].html===b)){return a}}return null},findItemByHtml:function(a){return this.options[(this.findIndexByHtml(a))]},toString:function(){return"PopupMenu"}});PopupMenu.make_popupmenu=function(b,c){var a=[];_.each(c,function(f,d){var e={html:d};if(f===null){e.header=true}else{if(jQuery.type(f)==="function"){e.func=f}}a.push(e)});return new PopupMenu($(b),a)};PopupMenu.convertLinksToOptions=function(c,a){c=$(c);a=a||"a";var b=[];c.find(a).each(function(g,e){var f={},d=$(g);f.html=d.text();if(d.attr("href")){var j=d.attr("href"),k=d.attr("target"),h=d.attr("confirm");f.func=function(){if((h)&&(!confirm(h))){return}switch(k){case"_parent":window.parent.location=j;break;case"_top":window.top.location=j;break;default:window.location=j}}}b.push(f)});return b};PopupMenu.fromExistingDom=function(d,c,a){d=$(d);c=$(c);var b=PopupMenu.convertLinksToOptions(c,a);c.remove();return new PopupMenu(d,b)};PopupMenu.make_popup_menus=function(c,b,d){c=c||document;b=b||"div[popupmenu]";d=d||function(e,f){return"#"+e.attr("popupmenu")};var a=[];$(c).find(b).each(function(){var e=$(this),f=$(c).find(d(e,c));a.push(PopupMenu.fromDom(f,e));f.addClass("popup")});return a};var faIconButton=function(a){a=a||{};a.tooltipConfig=a.tooltipConfig||{placement:"bottom"};a.classes=["icon-btn"].concat(a.classes||[]);if(a.disabled){a.classes.push("disabled")}var b=['<a class="',a.classes.join(" "),'"',((a.title)?(' title="'+a.title+'"'):("")),((a.target)?(' target="'+a.target+'"'):("")),' href="',((a.href)?(a.href):("javascript:void(0);")),'">','<span class="fa ',a.faIcon,'"></span>',"</a>"].join("");var c=$(b).tooltip(a.tooltipConfig);if(_.isFunction(a.onclick)){c.click(a.onclick)}return c};var searchInput=function(k){var a=27,h=13,i=$("<div/>"),b={initialVal:"",name:"search",placeholder:"search",classes:"",onclear:function(){},onsearch:function(l){},minSearchLen:0,escWillClear:true,oninit:function(){}};if(jQuery.type(k)==="object"){k=jQuery.extend(true,b,k)}function d(l){var m=$(this).parent().children("input");m.val("");m.trigger("clear:searchInput");k.onclear()}function j(m,l){$(this).trigger("search:searchInput",l);k.onsearch(l)}function c(){return['<input type="text" name="',k.name,'" placeholder="',k.placeholder,'" ','class="search-query ',k.classes,'" ',"/>"].join("")}function g(){return $(c()).css({width:"100%","padding-right":"24px"}).focus(function(l){$(this).select()}).keyup(function(m){if(m.which===a&&k.escWillClear){d.call(this,m)}else{var l=$(this).val();if((m.which===h)||(k.minSearchLen&&l.length>=k.minSearchLen)){j.call(this,m,l)}else{if(!l.length){d.call(this,m)}}}}).val(k.initialVal)}function f(){return'<span class="search-clear fa fa-times-circle"></span>'}function e(){return $(f()).css({position:"absolute",right:"15px","font-size":"1.4em","line-height":"23px",color:"grey"}).click(function(l){d.call(this,l)})}return i.append([g(),e()])};function LoadingIndicator(a,c){var b=this;c=jQuery.extend({cover:false},c||{});function d(){var e=['<div class="loading-indicator">','<div class="loading-indicator-text">','<span class="fa fa-spinner fa-spin fa-lg"></span>','<span class="loading-indicator-message">loading...</span>',"</div>","</div>"].join("\n");var g=$(e).hide().css(c.css||{position:"fixed"}),f=g.children(".loading-indicator-text");if(c.cover){g.css({"z-index":2,top:a.css("top"),bottom:a.css("bottom"),left:a.css("left"),right:a.css("right"),opacity:0.5,"background-color":"white","text-align":"center"});f=g.children(".loading-indicator-text").css({"margin-top":"20px"})}else{f=g.children(".loading-indicator-text").css({margin:"12px 0px 0px 10px",opacity:"0.85",color:"grey"});f.children(".loading-indicator-message").css({margin:"0px 8px 0px 0px","font-style":"italic"})}return g}b.show=function(f,e,g){f=f||"loading...";e=e||"fast";b.$indicator=d().insertBefore(a);b.message(f);b.$indicator.fadeIn(e,g);return b};b.message=function(e){b.$indicator.find("i").text(e)};b.hide=function(e,f){e=e||"fast";if(b.$indicator&&b.$indicator.size()){b.$indicator.fadeOut(e,function(){b.$indicator.remove();if(f){f()}})}else{if(f){f()}}return b};return b}; \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: More code cleanup and fixes for the tool shed's install and test framework. The test environment information for both the tool shed and Galaxy should now be stored correctly in the Tool test results container on the view or manage repository page in the tool shed.
by commits-noreply＠bitbucket.org 26 Nov '13

26 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6fc4a1141845/ Changeset: 6fc4a1141845 User: greg Date: 2013-11-26 17:40:25 Summary: More code cleanup and fixes for the tool shed's install and test framework. The test environment information for both the tool shed and Galaxy should now be stored correctly in the Tool test results container on the view or manage repository page in the tool shed. Affected #: 4 files diff -r 4cfffc6fce30b9dd89849975c473e188936318ad -r 6fc4a11418458903a35aac3fd4878ab6c4265483 lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -143,7 +143,8 @@ # Example URL: http://localhost:9009/api/repository_revisions/bb125606ff9ea620 try: repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id ) - repository_metadata_dict = repository_metadata.to_dict( value_mapper=self.__get_value_mapper( trans, repository_metadata ) ) + repository_metadata_dict = repository_metadata.to_dict( view='element', + value_mapper=self.__get_value_mapper( trans, repository_metadata ) ) repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', action='show', id=trans.security.encode_id( repository_metadata.id ) ) @@ -180,7 +181,8 @@ log.error( message, exc_info=True ) trans.response.status = 500 return message - repository_metadata_dict = repository_metadata.to_dict( value_mapper=self.__get_value_mapper( trans, repository_metadata ) ) + repository_metadata_dict = repository_metadata.to_dict( view='element', + value_mapper=self.__get_value_mapper( trans, repository_metadata ) ) repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', action='show', id=trans.security.encode_id( repository_metadata.id ) ) diff -r 4cfffc6fce30b9dd89849975c473e188936318ad -r 6fc4a11418458903a35aac3fd4878ab6c4265483 lib/galaxy/webapps/tool_shed/model/__init__.py --- a/lib/galaxy/webapps/tool_shed/model/__init__.py +++ b/lib/galaxy/webapps/tool_shed/model/__init__.py @@ -251,7 +251,7 @@ self.do_not_test = do_not_test self.test_install_error = test_install_error self.time_last_tested = time_last_tested - self.tool_test_results = tool_test_results + self.tool_test_results = tool_test_results or dict() self.has_repository_dependencies = has_repository_dependencies # We don't consider the special case has_repository_dependencies_only_if_compiling_contained_td here. self.includes_datatypes = includes_datatypes diff -r 4cfffc6fce30b9dd89849975c473e188936318ad -r 6fc4a11418458903a35aac3fd4878ab6c4265483 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -1,16 +1,10 @@ #!/usr/bin/env python -import ConfigParser -import logging + import os -import shutil import sys -import tempfile -import time -from optparse import OptionParser -from time import strftime - -new_path = [ os.path.join( os.getcwd(), "lib" ), os.path.join( os.getcwd(), "test" ) ] +new_path = [ os.path.join( os.getcwd(), "lib" ), + os.path.join( os.getcwd(), "test" ) ] new_path.extend( sys.path[ 1: ] ) sys.path = new_path @@ -18,48 +12,56 @@ eggs.require( "SQLAlchemy >= 0.4" ) eggs.require( 'mercurial' ) +import ConfigParser import galaxy.webapps.tool_shed.config as tool_shed_config import galaxy.webapps.tool_shed.model.mapping +import logging +import shutil +import tempfile +import time -from base.util import get_test_environment from base.util import get_database_version from base.util import get_repository_current_revision -from galaxy.model.orm import and_, not_, select +from galaxy.model.orm import and_ +from galaxy.model.orm import not_ +from galaxy.model.orm import select from mercurial import hg from mercurial import ui from mercurial import __version__ +from optparse import OptionParser +from time import strftime from tool_shed.util.shed_util_common import clone_repository from tool_shed.util.shed_util_common import get_configured_ui -log = logging.getLogger() -log.setLevel( 10 ) -log.addHandler( logging.StreamHandler( sys.stdout ) ) - +log = logging.getLogger( 'check_repositories_for_functional_tests' ) assert sys.version_info[ :2 ] >= ( 2, 6 ) -class FlagRepositoriesApplication( object ): - """Encapsulates the state of a Universe application""" +class RepositoryMetadataApplication( object ): + """Application that enables updating repository_metadata table records in the Tool Shed.""" + def __init__( self, config ): if config.database_connection is False: - config.database_connection = "sqlite:///%s?isolation_level=IMMEDIATE" % config.database + config.database_connection = "sqlite:///%s?isolation_level=IMMEDIATE" % str( config.database ) + log.debug( 'Using database connection: %s' % str( config.database_connection ) ) # Setup the database engine and ORM - self.model = galaxy.webapps.tool_shed.model.mapping.init( config.file_path, config.database_connection, engine_options={}, create_tables=False ) + self.model = galaxy.webapps.tool_shed.model.mapping.init( config.file_path, + config.database_connection, + engine_options={}, + create_tables=False ) self.hgweb_config_manager = self.model.hgweb_config_manager self.hgweb_config_manager.hgweb_config_dir = config.hgweb_config_dir - print "# Using configured hgweb.config file: ", self.hgweb_config_manager.hgweb_config + log.debug( 'Using hgweb.config file: %s' % str( self.hgweb_config_manager.hgweb_config ) ) + @property def sa_session( self ): - """ - Returns a SQLAlchemy session -- currently just gets the current - session from the threadlocal session context, but this is provided - to allow migration toward a more SQLAlchemy 0.4 style of use. - """ + """Returns a SQLAlchemy session.""" return self.model.context.current + def shutdown( self ): pass -def check_and_flag_repositories( app, info_only=False, verbosity=1 ): +def check_and_update_repository_metadata( app, info_only=False, verbosity=1 ): """ This method will iterate through all records in the repository_metadata table, checking each one for tool metadata, then checking the tool metadata for tests. Each tool's metadata should look something like: @@ -352,12 +354,12 @@ print "# %s - Checking repositories for tools with functional tests." % now print "# This tool shed is configured to listen on %s:%s." % ( config_parser.get( config_section, 'host' ), config_parser.get( config_section, 'port' ) ) - app = FlagRepositoriesApplication( config ) + app = RepositoryMetadataApplication( config ) if options.info_only: print "# Displaying info only ( --info_only )" if options.verbosity: print "# Displaying extra information ( --verbosity = %d )" % options.verbosity - check_and_flag_repositories( app, info_only=options.info_only, verbosity=options.verbosity ) + check_and_update_repository_metadata( app, info_only=options.info_only, verbosity=options.verbosity ) def should_set_do_not_test_flag( app, repository, changeset_revision ): ''' diff -r 4cfffc6fce30b9dd89849975c473e188936318ad -r 6fc4a11418458903a35aac3fd4878ab6c4265483 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -1,32 +1,70 @@ #!/usr/bin/env python +""" +This script cannot be run directly, because it needs to have test/functional/test_toolbox.py in sys.argv in +order to run functional tests on repository tools after installation. The install_and_test_tool_shed_repositories.sh +will execute this script with the appropriate parameters. +""" +import os +import sys +# Assume we are run from the galaxy root directory, add lib to the python path +cwd = os.getcwd() +sys.path.append( cwd ) +new_path = [ os.path.join( cwd, "scripts" ), + os.path.join( cwd, "lib" ), + os.path.join( cwd, 'test' ), + os.path.join( cwd, 'scripts', 'api' ) ] +new_path.extend( sys.path ) +sys.path = new_path -# NOTE: This script cannot be run directly, because it needs to have test/functional/test_toolbox.py in sys.argv in -# order to run functional tests on repository tools after installation. The install_and_test_tool_shed_repositories.sh -# will execute this script with the appropriate parameters. +from galaxy import eggs +eggs.require( "nose" ) +eggs.require( "Paste" ) +eggs.require( 'mercurial' ) +# This should not be required, but it is under certain conditions thanks to this bug: +# http://code.google.com/p/python-nose/issues/detail?id=284 +eggs.require( "pysqlite" ) -import atexit +import functional.test_toolbox as test_toolbox import httplib +import install_and_test_tool_shed_repositories.base.test_db_util as test_db_util +import install_and_test_tool_shed_repositories.functional.test_install_repositories as test_install_repositories import logging -import os -import os.path -import platform +import nose import random import re import shutil import socket import string -import sys import tempfile import time import threading -import unittest +import tool_shed.util.shed_util_common as suc import urllib + +from base.util import get_database_version +from base.util import get_repository_current_revision +from base.util import get_test_environment +from base.util import parse_tool_panel_config +from common import update +from datetime import datetime +from galaxy.app import UniverseApplication +from galaxy.util import asbool +from galaxy.util import unicodify +from galaxy.util.json import from_json_string +from galaxy.util.json import to_json_string +from galaxy.web import buildapp +from galaxy.web.framework.helpers import time_ago +from functional_tests import generate_config_file +from mercurial import __version__ +from nose.plugins import Plugin +from paste import httpserver from time import strftime +from tool_shed.util import tool_dependency_util +from tool_shed.util.xml_util import parse_xml -# Assume we are run from the galaxy root directory, add lib to the python path -cwd = os.getcwd() -sys.path.append( cwd ) +log = logging.getLogger( 'install_and_test_repositories' ) +assert sys.version_info[ :2 ] >= ( 2, 6 ) test_home_directory = os.path.join( cwd, 'test', 'install_and_test_tool_shed_repositories' ) default_test_file_dir = os.path.join( test_home_directory, 'test_data' ) @@ -36,134 +74,15 @@ default_galaxy_locales = 'en' default_galaxy_test_file_dir = "test-data" os.environ[ 'GALAXY_INSTALL_TEST_TMP_DIR' ] = galaxy_test_tmp_dir -new_path = [ os.path.join( cwd, "scripts" ), os.path.join( cwd, "lib" ), os.path.join( cwd, 'test' ), os.path.join( cwd, 'scripts', 'api' ) ] -new_path.extend( sys.path ) -sys.path = new_path - -from functional_tests import generate_config_file - -from galaxy import eggs - -eggs.require( "nose" ) -eggs.require( "NoseHTML" ) -eggs.require( "NoseTestDiff" ) -eggs.require( "twill==0.9" ) -eggs.require( "Paste" ) -eggs.require( "PasteDeploy" ) -eggs.require( "Cheetah" ) -eggs.require( "simplejson" ) -eggs.require( 'mercurial' ) - -import simplejson -import twill - -from datetime import datetime -from mercurial import __version__ - -# This should not be required, but it is under certain conditions, thanks to this bug: http://code.google.com/p/python-nose/issues/detail?id=284 -eggs.require( "pysqlite" ) - -import install_and_test_tool_shed_repositories.functional.test_install_repositories as test_install_repositories -import install_and_test_tool_shed_repositories.base.test_db_util as test_db_util -import functional.test_toolbox as test_toolbox - -from paste import httpserver - -# This is for the galaxy application. -import galaxy.app -from galaxy.app import UniverseApplication -from galaxy.web import buildapp -from galaxy.util import parse_xml, asbool -from galaxy.util.json import from_json_string, to_json_string - -import tool_shed.util.shed_util_common as suc -from tool_shed.util import tool_dependency_util - -from galaxy.web.framework.helpers import time_ago - -import nose.core -import nose.config -import nose.loader -import nose.plugins.manager -from nose.plugins import Plugin - -from base.util import get_database_version -from base.util import get_repository_current_revision -from base.util import get_test_environment -from base.util import parse_tool_panel_config - -from galaxy.util import unicodify - -from common import update - -log = logging.getLogger( 'install_and_test_repositories' ) default_galaxy_test_port_min = 10000 default_galaxy_test_port_max = 10999 default_galaxy_test_host = '127.0.0.1' default_galaxy_master_api_key = None -# should this serve static resources (scripts, images, styles, etc.) +# Should this serve static resources (scripts, images, styles, etc.)? STATIC_ENABLED = True -def get_static_settings(): - """Returns dictionary of the settings necessary for a galaxy App - to be wrapped in the static middleware. - - This mainly consists of the filesystem locations of url-mapped - static resources. - """ - cwd = os.getcwd() - static_dir = os.path.join( cwd, 'static' ) - #TODO: these should be copied from universe_wsgi.ini - return dict( - #TODO: static_enabled needed here? - static_enabled = True, - static_cache_time = 360, - static_dir = static_dir, - static_images_dir = os.path.join( static_dir, 'images', '' ), - static_favicon_dir = os.path.join( static_dir, 'favicon.ico' ), - static_scripts_dir = os.path.join( static_dir, 'scripts', '' ), - static_style_dir = os.path.join( static_dir, 'june_2007_style', 'blue' ), - static_robots_txt = os.path.join( static_dir, 'robots.txt' ), - ) - -def get_webapp_global_conf(): - """Get the global_conf dictionary sent as the first argument to app_factory. - """ - # (was originally sent '{}') - nothing here for now except static settings - global_conf = {} - if STATIC_ENABLED: - global_conf.update( get_static_settings() ) - return global_conf - -# Optionally, set the environment variable GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF -# to the location of a tool sheds configuration file that includes the tool shed -# that repositories will be installed from. - -tool_sheds_conf_xml = '''<?xml version="1.0"?> -<tool_sheds> - <tool_shed name="Galaxy main tool shed" url="http://toolshed.g2.bx.psu.edu/"/> - <tool_shed name="Galaxy test tool shed" url="http://testtoolshed.g2.bx.psu.edu/"/> -</tool_sheds> -''' - -# Create a blank shed_tool_conf.xml to hold the installed repositories. -shed_tool_conf_xml_template = '''<?xml version="1.0"?> -<toolbox tool_path="${shed_tool_path}"> -</toolbox> -''' - -# Since we will be running functional tests, we'll need the upload tool, but the rest can be omitted. -tool_conf_xml = '''<?xml version="1.0"?> -<toolbox> - <section name="Get Data" id="getext"> - <tool file="data_source/upload.xml"/> - </section> -</toolbox> -''' - - job_conf_xml = '''<?xml version="1.0"?><job_conf> @@ -182,17 +101,48 @@ </job_conf> ''' +# Create a blank shed_tool_conf.xml to define the installed repositories. +shed_tool_conf_xml_template = '''<?xml version="1.0"?> +<toolbox tool_path="${shed_tool_path}"> +</toolbox> +''' + +# Since we will be running functional tests we'll need the upload tool, but the rest can be omitted. +tool_conf_xml = '''<?xml version="1.0"?> +<toolbox> + <section name="Get Data" id="getext"> + <tool file="data_source/upload.xml"/> + </section> +</toolbox> +''' + +# Set up an empty shed_tool_data_table_conf.xml. +tool_data_table_conf_xml_template = '''<?xml version="1.0"?> +<tables> +</tables> +''' + +# Optionally set the environment variable GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF to the location of a +# tool shed's configuration file that includes the tool shed from which repositories will be installed. +tool_sheds_conf_xml = '''<?xml version="1.0"?> +<tool_sheds> + <tool_shed name="Galaxy main tool shed" url="http://toolshed.g2.bx.psu.edu/"/> + <tool_shed name="Galaxy test tool shed" url="http://testtoolshed.g2.bx.psu.edu/"/> +</tool_sheds> +''' + # If we have a tool_data_table_conf.test.xml, set it up to be loaded when the UniverseApplication is started. # This allows one to specify a set of tool data that is used exclusively for testing, and not loaded into any # Galaxy instance. By default, this will be in the test-data-repo/location directory generated by buildbot_setup.sh. if os.path.exists( 'tool_data_table_conf.test.xml' ): additional_tool_data_tables = 'tool_data_table_conf.test.xml' - additional_tool_data_path = os.environ.get( 'GALAXY_INSTALL_TEST_EXTRA_TOOL_DATA_PATH', os.path.join( 'test-data-repo', 'location' ) ) + additional_tool_data_path = os.environ.get( 'GALAXY_INSTALL_TEST_EXTRA_TOOL_DATA_PATH', + os.path.join( 'test-data-repo', 'location' ) ) else: additional_tool_data_tables = None additional_tool_data_path = None -# Also set up default tool data tables. +# Set up default tool data tables. if os.path.exists( 'tool_data_table_conf.xml' ): tool_data_table_conf = 'tool_data_table_conf.xml' elif os.path.exists( 'tool_data_table_conf.xml.sample' ): @@ -200,28 +150,14 @@ else: tool_data_table_conf = None -# And set up a blank shed_tool_data_table_conf.xml. -tool_data_table_conf_xml_template = '''<?xml version="1.0"?> -<tables> -</tables> -''' - -# The tool shed url and api key must be set for this script to work correctly. Additionally, if the tool shed url does not -# point to one of the defaults, the GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF needs to point to a tool sheds configuration file -# that contains a definition for that tool shed. - +# The GALAXY_INSTALL_TEST_TOOL_SHED_URL and GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY environment variables must be +# set for this script to work correctly. If the value of GALAXY_INSTALL_TEST_TOOL_SHED_URL does not refer to one +# of the defaults, the GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF must refer to a tool shed configuration file that contains +# a definition for that tool shed. galaxy_tool_shed_url = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_URL', None ) tool_shed_api_key = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY', None ) exclude_list_file = os.environ.get( 'GALAXY_INSTALL_TEST_EXCLUDE_REPOSITORIES', 'install_test_exclude.xml' ) -if tool_shed_api_key is None: - print "This script requires the GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY environment variable to be set and non-empty." - exit( 1 ) - -if galaxy_tool_shed_url is None: - print "This script requires the GALAXY_INSTALL_TEST_TOOL_SHED_URL environment variable to be set and non-empty." - exit( 1 ) - if 'GALAXY_INSTALL_TEST_SECRET' not in os.environ: galaxy_encode_secret = 'changethisinproductiontoo' os.environ[ 'GALAXY_INSTALL_TEST_SECRET' ] = galaxy_encode_secret @@ -237,6 +173,7 @@ else: testing_single_repository[ 'changeset_revision' ] = None + class ReportResults( Plugin ): '''Simple Nose plugin to record the IDs of all tests run, regardless of success.''' name = "reportresults" @@ -481,6 +418,24 @@ str( repository_dict.get( 'owner', None ) ) ) ) return repository_dicts, error_message +def get_static_settings(): + """ + Return a dictionary of the settings necessary for a Galaxy application to be wrapped in the static + middleware. This mainly consists of the file system locations of url-mapped static resources. + """ + cwd = os.getcwd() + static_dir = os.path.join( cwd, 'static' ) + #TODO: these should be copied from universe_wsgi.ini + #TODO: static_enabled needed here? + return dict( static_enabled = True, + static_cache_time = 360, + static_dir = static_dir, + static_images_dir = os.path.join( static_dir, 'images', '' ), + static_favicon_dir = os.path.join( static_dir, 'favicon.ico' ), + static_scripts_dir = os.path.join( static_dir, 'scripts', '' ), + static_style_dir = os.path.join( static_dir, 'june_2007_style', 'blue' ), + static_robots_txt = os.path.join( static_dir, 'robots.txt' ) ) + def get_tool_info_from_test_id( test_id ): """ Test IDs come in the form test_tool_number @@ -500,8 +455,17 @@ if error_message: return None, error_message tool_test_results = repository_metadata.get( 'tool_test_results', {} ) + if tool_test_results is None: + return None, error_message return tool_test_results, error_message +def get_webapp_global_conf(): + """Return the global_conf dictionary sent as the first argument to app_factory.""" + global_conf = {} + if STATIC_ENABLED: + global_conf.update( get_static_settings() ) + return global_conf + def handle_missing_dependencies( app, repository, missing_tool_dependencies, repository_dict, tool_test_results_dict, results_dict ): """Handle missing repository or tool dependencies for an installed repository.""" # If a tool dependency fails to install correctly, this should be considered an installation error, @@ -671,7 +635,7 @@ # that are missing test components. We need to be careful to not lose this information. For all other repositories, # no changes will have been made to this dictionary by the preparation script, and tool_test_results_dict will be None. # Initialize the tool_test_results_dict dictionary with the information about the current test environment. - test_environment_dict = tool_test_results_dict.get( 'test_environent', None ) + test_environment_dict = tool_test_results_dict.get( 'test_environment', None ) test_environment_dict = get_test_environment( test_environment_dict ) test_environment_dict[ 'galaxy_database_version' ] = get_database_version( app ) test_environment_dict[ 'galaxy_revision' ] = get_repository_current_revision( os.getcwd() ) @@ -772,261 +736,21 @@ return None, error_message return parsed_json, error_message -def parse_exclude_list( xml_filename ): - """Return a list of repositories to exclude from testing.""" - # This method should return a list with the following structure: - # [{ 'reason': The default reason or the reason specified in this section, - # 'repositories': [( name, owner, changeset revision if changeset revision else None ), - # ( name, owner, changeset revision if changeset revision else None )]}] - exclude_list = [] - exclude_verbose = [] - xml_tree = parse_xml( xml_filename ) - tool_sheds = xml_tree.findall( 'repositories' ) - xml_element = [] - exclude_count = 0 - for tool_shed in tool_sheds: - if galaxy_tool_shed_url != tool_shed.attrib[ 'tool_shed' ]: - continue - else: - xml_element = tool_shed - for reason_section in xml_element: - reason_text = reason_section.find( 'text' ).text - repositories = reason_section.findall( 'repository' ) - exclude_dict = dict( reason=reason_text, repositories=[] ) - for repository in repositories: - repository_tuple = get_repository_tuple_from_elem( repository ) - if repository_tuple not in exclude_dict[ 'repositories' ]: - exclude_verbose.append( repository_tuple ) - exclude_count += 1 - exclude_dict[ 'repositories' ].append( repository_tuple ) - exclude_list.append( exclude_dict ) - log.debug( '%s repositories excluded from testing...' % str( exclude_count ) ) - if '-list_repositories' in sys.argv: - for name, owner, changeset_revision in exclude_verbose: - if changeset_revision: - log.debug( 'Repository %s owned by %s, changeset revision %s.' % ( str( name ), str( owner ), str( changeset_revision ) ) ) - else: - log.debug( 'Repository %s owned by %s, all revisions.' % ( str( name ), str( owner ) ) ) - return exclude_list - -def register_test_result( url, test_results_dict, repository_dict, params ): - """ - Update the repository metadata tool_test_results and appropriate flags using the Tool SHed API. This method - updates tool_test_results with the relevant data, sets the do_not_test and tools_functionally correct flags - to the appropriate values and updates the time_last_tested field to the value of the received time_tested. - """ - if '-info_only' in sys.argv or 'GALAXY_INSTALL_TEST_INFO_ONLY' in os.environ: - return {} - else: - metadata_revision_id = repository_dict.get( 'id', None ) - log.debug("RRR In register_test_result, metadata_revision_id: %s" % str( metadata_revision_id )) - if metadata_revision_id is not None: - # Set the time_last_tested entry so that the repository_metadata.time_last_tested will be set in the tool shed. - time_tested = datetime.utcnow() - test_results_dict[ 'time_last_tested' ] = time_ago( time_tested ) - params[ 'tool_test_results' ] = test_results_dict - url = '%s' % ( suc.url_join( galaxy_tool_shed_url,'api', 'repository_revisions', str( metadata_revision_id ) ) ) - try: - return update( tool_shed_api_key, url, params, return_formatted=False ) - except Exception, e: - log.exception( 'Error attempting to register test results: %s' % str( e ) ) - return {} - -def remove_generated_tests( app ): - """ - Delete any configured tool functional tests from the test_toolbox.__dict__, otherwise nose will find them - and try to re-run the tests after uninstalling the repository, which will cause false failure reports, - since the test data has been deleted from disk by now. - """ - tests_to_delete = [] - tools_to_delete = [] - global test_toolbox - for key in test_toolbox.__dict__: - if key.startswith( 'TestForTool_' ): - log.debug( 'Tool test found in test_toolbox, deleting: %s' % str( key ) ) - # We can't delete this test just yet, we're still iterating over __dict__. - tests_to_delete.append( key ) - tool_id = key.replace( 'TestForTool_', '' ) - for tool in app.toolbox.tools_by_id: - if tool.replace( '_', ' ' ) == tool_id.replace( '_', ' ' ): - tools_to_delete.append( tool ) - for key in tests_to_delete: - # Now delete the tests found in the previous loop. - del test_toolbox.__dict__[ key ] - for tool in tools_to_delete: - del app.toolbox.tools_by_id[ tool ] - -def remove_install_tests(): - """ - Delete any configured repository installation tests from the test_toolbox.__dict__, otherwise nose will find them - and try to install the repository again while running tool functional tests. - """ - tests_to_delete = [] - global test_toolbox - # Push all the toolbox tests to module level - for key in test_install_repositories.__dict__: - if key.startswith( 'TestInstallRepository_' ): - log.debug( 'Repository installation process found, deleting: %s' % str( key ) ) - # We can't delete this test just yet, we're still iterating over __dict__. - tests_to_delete.append( key ) - for key in tests_to_delete: - # Now delete the tests found in the previous loop. - del test_install_repositories.__dict__[ key ] - -def run_tests( test_config ): - loader = nose.loader.TestLoader( config=test_config ) - test_config.plugins.addPlugin( ReportResults() ) - plug_loader = test_config.plugins.prepareTestLoader( loader ) - if plug_loader is not None: - loader = plug_loader - tests = loader.loadTestsFromNames( test_config.testNames ) - test_runner = nose.core.TextTestRunner( stream=test_config.stream, - verbosity=test_config.verbosity, - config=test_config ) - plug_runner = test_config.plugins.prepareTestRunner( test_runner ) - if plug_runner is not None: - test_runner = plug_runner - result = test_runner.run( tests ) - return result, test_config.plugins._plugins - -def show_summary_output( repository_dicts ): - repositories_by_owner = {} - for repository in repository_dicts: - if repository[ 'owner' ] not in repositories_by_owner: - repositories_by_owner[ repository[ 'owner' ] ] = [] - repositories_by_owner[ repository[ 'owner' ] ].append( repository ) - for owner in repositories_by_owner: - print "# " - for repository in repositories_by_owner[ owner ]: - print "# %s owned by %s, changeset revision %s" % ( repository[ 'name' ], repository[ 'owner' ], repository[ 'changeset_revision' ] ) - -def test_repository_tools( app, repository, repository_dict, tool_test_results_dict, results_dict ): - """Test tools contained in the received repository.""" - name = str( repository.name ) - owner = str( repository.owner ) - changeset_revision = str( repository.changeset_revision ) - # Set the module-level variable 'toolbox', so that test.functional.test_toolbox will generate the appropriate test methods. - test_toolbox.toolbox = app.toolbox - # Generate the test methods for this installed repository. We need to pass in True here, or it will look - # in $GALAXY_HOME/test-data for test data, which may result in missing or invalid test files. - test_toolbox.build_tests( testing_shed_tools=True, master_api_key=default_galaxy_master_api_key ) - # Set up nose to run the generated functional tests. - test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) - test_config.configure( sys.argv ) - # Run the configured tests. - result, test_plugins = run_tests( test_config ) - success = result.wasSuccessful() - # Use the ReportResults nose plugin to get a list of tests that passed. - for plugin in test_plugins: - if hasattr( plugin, 'getTestStatus' ): - test_identifier = '%s/%s' % ( owner, name ) - passed_tests = plugin.getTestStatus( test_identifier ) - break - tool_test_results_dict[ 'passed_tests' ] = [] - for test_id in passed_tests: - # Normalize the tool ID and version display. - tool_id, tool_version = get_tool_info_from_test_id( test_id ) - test_result = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version ) - tool_test_results_dict[ 'passed_tests' ].append( test_result ) - if success: - # This repository's tools passed all functional tests. Update the repository_metadata table in the tool shed's database - # to reflect that. Call the register_test_result method, which executes a PUT request to the repository_revisions API - # controller with the status of the test. This also sets the do_not_test and tools_functionally correct flags, and - # updates the time_last_tested field to today's date. - results_dict[ 'repositories_passed' ].append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) - params = dict( tools_functionally_correct=True, - do_not_test=False, - test_install_error=False ) - register_test_result( galaxy_tool_shed_url, tool_test_results_dict, repository_dict, params ) - log.debug( 'Revision %s of repository %s installed and passed functional tests.' % ( str( changeset_revision ), str( name ) ) ) - else: - tool_test_results_dict[ 'failed_tests' ].append( extract_log_data( result, from_tool_test=True ) ) - results_dict[ 'repositories_failed' ].append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) - set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_dict ) - params = dict( tools_functionally_correct=False, - test_install_error=False, - do_not_test=str( set_do_not_test ) ) - register_test_result( galaxy_tool_shed_url, tool_test_results_dict, repository_dict, params ) - log.debug( 'Revision %s of repository %s installed successfully but did not pass functional tests.' % \ - ( str( changeset_revision ), str( name ) ) ) - # Run the uninstall method. This removes tool functional test methods from the test_toolbox module and uninstalls the - # repository using Twill. - deactivate = asbool( os.environ.get( 'GALAXY_INSTALL_TEST_KEEP_TOOL_DEPENDENCIES', False ) ) - if deactivate: - log.debug( 'Deactivating changeset revision %s of repository %s' % ( str( changeset_revision ), str( name ) ) ) - # We are deactivating this repository and all of its repository dependencies. - deactivate_repository( app, repository_dict ) - else: - log.debug( 'Uninstalling changeset revision %s of repository %s' % ( str( changeset_revision ), str( name ) ) ) - # We are uninstalling this repository and all of its repository dependencies. - uninstall_repository( app, repository_dict ) - - # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. - test_toolbox.toolbox = app.toolbox - return results_dict - -def uninstall_repository( app, repository_dict ): - """Attempt to uninstall a repository.""" - sa_session = app.model.context.current - # Clean out any generated tests. This is necessary for Twill. - remove_generated_tests( app ) - # The dict contains the only repository the app should have installed at this point. - name = str( repository_dict[ 'name' ] ) - owner = str( repository_dict[ 'owner' ] ) - changeset_revision = str( repository_dict[ 'changeset_revision' ] ) - repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) - # We have to do this through Twill, in order to maintain app.toolbox and shed_tool_conf.xml in a state that is valid for future tests. - for required_repository in repository.repository_dependencies: - repository_dict = dict( name=str( required_repository.name ), - owner=str( required_repository.owner ), - changeset_revision=str( required_repository.changeset_revision ) ) - # Generate a test method to uninstall this repository through the embedded Galaxy application's web interface. - test_install_repositories.generate_deactivate_or_uninstall_method( repository_dict, deactivate=False ) - log.debug( 'Changeset revision %s of %s repository %s selected for uninstallation.' % \ - ( str( required_repository.changeset_revision ), str( required_repository.status ), str( required_repository.name ) ) ) - repository_dict = dict( name=name, owner=owner, changeset_revision=changeset_revision ) - test_install_repositories.generate_deactivate_or_uninstall_method( repository_dict, deactivate=False ) - log.debug( 'Changeset revision %s of %s repository %s selected for uninstallation.' % ( changeset_revision, str( repository.status ), name ) ) - # Set up nose to run the generated uninstall method as a functional test. - test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) - test_config.configure( sys.argv ) - # Run the uninstall method. This method uses the Galaxy web interface to uninstall the previously installed - # repository and delete it from disk. - result, _ = run_tests( test_config ) - success = result.wasSuccessful() - if not success: - log.debug( 'Repository %s failed to uninstall.' % str( name ) ) - -def uninstall_tool_dependency( app, tool_dependency ): - """Attempt to uninstall a tool dependency.""" - sa_session = app.model.context.current - # Clean out any generated tests. This is necessary for Twill. - tool_dependency_install_path = tool_dependency.installation_directory( app ) - uninstalled, error_message = tool_dependency_util.remove_tool_dependency( app, tool_dependency ) - if error_message: - log.debug( 'There was an error attempting to remove directory: %s' % str( tool_dependency_install_path ) ) - log.debug( error_message ) - else: - log.debug( 'Successfully removed tool dependency installation directory: %s' % str( tool_dependency_install_path ) ) - if not uninstalled or tool_dependency.status != app.model.ToolDependency.installation_status.UNINSTALLED: - tool_dependency.status = app.model.ToolDependency.installation_status.UNINSTALLED - sa_session.add( tool_dependency ) - sa_session.flush() - if os.path.exists( tool_dependency_install_path ): - log.debug( 'Uninstallation of tool dependency succeeded, but the installation path still exists on the filesystem. It is now being explicitly deleted.') - suc.remove_dir( tool_dependency_install_path ) - def main(): if tool_shed_api_key is None: # If the tool shed URL specified in any dict is not present in the tool_sheds_conf.xml, the installation will fail. log.debug( 'Cannot proceed without a valid tool shed API key set in the enviroment variable GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY.' ) return 1 + if galaxy_tool_shed_url is None: + log.debug( 'Cannot proceed without a valid Tool Shed base URL set in the environment variable GALAXY_INSTALL_TEST_TOOL_SHED_URL.' ) + return 1 # ---- Configuration ------------------------------------------------------ galaxy_test_host = os.environ.get( 'GALAXY_INSTALL_TEST_HOST', default_galaxy_test_host ) - # Set the GALAXY_INSTALL_TEST_HOST variable so that Twill will have the Galaxy url to install repositories into. + # Set the GALAXY_INSTALL_TEST_HOST variable so that Twill will have the Galaxy url to which to + # install repositories. os.environ[ 'GALAXY_INSTALL_TEST_HOST' ] = galaxy_test_host - # Set the GALAXY_TEST_HOST environment variable so that the toolbox tests will have the Galaxy url to run tool functional - # tests on. + # Set the GALAXY_TEST_HOST environment variable so that the toolbox tests will have the Galaxy url + # on which to to run tool functional tests. os.environ[ 'GALAXY_TEST_HOST' ] = galaxy_test_host galaxy_test_port = os.environ.get( 'GALAXY_INSTALL_TEST_PORT', str( default_galaxy_test_port_max ) ) os.environ[ 'GALAXY_TEST_PORT' ] = galaxy_test_port @@ -1040,17 +764,25 @@ if not os.path.isdir( galaxy_test_tmp_dir ): os.mkdir( galaxy_test_tmp_dir ) # Set up the configuration files for the Galaxy instance. - shed_tool_data_table_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_SHED_TOOL_DATA_TABLE_CONF', os.path.join( galaxy_test_tmp_dir, 'test_shed_tool_data_table_conf.xml' ) ) - galaxy_tool_data_table_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_DATA_TABLE_CONF', tool_data_table_conf ) - galaxy_tool_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_CONF', os.path.join( galaxy_test_tmp_dir, 'test_tool_conf.xml' ) ) - galaxy_job_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_JOB_CONF', os.path.join( galaxy_test_tmp_dir, 'test_job_conf.xml' ) ) - galaxy_shed_tool_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_SHED_TOOL_CONF', os.path.join( galaxy_test_tmp_dir, 'test_shed_tool_conf.xml' ) ) - galaxy_migrated_tool_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_MIGRATED_TOOL_CONF', os.path.join( galaxy_test_tmp_dir, 'test_migrated_tool_conf.xml' ) ) - galaxy_tool_sheds_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF', os.path.join( galaxy_test_tmp_dir, 'test_tool_sheds_conf.xml' ) ) - galaxy_shed_tools_dict = os.environ.get( 'GALAXY_INSTALL_TEST_SHED_TOOL_DICT_FILE', os.path.join( galaxy_test_tmp_dir, 'shed_tool_dict' ) ) + shed_tool_data_table_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_SHED_TOOL_DATA_TABLE_CONF', + os.path.join( galaxy_test_tmp_dir, 'test_shed_tool_data_table_conf.xml' ) ) + galaxy_tool_data_table_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_DATA_TABLE_CONF', + tool_data_table_conf ) + galaxy_tool_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_CONF', + os.path.join( galaxy_test_tmp_dir, 'test_tool_conf.xml' ) ) + galaxy_job_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_JOB_CONF', + os.path.join( galaxy_test_tmp_dir, 'test_job_conf.xml' ) ) + galaxy_shed_tool_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_SHED_TOOL_CONF', + os.path.join( galaxy_test_tmp_dir, 'test_shed_tool_conf.xml' ) ) + galaxy_migrated_tool_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_MIGRATED_TOOL_CONF', + os.path.join( galaxy_test_tmp_dir, 'test_migrated_tool_conf.xml' ) ) + galaxy_tool_sheds_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF', + os.path.join( galaxy_test_tmp_dir, 'test_tool_sheds_conf.xml' ) ) + galaxy_shed_tools_dict = os.environ.get( 'GALAXY_INSTALL_TEST_SHED_TOOL_DICT_FILE', + os.path.join( galaxy_test_tmp_dir, 'shed_tool_dict' ) ) file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( {} ) ) - # Set the GALAXY_TOOL_SHED_TEST_FILE environment variable to the path of the shed_tools_dict file, so that test.base.twilltestcase.setUp - # will find and parse it properly. + # Set the GALAXY_TOOL_SHED_TEST_FILE environment variable to the path of the shed_tools_dict file so that + # test.base.twilltestcase.setUp will find and parse it properly. os.environ[ 'GALAXY_TOOL_SHED_TEST_FILE' ] = galaxy_shed_tools_dict if 'GALAXY_INSTALL_TEST_TOOL_DATA_PATH' in os.environ: tool_data_path = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_DATA_PATH' ) @@ -1261,20 +993,267 @@ if os.path.exists( dir ): try: shutil.rmtree( dir ) - log.debug( "Cleaned up temporary files in %s", dir ) + log.debug( "Cleaned up temporary files in %s", str( dir ) ) except: pass else: log.debug( 'GALAXY_INSTALL_TEST_NO_CLEANUP set, not cleaning up.' ) - # Normally, the value of 'success' would determine whether this test suite is marked as passed or failed - # in the automated buildbot framework. However, due to the procedure used here, we only want to report - # failure if a repository fails to install correctly. Therefore, we have overriden the value of 'success' - # here based on what actions the script has executed. + # Normally the value of 'success' would determine whether this test suite is marked as passed or failed + # in the automated buildbot framework. However, due to the procedure used here we only want to report + # failure if a repository fails to install correctly, so we have overriden the value of 'success' here + # based on what actions the script has executed. if success: return 0 else: return 1 +def parse_exclude_list( xml_filename ): + """Return a list of repositories to exclude from testing.""" + # This method should return a list with the following structure: + # [{ 'reason': The default reason or the reason specified in this section, + # 'repositories': [( name, owner, changeset revision if changeset revision else None ), + # ( name, owner, changeset revision if changeset revision else None )]}] + exclude_list = [] + exclude_verbose = [] + xml_tree, error_message = parse_xml( xml_filename ) + if error_message: + log.debug( 'The xml document %s defining the exclude list is invalid, so no repositories will be excluded from testing: %s' % \ + ( str( xml_filename ), str( error_message ) ) ) + return exclude_list + tool_sheds = xml_tree.findall( 'repositories' ) + xml_element = [] + exclude_count = 0 + for tool_shed in tool_sheds: + if galaxy_tool_shed_url != tool_shed.attrib[ 'tool_shed' ]: + continue + else: + xml_element = tool_shed + for reason_section in xml_element: + reason_text = reason_section.find( 'text' ).text + repositories = reason_section.findall( 'repository' ) + exclude_dict = dict( reason=reason_text, repositories=[] ) + for repository in repositories: + repository_tuple = get_repository_tuple_from_elem( repository ) + if repository_tuple not in exclude_dict[ 'repositories' ]: + exclude_verbose.append( repository_tuple ) + exclude_count += 1 + exclude_dict[ 'repositories' ].append( repository_tuple ) + exclude_list.append( exclude_dict ) + log.debug( '%s repositories will be excluded from testing...' % str( exclude_count ) ) + if '-list_repositories' in sys.argv: + for name, owner, changeset_revision in exclude_verbose: + if changeset_revision: + log.debug( 'Repository %s owned by %s, changeset revision %s.' % ( str( name ), str( owner ), str( changeset_revision ) ) ) + else: + log.debug( 'Repository %s owned by %s, all revisions.' % ( str( name ), str( owner ) ) ) + return exclude_list + +def register_test_result( url, test_results_dict, repository_dict, params ): + """ + Update the repository metadata tool_test_results and appropriate flags using the Tool SHed API. This method + updates tool_test_results with the relevant data, sets the do_not_test and tools_functionally correct flags + to the appropriate values and updates the time_last_tested field to the value of the received time_tested. + """ + if '-info_only' in sys.argv or 'GALAXY_INSTALL_TEST_INFO_ONLY' in os.environ: + return {} + else: + metadata_revision_id = repository_dict.get( 'id', None ) + if metadata_revision_id is not None: + # Set the time_last_tested entry so that the repository_metadata.time_last_tested will be set in the tool shed. + time_tested = datetime.utcnow() + test_results_dict[ 'time_last_tested' ] = time_ago( time_tested ) + params[ 'tool_test_results' ] = test_results_dict + url = '%s' % ( suc.url_join( galaxy_tool_shed_url,'api', 'repository_revisions', str( metadata_revision_id ) ) ) + try: + return update( tool_shed_api_key, url, params, return_formatted=False ) + except Exception, e: + log.exception( 'Error attempting to register test results: %s' % str( e ) ) + return {} + +def remove_generated_tests( app ): + """ + Delete any configured tool functional tests from the test_toolbox.__dict__, otherwise nose will find them + and try to re-run the tests after uninstalling the repository, which will cause false failure reports, + since the test data has been deleted from disk by now. + """ + tests_to_delete = [] + tools_to_delete = [] + global test_toolbox + for key in test_toolbox.__dict__: + if key.startswith( 'TestForTool_' ): + log.debug( 'Tool test found in test_toolbox, deleting: %s' % str( key ) ) + # We can't delete this test just yet, we're still iterating over __dict__. + tests_to_delete.append( key ) + tool_id = key.replace( 'TestForTool_', '' ) + for tool in app.toolbox.tools_by_id: + if tool.replace( '_', ' ' ) == tool_id.replace( '_', ' ' ): + tools_to_delete.append( tool ) + for key in tests_to_delete: + # Now delete the tests found in the previous loop. + del test_toolbox.__dict__[ key ] + for tool in tools_to_delete: + del app.toolbox.tools_by_id[ tool ] + +def remove_install_tests(): + """ + Delete any configured repository installation tests from the test_toolbox.__dict__, otherwise nose will find them + and try to install the repository again while running tool functional tests. + """ + tests_to_delete = [] + global test_toolbox + # Push all the toolbox tests to module level + for key in test_install_repositories.__dict__: + if key.startswith( 'TestInstallRepository_' ): + log.debug( 'Repository installation process found, deleting: %s' % str( key ) ) + # We can't delete this test just yet, we're still iterating over __dict__. + tests_to_delete.append( key ) + for key in tests_to_delete: + # Now delete the tests found in the previous loop. + del test_install_repositories.__dict__[ key ] + +def run_tests( test_config ): + loader = nose.loader.TestLoader( config=test_config ) + test_config.plugins.addPlugin( ReportResults() ) + plug_loader = test_config.plugins.prepareTestLoader( loader ) + if plug_loader is not None: + loader = plug_loader + tests = loader.loadTestsFromNames( test_config.testNames ) + test_runner = nose.core.TextTestRunner( stream=test_config.stream, + verbosity=test_config.verbosity, + config=test_config ) + plug_runner = test_config.plugins.prepareTestRunner( test_runner ) + if plug_runner is not None: + test_runner = plug_runner + result = test_runner.run( tests ) + return result, test_config.plugins._plugins + +def show_summary_output( repository_dicts ): + repositories_by_owner = {} + for repository in repository_dicts: + if repository[ 'owner' ] not in repositories_by_owner: + repositories_by_owner[ repository[ 'owner' ] ] = [] + repositories_by_owner[ repository[ 'owner' ] ].append( repository ) + for owner in repositories_by_owner: + print "# " + for repository in repositories_by_owner[ owner ]: + print "# %s owned by %s, changeset revision %s" % ( repository[ 'name' ], repository[ 'owner' ], repository[ 'changeset_revision' ] ) + +def test_repository_tools( app, repository, repository_dict, tool_test_results_dict, results_dict ): + """Test tools contained in the received repository.""" + name = str( repository.name ) + owner = str( repository.owner ) + changeset_revision = str( repository.changeset_revision ) + # Set the module-level variable 'toolbox', so that test.functional.test_toolbox will generate the appropriate test methods. + test_toolbox.toolbox = app.toolbox + # Generate the test methods for this installed repository. We need to pass in True here, or it will look + # in $GALAXY_HOME/test-data for test data, which may result in missing or invalid test files. + test_toolbox.build_tests( testing_shed_tools=True, master_api_key=default_galaxy_master_api_key ) + # Set up nose to run the generated functional tests. + test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) + test_config.configure( sys.argv ) + # Run the configured tests. + result, test_plugins = run_tests( test_config ) + success = result.wasSuccessful() + # Use the ReportResults nose plugin to get a list of tests that passed. + for plugin in test_plugins: + if hasattr( plugin, 'getTestStatus' ): + test_identifier = '%s/%s' % ( owner, name ) + passed_tests = plugin.getTestStatus( test_identifier ) + break + tool_test_results_dict[ 'passed_tests' ] = [] + for test_id in passed_tests: + # Normalize the tool ID and version display. + tool_id, tool_version = get_tool_info_from_test_id( test_id ) + test_result = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version ) + tool_test_results_dict[ 'passed_tests' ].append( test_result ) + if success: + # This repository's tools passed all functional tests. Update the repository_metadata table in the tool shed's database + # to reflect that. Call the register_test_result method, which executes a PUT request to the repository_revisions API + # controller with the status of the test. This also sets the do_not_test and tools_functionally correct flags, and + # updates the time_last_tested field to today's date. + results_dict[ 'repositories_passed' ].append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + params = dict( tools_functionally_correct=True, + do_not_test=False, + test_install_error=False ) + register_test_result( galaxy_tool_shed_url, tool_test_results_dict, repository_dict, params ) + log.debug( 'Revision %s of repository %s installed and passed functional tests.' % ( str( changeset_revision ), str( name ) ) ) + else: + tool_test_results_dict[ 'failed_tests' ].append( extract_log_data( result, from_tool_test=True ) ) + results_dict[ 'repositories_failed' ].append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_dict ) + params = dict( tools_functionally_correct=False, + test_install_error=False, + do_not_test=str( set_do_not_test ) ) + register_test_result( galaxy_tool_shed_url, tool_test_results_dict, repository_dict, params ) + log.debug( 'Revision %s of repository %s installed successfully but did not pass functional tests.' % \ + ( str( changeset_revision ), str( name ) ) ) + # Run the uninstall method. This removes tool functional test methods from the test_toolbox module and uninstalls the + # repository using Twill. + deactivate = asbool( os.environ.get( 'GALAXY_INSTALL_TEST_KEEP_TOOL_DEPENDENCIES', False ) ) + if deactivate: + log.debug( 'Deactivating changeset revision %s of repository %s' % ( str( changeset_revision ), str( name ) ) ) + # We are deactivating this repository and all of its repository dependencies. + deactivate_repository( app, repository_dict ) + else: + log.debug( 'Uninstalling changeset revision %s of repository %s' % ( str( changeset_revision ), str( name ) ) ) + # We are uninstalling this repository and all of its repository dependencies. + uninstall_repository( app, repository_dict ) + + # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. + test_toolbox.toolbox = app.toolbox + return results_dict + +def uninstall_repository( app, repository_dict ): + """Attempt to uninstall a repository.""" + sa_session = app.model.context.current + # Clean out any generated tests. This is necessary for Twill. + remove_generated_tests( app ) + # The dict contains the only repository the app should have installed at this point. + name = str( repository_dict[ 'name' ] ) + owner = str( repository_dict[ 'owner' ] ) + changeset_revision = str( repository_dict[ 'changeset_revision' ] ) + repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) + # We have to do this through Twill, in order to maintain app.toolbox and shed_tool_conf.xml in a state that is valid for future tests. + for required_repository in repository.repository_dependencies: + repository_dict = dict( name=str( required_repository.name ), + owner=str( required_repository.owner ), + changeset_revision=str( required_repository.changeset_revision ) ) + # Generate a test method to uninstall this repository through the embedded Galaxy application's web interface. + test_install_repositories.generate_deactivate_or_uninstall_method( repository_dict, deactivate=False ) + log.debug( 'Changeset revision %s of %s repository %s selected for uninstallation.' % \ + ( str( required_repository.changeset_revision ), str( required_repository.status ), str( required_repository.name ) ) ) + repository_dict = dict( name=name, owner=owner, changeset_revision=changeset_revision ) + test_install_repositories.generate_deactivate_or_uninstall_method( repository_dict, deactivate=False ) + log.debug( 'Changeset revision %s of %s repository %s selected for uninstallation.' % ( changeset_revision, str( repository.status ), name ) ) + # Set up nose to run the generated uninstall method as a functional test. + test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) + test_config.configure( sys.argv ) + # Run the uninstall method. This method uses the Galaxy web interface to uninstall the previously installed + # repository and delete it from disk. + result, _ = run_tests( test_config ) + success = result.wasSuccessful() + if not success: + log.debug( 'Repository %s failed to uninstall.' % str( name ) ) + +def uninstall_tool_dependency( app, tool_dependency ): + """Attempt to uninstall a tool dependency.""" + sa_session = app.model.context.current + # Clean out any generated tests. This is necessary for Twill. + tool_dependency_install_path = tool_dependency.installation_directory( app ) + uninstalled, error_message = tool_dependency_util.remove_tool_dependency( app, tool_dependency ) + if error_message: + log.debug( 'There was an error attempting to remove directory: %s' % str( tool_dependency_install_path ) ) + log.debug( error_message ) + else: + log.debug( 'Successfully removed tool dependency installation directory: %s' % str( tool_dependency_install_path ) ) + if not uninstalled or tool_dependency.status != app.model.ToolDependency.installation_status.UNINSTALLED: + tool_dependency.status = app.model.ToolDependency.installation_status.UNINSTALLED + sa_session.add( tool_dependency ) + sa_session.flush() + if os.path.exists( tool_dependency_install_path ): + log.debug( 'Uninstallation of tool dependency succeeded, but the installation path still exists on the filesystem. It is now being explicitly deleted.') + suc.remove_dir( tool_dependency_install_path ) + if __name__ == "__main__": # The tool_test_results_dict should always have the following structure: # { Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Clean up the install_repository method in twilltestcase.py. Fix the update manager test when running all functional tests.
by commits-noreply＠bitbucket.org 26 Nov '13

26 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/4cfffc6fce30/ Changeset: 4cfffc6fce30 User: Dave Bouvier Date: 2013-11-26 17:01:47 Summary: Clean up the install_repository method in twilltestcase.py. Fix the update manager test when running all functional tests. Affected #: 3 files diff -r 359a822b2e5da89973ce042942a9a10dd2b229e1 -r 4cfffc6fce30b9dd89849975c473e188936318ad test/install_and_test_tool_shed_repositories/base/twilltestcase.py --- a/test/install_and_test_tool_shed_repositories/base/twilltestcase.py +++ b/test/install_and_test_tool_shed_repositories/base/twilltestcase.py @@ -27,6 +27,21 @@ self.shed_tools_dict = {} self.home() + def deactivate_or_uninstall_repository( self, installed_repository, deactivate=False ): + url = '/admin_toolshed/deactivate_or_uninstall_repository?id=%s' % self.security.encode_id( installed_repository.id ) + self.visit_url( url ) + if deactivate: + tc.fv ( 1, "remove_from_disk", 'false' ) + else: + tc.fv ( 1, "remove_from_disk", 'true' ) + tc.submit( 'deactivate_or_uninstall_repository_button' ) + strings_displayed = [ 'The repository named' ] + if deactivate: + strings_displayed.append( 'has been deactivated' ) + else: + strings_displayed.append( 'has been uninstalled' ) + self.check_for_strings( strings_displayed, strings_not_displayed=[] ) + def initiate_installation_process( self, install_tool_dependencies=False, install_repository_dependencies=True, @@ -80,36 +95,40 @@ self.visit_url( url ) # This section is tricky, due to the way twill handles form submission. The tool dependency checkbox needs to # be hacked in through tc.browser, putting the form field in kwd doesn't work. - if 'install_tool_dependencies' in self.last_page(): - form = tc.browser.get_form( 'select_tool_panel_section' ) - checkbox = form.find_control( id="install_tool_dependencies" ) - checkbox.disabled = False - if install_tool_dependencies: - checkbox.selected = True - kwd[ 'install_tool_dependencies' ] = 'True' - else: - checkbox.selected = False - kwd[ 'install_tool_dependencies' ] = 'False' - if 'install_repository_dependencies' in self.last_page(): - form = tc.browser.get_form( 'select_tool_panel_section' ) - checkbox = form.find_control( id="install_repository_dependencies" ) - checkbox.disabled = False - if install_repository_dependencies: - checkbox.selected = True - kwd[ 'install_repository_dependencies' ] = 'True' - else: - checkbox.selected = False - kwd[ 'install_repository_dependencies' ] = 'False' - if 'shed_tool_conf' not in kwd: - kwd[ 'shed_tool_conf' ] = self.shed_tool_conf - if new_tool_panel_section: - kwd[ 'new_tool_panel_section' ] = new_tool_panel_section - self.submit_form( 1, 'select_tool_panel_section_button', **kwd ) + form = tc.browser.get_form( 'select_tool_panel_section' ) + if form is None: + form = tc.browser.get_form( 'select_shed_tool_panel_config' ) + assert form is not None, 'Could not find form select_shed_tool_panel_config or select_tool_panel_section.' + kwd = self.set_form_value( form, kwd, 'install_tool_dependencies', install_tool_dependencies ) + kwd = self.set_form_value( form, kwd, 'install_repository_dependencies', install_repository_dependencies ) + kwd = self.set_form_value( form, kwd, 'shed_tool_conf', self.shed_tool_conf ) + if new_tool_panel_section is not None: + kwd = self.set_form_value( form, kwd, 'new_tool_panel_section', new_tool_panel_section ) + submit_button_control = form.find_control( type='submit' ) + assert submit_button_control is not None, 'No submit button found for form %s.' % form.attrs.get( 'id' ) + self.submit_form( form.attrs.get( 'id' ), str( submit_button_control.name ), **kwd ) self.check_for_strings( post_submit_strings_displayed, strings_not_displayed ) repository_ids = self.initiate_installation_process( new_tool_panel_section=new_tool_panel_section ) log.debug( 'Waiting for the installation of repository IDs: %s' % str( repository_ids ) ) self.wait_for_repository_installation( repository_ids ) + def set_form_value( self, form, kwd, field_name, field_value ): + ''' + Set the form field field_name to field_value if it exists, and return the provided dict containing that value. If + the field does not exist in the provided form, return a dict without that index. + ''' + form_id = form.attrs.get( 'id' ) + controls = [ control for control in form.controls if str( control.name ) == field_name ] + if len( controls ) > 0: + log.debug( 'Setting field %s of form %s to %s.' % ( field_name, form_id, str( field_value ) ) ) + tc.formvalue( form_id, field_name, str( field_value ) ) + kwd[ field_name ] = str( field_value ) + else: + if field_name in kwd: + log.debug( 'No field %s in form %s, discarding from return value.' % ( str( control ), str( form_id ) ) ) + del( kwd[ field_name ] ) + return kwd + def visit_url( self, url, allowed_codes=[ 200 ] ): new_url = tc.go( url ) return_code = tc.browser.get_code() @@ -140,17 +159,3 @@ break time.sleep( 1 ) - def deactivate_or_uninstall_repository( self, installed_repository, deactivate=False ): - url = '/admin_toolshed/deactivate_or_uninstall_repository?id=%s' % self.security.encode_id( installed_repository.id ) - self.visit_url( url ) - if deactivate: - tc.fv ( 1, "remove_from_disk", 'false' ) - else: - tc.fv ( 1, "remove_from_disk", 'true' ) - tc.submit( 'deactivate_or_uninstall_repository_button' ) - strings_displayed = [ 'The repository named' ] - if deactivate: - strings_displayed.append( 'has been deactivated' ) - else: - strings_displayed.append( 'has been uninstalled' ) - self.check_for_strings( strings_displayed, strings_not_displayed=[] ) diff -r 359a822b2e5da89973ce042942a9a10dd2b229e1 -r 4cfffc6fce30b9dd89849975c473e188936318ad test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -868,34 +868,41 @@ ( changeset_revision, repository_id, self.galaxy_url ) self.visit_url( url ) self.check_for_strings( strings_displayed, strings_not_displayed ) - # This section is tricky, due to the way twill handles form submission. The tool dependency checkbox needs to + # This section is tricky, due to the way twill handles form submission. The tool dependency checkbox needs to # be hacked in through tc.browser, putting the form field in kwd doesn't work. form = tc.browser.get_form( 'select_tool_panel_section' ) - submit_button = 'select_tool_panel_section_button' if form is None: form = tc.browser.get_form( 'select_shed_tool_panel_config' ) - submit_button = 'select_shed_tool_panel_config_button' - if 'install_tool_dependencies' in self.last_page(): - checkbox = form.find_control( id="install_tool_dependencies" ) - checkbox.disabled = False - if install_tool_dependencies: - checkbox.selected = True - kwd[ 'install_tool_dependencies' ] = 'True' - else: - checkbox.selected = False - kwd[ 'install_tool_dependencies' ] = 'False' - if 'install_repository_dependencies' in self.last_page(): - kwd[ 'install_repository_dependencies' ] = str( install_repository_dependencies ).lower() - if 'shed_tool_conf' not in kwd: - kwd[ 'shed_tool_conf' ] = self.shed_tool_conf - if new_tool_panel_section: - kwd[ 'new_tool_panel_section' ] = new_tool_panel_section - if not includes_tools_for_display_in_tool_panel: - self.check_for_strings( strings_displayed=[ 'Choose the configuration file' ] ) - self.submit_form( 1, submit_button, **kwd ) + assert form is not None, 'Could not find form select_shed_tool_panel_config or select_tool_panel_section.' + kwd = self.set_form_value( form, kwd, 'install_tool_dependencies', install_tool_dependencies ) + kwd = self.set_form_value( form, kwd, 'install_repository_dependencies', install_repository_dependencies ) + kwd = self.set_form_value( form, kwd, 'shed_tool_conf', self.shed_tool_conf ) + if new_tool_panel_section is not None: + kwd = self.set_form_value( form, kwd, 'new_tool_panel_section', new_tool_panel_section ) + submit_button_control = form.find_control( type='submit' ) + assert submit_button_control is not None, 'No submit button found for form %s.' % form.attrs.get( 'id' ) + self.submit_form( form.attrs.get( 'id' ), str( submit_button_control.name ), **kwd ) self.check_for_strings( post_submit_strings_displayed, strings_not_displayed ) repository_ids = self.initiate_installation_process( new_tool_panel_section=new_tool_panel_section ) + log.debug( 'Waiting for the installation of repository IDs: %s' % str( repository_ids ) ) self.wait_for_repository_installation( repository_ids ) + + def set_form_value( self, form, kwd, field_name, field_value ): + ''' + Set the form field field_name to field_value if it exists, and return the provided dict containing that value. If + the field does not exist in the provided form, return a dict without that index. + ''' + form_id = form.attrs.get( 'id' ) + controls = [ control for control in form.controls if str( control.name ) == field_name ] + if len( controls ) > 0: + log.debug( 'Setting field %s of form %s to %s.' % ( field_name, form_id, str( field_value ) ) ) + tc.formvalue( form_id, field_name, str( field_value ) ) + kwd[ field_name ] = str( field_value ) + else: + if field_name in kwd: + log.debug( 'No field %s in form %s, discarding from return value.' % ( str( control ), str( form_id ) ) ) + del( kwd[ field_name ] ) + return kwd def load_citable_url( self, username, diff -r 359a822b2e5da89973ce042942a9a10dd2b229e1 -r 4cfffc6fce30b9dd89849975c473e188936318ad test/tool_shed/functional/test_1410_update_manager.py --- a/test/tool_shed/functional/test_1410_update_manager.py +++ b/test/tool_shed/functional/test_1410_update_manager.py @@ -126,9 +126,6 @@ ok_icon = '/static/june_2007_style/blue/ok_small.png' ok_title = 'This is the latest installable revision of this repository' updates_icon = '/static/images/icon_warning_sml.gif' - repository_id = self.security.encode_id( repository.id ) - html = '<label id="%s" for="%s"><img src="%s" class="icon-button" title="%s"/><img src="%s' % \ - ( repository_id, repository_id, ok_icon, ok_title, updates_icon ) - strings_displayed = [ html ] + strings_displayed = [ '<img src="%s" class="icon-button" title="%s"/><img src="%s' % ( ok_icon, ok_title, updates_icon ) ] self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 4 new changesets
by commits-noreply＠bitbucket.org 26 Nov '13

26 Nov '13

4 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/934fe4d500ec/ Changeset: 934fe4d500ec Branch: page-api User: Kyle Ellrott Date: 2013-11-21 22:50:59 Summary: Adding page search to the search API. Also adding plural id encoding (a list named '*_ids' will get id encoded) Affected #: 3 files diff -r 253f888144aaa4ae4eadaf15de493b3144fc5cd3 -r 934fe4d500ec7fba8d41c634eb70dc0381dee41c lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3140,7 +3140,8 @@ self.session = session self.openid = openid -class Page( object ): +class Page( object, Dictifiable ): + dict_element_visible_keys = [ 'id', 'title', 'latest_revision_id', 'slug' ] def __init__( self ): self.id = None self.user = None @@ -3151,6 +3152,14 @@ self.importable = None self.published = None + def to_dict( self, view='element' ): + rval = super( Page, self ).to_dict( view=view ) + rev = [] + for a in self.revisions: + rev.append(a.id) + rval['revision_ids'] = rev + return rval + class PageRevision( object ): def __init__( self ): self.user = None diff -r 253f888144aaa4ae4eadaf15de493b3144fc5cd3 -r 934fe4d500ec7fba8d41c634eb70dc0381dee41c lib/galaxy/model/search.py --- a/lib/galaxy/model/search.py +++ b/lib/galaxy/model/search.py @@ -35,7 +35,8 @@ History, Library, LibraryFolder, LibraryDataset,StoredWorkflowTagAssociation, StoredWorkflow, HistoryTagAssociation,HistoryDatasetAssociationTagAssociation, ExtendedMetadata, ExtendedMetadataIndex, HistoryAnnotationAssociation, Job, JobParameter, -JobToInputLibraryDatasetAssociation, JobToInputDatasetAssociation, JobToOutputDatasetAssociation, ToolVersion ) +JobToInputLibraryDatasetAssociation, JobToInputDatasetAssociation, JobToOutputDatasetAssociation, ToolVersion, +Page ) from galaxy.util.json import to_json_string from sqlalchemy import and_ @@ -469,6 +470,20 @@ +################## +#Page Searching +################## + +class PageView(ViewQueryBaseClass): + DOMAIN = "page" + FIELDS = { + 'id' : ViewField('id', sqlalchemy_field=Page.id, id_decode=True), + 'title' : ViewField('title', sqlalchemy_field=Page.title), + } + + def search(self, trans): + self.query = trans.sa_session.query( Page ) + """ The view mapping takes a user's name for a table and maps it to a View class that will handle queries @@ -486,6 +501,7 @@ 'workflow' : WorkflowView, 'tool' : ToolView, 'job' : JobView, + 'page' : PageView } """ diff -r 253f888144aaa4ae4eadaf15de493b3144fc5cd3 -r 934fe4d500ec7fba8d41c634eb70dc0381dee41c lib/galaxy/web/base/controller.py --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -166,6 +166,14 @@ rval[k] = trans.security.encode_id( v ) except: pass # probably already encoded + if (k.endswith("_ids") and type(v) == list): + try: + o = [] + for i in v: + o.append(trans.security.encode_id( i )) + rval[k] = o + except: + pass else: if recursive and type(v) == dict: rval[k] = self.encode_all_ids(trans, v, recursive) https://bitbucket.org/galaxy/galaxy-central/commits/ebe7a63e01e9/ Changeset: ebe7a63e01e9 Branch: page-api User: Kyle Ellrott Date: 2013-11-21 23:07:52 Summary: Adding page revisions to search api Affected #: 2 files diff -r 934fe4d500ec7fba8d41c634eb70dc0381dee41c -r ebe7a63e01e91bc75e447c5c159a2f142b7b894c lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3160,12 +3160,19 @@ rval['revision_ids'] = rev return rval -class PageRevision( object ): +class PageRevision( object, Dictifiable ): + dict_element_visible_keys = [ 'id', 'page_id', 'title', 'content' ] def __init__( self ): self.user = None self.title = None self.content = None + def to_dict( self, view='element' ): + rval = super( PageRevision, self ).to_dict( view=view ) + rval['create_time'] = str(self.create_time) + rval['update_time'] = str(self.update_time) + return rval + class PageUserShareAssociation( object ): def __init__( self ): self.page = None diff -r 934fe4d500ec7fba8d41c634eb70dc0381dee41c -r ebe7a63e01e91bc75e447c5c159a2f142b7b894c lib/galaxy/model/search.py --- a/lib/galaxy/model/search.py +++ b/lib/galaxy/model/search.py @@ -36,7 +36,7 @@ StoredWorkflow, HistoryTagAssociation,HistoryDatasetAssociationTagAssociation, ExtendedMetadata, ExtendedMetadataIndex, HistoryAnnotationAssociation, Job, JobParameter, JobToInputLibraryDatasetAssociation, JobToInputDatasetAssociation, JobToOutputDatasetAssociation, ToolVersion, -Page ) +Page, PageRevision ) from galaxy.util.json import to_json_string from sqlalchemy import and_ @@ -484,6 +484,27 @@ def search(self, trans): self.query = trans.sa_session.query( Page ) + + + +################## +#Page Revision Searching +################## + + +class PageRevisionView(ViewQueryBaseClass): + DOMAIN = "page_revision" + FIELDS = { + 'id' : ViewField('id', sqlalchemy_field=PageRevision.id, id_decode=True), + 'title' : ViewField('title', sqlalchemy_field=PageRevision.title), + 'page_id' : ViewField('page_id', sqlalchemy_field=PageRevision.page_id, id_decode=True), + } + + def search(self, trans): + self.query = trans.sa_session.query( PageRevision ) + + + """ The view mapping takes a user's name for a table and maps it to a View class that will handle queries @@ -501,7 +522,8 @@ 'workflow' : WorkflowView, 'tool' : ToolView, 'job' : JobView, - 'page' : PageView + 'page' : PageView, + 'page_revision' : PageRevisionView, } """ https://bitbucket.org/galaxy/galaxy-central/commits/d50335029705/ Changeset: d50335029705 Branch: page-api User: Kyle Ellrott Date: 2013-11-21 23:29:42 Summary: Adding security checks to pages found via search api. Also fixing some bugs related to search security checks. Affected #: 1 file diff -r ebe7a63e01e91bc75e447c5c159a2f142b7b894c -r d50335029705d4e587a7a6428b9da6e4fc4890cf lib/galaxy/webapps/galaxy/api/search.py --- a/lib/galaxy/webapps/galaxy/api/search.py +++ b/lib/galaxy/webapps/galaxy/api/search.py @@ -5,7 +5,7 @@ from galaxy import web from galaxy.web.base.controller import SharableItemSecurityMixin, BaseAPIController from galaxy.model.search import GalaxySearchEngine - +from galaxy.exceptions import ItemAccessibilityException log = logging.getLogger( __name__ ) @@ -37,15 +37,28 @@ if trans.user_is_admin(): append = True if not append: - if type( item ) in ( trans.app.model.LibraryFolder, trans.app.model.LibraryDatasetDatasetAssociation, trans.app.model.LibraryDataset ): + if type( item ) in [ trans.app.model.LibraryFolder, trans.app.model.LibraryDatasetDatasetAssociation, trans.app.model.LibraryDataset ]: if (trans.app.security_agent.can_access_library_item( trans.get_current_user_roles(), item, trans.user ) ): append = True - elif type( item ) in trans.app.model.Job: + elif type( item ) in [ trans.app.model.Job ]: if item.used_id == trans.user or trans.user_is_admin(): append = True + elif type( item ) in [ trans.app.model.Page ]: + try: + if self.security_check( trans, item, False, True): + append = True + except ItemAccessibilityException: + append = False + elif type ( item ) in [ trans.app.model.PageRevision ]: + try: + if self.security_check( trans, item.page, False, True): + append = True + except ItemAccessibilityException: + append = False elif hasattr(item, 'dataset'): if trans.app.security_agent.can_access_dataset( current_user_roles, item.dataset ): append = True + if append: row = query.item_to_api_value(item) out.append( self.encode_all_ids( trans, row, True) ) https://bitbucket.org/galaxy/galaxy-central/commits/359a822b2e5d/ Changeset: 359a822b2e5d User: jmchilton Date: 2013-11-26 15:04:27 Summary: Merged in kellrott/galaxy-central/page-api (pull request #266) Adding Pages and PageRevisions to the search API Affected #: 4 files diff -r c0384bad246d60a0ca737685cc67eb17331b36fa -r 359a822b2e5da89973ce042942a9a10dd2b229e1 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3140,7 +3140,8 @@ self.session = session self.openid = openid -class Page( object ): +class Page( object, Dictifiable ): + dict_element_visible_keys = [ 'id', 'title', 'latest_revision_id', 'slug' ] def __init__( self ): self.id = None self.user = None @@ -3151,12 +3152,27 @@ self.importable = None self.published = None -class PageRevision( object ): + def to_dict( self, view='element' ): + rval = super( Page, self ).to_dict( view=view ) + rev = [] + for a in self.revisions: + rev.append(a.id) + rval['revision_ids'] = rev + return rval + +class PageRevision( object, Dictifiable ): + dict_element_visible_keys = [ 'id', 'page_id', 'title', 'content' ] def __init__( self ): self.user = None self.title = None self.content = None + def to_dict( self, view='element' ): + rval = super( PageRevision, self ).to_dict( view=view ) + rval['create_time'] = str(self.create_time) + rval['update_time'] = str(self.update_time) + return rval + class PageUserShareAssociation( object ): def __init__( self ): self.page = None diff -r c0384bad246d60a0ca737685cc67eb17331b36fa -r 359a822b2e5da89973ce042942a9a10dd2b229e1 lib/galaxy/model/search.py --- a/lib/galaxy/model/search.py +++ b/lib/galaxy/model/search.py @@ -35,7 +35,8 @@ History, Library, LibraryFolder, LibraryDataset,StoredWorkflowTagAssociation, StoredWorkflow, HistoryTagAssociation,HistoryDatasetAssociationTagAssociation, ExtendedMetadata, ExtendedMetadataIndex, HistoryAnnotationAssociation, Job, JobParameter, -JobToInputLibraryDatasetAssociation, JobToInputDatasetAssociation, JobToOutputDatasetAssociation, ToolVersion ) +JobToInputLibraryDatasetAssociation, JobToInputDatasetAssociation, JobToOutputDatasetAssociation, ToolVersion, +Page, PageRevision ) from galaxy.util.json import to_json_string from sqlalchemy import and_ @@ -469,6 +470,41 @@ +################## +#Page Searching +################## + +class PageView(ViewQueryBaseClass): + DOMAIN = "page" + FIELDS = { + 'id' : ViewField('id', sqlalchemy_field=Page.id, id_decode=True), + 'title' : ViewField('title', sqlalchemy_field=Page.title), + } + + def search(self, trans): + self.query = trans.sa_session.query( Page ) + + + + +################## +#Page Revision Searching +################## + + +class PageRevisionView(ViewQueryBaseClass): + DOMAIN = "page_revision" + FIELDS = { + 'id' : ViewField('id', sqlalchemy_field=PageRevision.id, id_decode=True), + 'title' : ViewField('title', sqlalchemy_field=PageRevision.title), + 'page_id' : ViewField('page_id', sqlalchemy_field=PageRevision.page_id, id_decode=True), + } + + def search(self, trans): + self.query = trans.sa_session.query( PageRevision ) + + + """ The view mapping takes a user's name for a table and maps it to a View class that will handle queries @@ -486,6 +522,8 @@ 'workflow' : WorkflowView, 'tool' : ToolView, 'job' : JobView, + 'page' : PageView, + 'page_revision' : PageRevisionView, } """ diff -r c0384bad246d60a0ca737685cc67eb17331b36fa -r 359a822b2e5da89973ce042942a9a10dd2b229e1 lib/galaxy/web/base/controller.py --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -166,6 +166,14 @@ rval[k] = trans.security.encode_id( v ) except: pass # probably already encoded + if (k.endswith("_ids") and type(v) == list): + try: + o = [] + for i in v: + o.append(trans.security.encode_id( i )) + rval[k] = o + except: + pass else: if recursive and type(v) == dict: rval[k] = self.encode_all_ids(trans, v, recursive) diff -r c0384bad246d60a0ca737685cc67eb17331b36fa -r 359a822b2e5da89973ce042942a9a10dd2b229e1 lib/galaxy/webapps/galaxy/api/search.py --- a/lib/galaxy/webapps/galaxy/api/search.py +++ b/lib/galaxy/webapps/galaxy/api/search.py @@ -5,7 +5,7 @@ from galaxy import web from galaxy.web.base.controller import SharableItemSecurityMixin, BaseAPIController from galaxy.model.search import GalaxySearchEngine - +from galaxy.exceptions import ItemAccessibilityException log = logging.getLogger( __name__ ) @@ -37,15 +37,28 @@ if trans.user_is_admin(): append = True if not append: - if type( item ) in ( trans.app.model.LibraryFolder, trans.app.model.LibraryDatasetDatasetAssociation, trans.app.model.LibraryDataset ): + if type( item ) in [ trans.app.model.LibraryFolder, trans.app.model.LibraryDatasetDatasetAssociation, trans.app.model.LibraryDataset ]: if (trans.app.security_agent.can_access_library_item( trans.get_current_user_roles(), item, trans.user ) ): append = True - elif type( item ) in trans.app.model.Job: + elif type( item ) in [ trans.app.model.Job ]: if item.used_id == trans.user or trans.user_is_admin(): append = True + elif type( item ) in [ trans.app.model.Page ]: + try: + if self.security_check( trans, item, False, True): + append = True + except ItemAccessibilityException: + append = False + elif type ( item ) in [ trans.app.model.PageRevision ]: + try: + if self.security_check( trans, item.page, False, True): + append = True + except ItemAccessibilityException: + append = False elif hasattr(item, 'dataset'): if trans.app.security_agent.can_access_dataset( current_user_roles, item.dataset ): append = True + if append: row = query.item_to_api_value(item) out.append( self.encode_all_ids( trans, row, True) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix for the previous fix for rendering the "Skip tool test" section on the manage_repository page in the tool shed.
by commits-noreply＠bitbucket.org 26 Nov '13

26 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c0384bad246d/ Changeset: c0384bad246d User: greg Date: 2013-11-26 00:45:37 Summary: Fix for the previous fix for rendering the "Skip tool test" section on the manage_repository page in the tool shed. Affected #: 1 file diff -r 1a8070a3b08a65ae386b849548ecf921d916cf7a -r c0384bad246d60a0ca737685cc67eb17331b36fa templates/webapps/tool_shed/repository/manage_repository.mako --- a/templates/webapps/tool_shed/repository/manage_repository.mako +++ b/templates/webapps/tool_shed/repository/manage_repository.mako @@ -72,6 +72,17 @@ tip_str = '' sharable_link_label = 'Sharable link to this repository revision:' sharable_link_changeset_revision = changeset_revision + + if repository_metadata is None: + can_render_skip_tool_test_section = False + else: + if repository_metadata.changeset_revision is None: + can_render_skip_tool_test_section = False + else: + if includes_tools or repository.type == TOOL_DEPENDENCY_DEFINITION: + can_render_skip_tool_test_section = True + else: + can_render_skip_tool_test_section = False %><%! @@ -200,7 +211,7 @@ </div></div> ${render_repository_items( metadata, containers_dict, can_set_metadata=True, render_repository_actions_for='tool_shed' )} -%if includes_tools or repository.type == TOOL_DEPENDENCY_DEFINITION: +%if can_render_skip_tool_test_section: <p/><div class="toolForm"> %if repository.type == TOOL_DEPENDENCY_DEFINITION: @@ -209,7 +220,7 @@ <div class="toolFormTitle">Automated tool tests</div> %endif <div class="toolFormBody"> - <form name="skip_tool_tests" id="skip_tool_tests" action="${h.url_for( controller='repository', action='manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=changeset_revision )}" method="post" > + <form name="skip_tool_tests" id="skip_tool_tests" action="${h.url_for( controller='repository', action='manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=str( repository_metadata.changeset_revision ) )}" method="post" ><div class="form-row"> %if repository.type == TOOL_DEPENDENCY_DEFINITION: <label>Skip automated testing of this tool dependency recipe</label> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: afgane: Revert API tool installation fix because it break the GUI installs; API will still need a fix
by commits-noreply＠bitbucket.org 26 Nov '13

26 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1a8070a3b08a/ Changeset: 1a8070a3b08a User: afgane Date: 2013-11-26 00:06:21 Summary: Revert API tool installation fix because it break the GUI installs; API will still need a fix Affected #: 1 file diff -r 011df6fa309e9b4a052d5342218e888deabd063f -r 1a8070a3b08a65ae386b849548ecf921d916cf7a lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -673,7 +673,7 @@ tool_panel_section_id=section_id, new_tool_panel_section=new_tool_panel_section ) elif tool_panel_section: - tool_panel_section_key = 'section_%s' % str( tool_panel_section.id ) + tool_panel_section_key = 'section_%s' % str( tool_panel_section ) tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ] else: return None, None Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix for rendering Skip tool tests" form on the manage_repository page in the tool shed.
by commits-noreply＠bitbucket.org 25 Nov '13

25 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/011df6fa309e/ Changeset: 011df6fa309e User: greg Date: 2013-11-25 22:53:40 Summary: Fix for rendering Skip tool tests" form on the manage_repository page in the tool shed. Affected #: 1 file diff -r 11eb2e028791aca01c206fa5e898e1feb28f4651 -r 011df6fa309e9b4a052d5342218e888deabd063f templates/webapps/tool_shed/repository/manage_repository.mako --- a/templates/webapps/tool_shed/repository/manage_repository.mako +++ b/templates/webapps/tool_shed/repository/manage_repository.mako @@ -209,7 +209,7 @@ <div class="toolFormTitle">Automated tool tests</div> %endif <div class="toolFormBody"> - <form name="skip_tool_tests" id="skip_tool_tests" action="${h.url_for( controller='repository', action='manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=repository_metadata.changeset_revision )}" method="post" > + <form name="skip_tool_tests" id="skip_tool_tests" action="${h.url_for( controller='repository', action='manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=changeset_revision )}" method="post" ><div class="form-row"> %if repository.type == TOOL_DEPENDENCY_DEFINITION: <label>Skip automated testing of this tool dependency recipe</label> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Minor fixes for the prep script for installing and testing tool shed repositories of type tool_dependency_definition.
by commits-noreply＠bitbucket.org 25 Nov '13

25 Nov '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/11eb2e028791/ Changeset: 11eb2e028791 User: greg Date: 2013-11-25 22:43:01 Summary: Minor fixes for the prep script for installing and testing tool shed repositories of type tool_dependency_definition. Affected #: 1 file diff -r ec2a03eea18ee82c71618d291394da81414cfb45 -r 11eb2e028791aca01c206fa5e898e1feb28f4651 lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py --- a/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py +++ b/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py @@ -28,7 +28,9 @@ from base.util import get_database_version from base.util import get_repository_current_revision from base.util import get_test_environment -from galaxy.model.orm import and_, not_, select +from galaxy.model.orm import and_ +from galaxy.model.orm import not_ +from galaxy.model.orm import select from galaxy.web import url_for from tool_shed.repository_types.util import TOOL_DEPENDENCY_DEFINITION @@ -138,21 +140,6 @@ app.model.RepositoryMetadata.table.c.repository_id.in_( tool_dependency_defintion_repository_ids ), not_( app.model.RepositoryMetadata.table.c.id.in_( skip_metadata_ids ) ) ) ): records_checked += 1 - # Create the repository_status dictionary, using the dictionary from the previous test run if available. - if repository_metadata.tool_test_results: - repository_status = repository_metadata.tool_test_results - else: - repository_status = {} - # Initialize the repository_status dictionary with the information about the current test environment. - last_test_environment = repository_status.get( 'test_environment', None ) - if last_test_environment is None: - test_environment = get_test_environment() - else: - test_environment = get_test_environment( last_test_environment ) - test_environment[ 'tool_shed_database_version' ] = get_database_version( app ) - test_environment[ 'tool_shed_mercurial_version' ] = __version__.version - test_environment[ 'tool_shed_revision' ] = get_repository_current_revision( os.getcwd() ) - repository_status[ 'test_environment' ] = test_environment # Check the next repository revision. changeset_revision = str( repository_metadata.changeset_revision ) name = repository.name @@ -174,7 +161,18 @@ print 'Revision %s of %s owned by %s has invalid metadata.' % ( changeset_revision, name, owner ) invalid_metadata += 1 if not info_only: - repository_metadata.tool_test_results = repository_status + # Create the tool_test_results_dict dictionary, using the dictionary from the previous test run if available. + if repository_metadata.tool_test_results: + tool_test_results_dict = repository_metadata.tool_test_results + else: + tool_test_results_dict = {} + # Initialize the tool_test_results_dict dictionary with the information about the current test environment. + test_environment_dict = tool_test_results_dict.get( 'test_environment', {} ) + test_environment_dict[ 'tool_shed_database_version' ] = get_database_version( app ) + test_environment_dict[ 'tool_shed_mercurial_version' ] = __version__.version + test_environment_dict[ 'tool_shed_revision' ] = get_repository_current_revision( os.getcwd() ) + tool_test_results_dict[ 'test_environment' ] = test_environment_dict + repository_metadata.tool_test_results = tool_test_results_dict app.sa_session.add( repository_metadata ) app.sa_session.flush() stop = time.time() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 25 Nov '13

25 Nov '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f8fc2bc86635/ Changeset: f8fc2bc86635 User: Dave Bouvier Date: 2013-11-25 22:23:51 Summary: Add API method to list datatype sniffers. Add functional tests for repositories defining datatype sniffers. Affected #: 1 file diff -r d0a3879c0708983fccb24b809bfe29d96a8dbb59 -r f8fc2bc8663593075a820157606cca11b905e0e9 test/tool_shed/test_data/proteomics_datatypes/proteomics_datatypes.tar Binary file test/tool_shed/test_data/proteomics_datatypes/proteomics_datatypes.tar has changed https://bitbucket.org/galaxy/galaxy-central/commits/ec2a03eea18e/ Changeset: ec2a03eea18e User: Dave Bouvier Date: 2013-11-25 22:29:41 Summary: Add API method to list datatype sniffers. Add functional tests for repositories defining datatype sniffers. Affected #: 4 files diff -r f8fc2bc8663593075a820157606cca11b905e0e9 -r ec2a03eea18ee82c71618d291394da81414cfb45 lib/galaxy/webapps/galaxy/api/datatypes.py --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -4,20 +4,45 @@ from galaxy import web from galaxy.web.base.controller import BaseAPIController +from galaxy.util import asbool import logging log = logging.getLogger( __name__ ) + class DatatypesController( BaseAPIController ): + @web.expose_api_anonymous def index( self, trans, **kwd ): """ GET /api/datatypes - Return an object containing datatypes. + Return an object containing upload datatypes. """ + upload_only = asbool( kwd.get( 'upload_only', True ) ) try: - return trans.app.datatypes_registry.upload_file_formats + if upload_only: + return trans.app.datatypes_registry.upload_file_formats + else: + return [ ext for ext in trans.app.datatypes_registry.datatypes_by_extension ] except Exception, exception: log.error( 'could not get datatypes: %s', str( exception ), exc_info=True ) trans.response.status = 500 return { 'error': str( exception ) } + + @web.expose_api_anonymous + def sniffers( self, trans, **kwd ): + ''' + GET /api/datatypes/sniffers + Return a list of sniffers. + ''' + try: + rval = [] + for sniffer_elem in trans.app.datatypes_registry.sniffer_elems: + datatype = sniffer_elem.get( 'type' ) + if datatype is not None: + rval.append( datatype ) + return rval + except Exception, exception: + log.error( 'could not get datatypes: %s', str( exception ), exc_info=True ) + trans.response.status = 500 + return { 'error': str( exception ) } diff -r f8fc2bc8663593075a820157606cca11b905e0e9 -r ec2a03eea18ee82c71618d291394da81414cfb45 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -163,7 +163,11 @@ webapp.mapper.resource( 'workflow', 'workflows', path_prefix='/api' ) webapp.mapper.resource_with_deleted( 'history', 'histories', path_prefix='/api' ) webapp.mapper.resource( 'configuration', 'configuration', path_prefix='/api' ) - webapp.mapper.resource( 'datatype', 'datatypes', path_prefix='/api' ) + webapp.mapper.resource( 'datatype', + 'datatypes', + path_prefix='/api', + collection={ 'sniffers': 'GET' }, + parent_resources=dict( member_name='datatype', collection_name='datatypes' ) ) #webapp.mapper.connect( 'run_workflow', '/api/workflow/{workflow_id}/library/{library_id}', controller='workflows', action='run', workflow_id=None, library_id=None, conditions=dict(method=["GET"]) ) webapp.mapper.resource( 'search', 'search', path_prefix='/api' ) diff -r f8fc2bc8663593075a820157606cca11b905e0e9 -r ec2a03eea18ee82c71618d291394da81414cfb45 test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -618,13 +618,11 @@ return temp_path def get_datatypes_count( self ): - url = '/admin/view_datatypes_registry' + url = '/api/datatypes?upload_only=false' self.visit_galaxy_url( url ) html = self.last_page() - datatypes_count = re.search( 'registry contains (\d+) data types', html ) - if datatypes_count: - return datatypes_count.group( 1 ) - return None + datatypes = from_json_string( html ) + return len( datatypes ) def get_env_sh_path( self, tool_dependency_name, tool_dependency_version, repository ): '''Return the absolute path to an installed repository's env.sh file.''' @@ -735,6 +733,13 @@ repo = self.get_hg_repo( self.get_repo_path( repository ) ) return str( repo.changectx( repo.changelog.tip() ) ) + def get_sniffers_count( self ): + url = '/api/datatypes/sniffers' + self.visit_galaxy_url( url ) + html = self.last_page() + sniffers = from_json_string( html ) + return len( sniffers ) + def get_tools_from_repository_metadata( self, repository, include_invalid=False ): '''Get a list of valid and (optionally) invalid tool dicts from the repository metadata.''' valid_tools = [] diff -r f8fc2bc8663593075a820157606cca11b905e0e9 -r ec2a03eea18ee82c71618d291394da81414cfb45 test/tool_shed/functional/test_1450_installing_datatypes_sniffers.py --- /dev/null +++ b/test/tool_shed/functional/test_1450_installing_datatypes_sniffers.py @@ -0,0 +1,191 @@ +from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os +import tool_shed.base.test_db_util as test_db_util + +repository_name = 'proteomics_datatypes_1450' +repository_description = "Proteomics datatypes" +repository_long_description = "Datatypes used in proteomics" + +category_name = 'Test 1450 Datatype Sniffers' +category_description = 'Test 1450 - Installing Datatype Sniffers' + + +''' +1. Get a count of datatypes and sniffers. +2. Install proteomics_datatypes_1450. +3. Verify the count of datatypes and sniffers is the previous count + the datatypes contained within proteomics_datatypes_1450. +4. Deactivate proteomics_datatypes_1450, verify the count of datatypes and sniffers is equal to the count determined in step 1. +5. Reactivate proteomics_datatypes_1450, verify that the count of datatypes and sniffers has been increased by the contents of the repository. +6. Uninstall proteomics_datatypes_1450, verify the count of datatypes and sniffers is equal to the count determined in step 1. +7. Reinstall proteomics_datatypes_1450, verify that the count of datatypes and sniffers has been increased by the contents of the repository. +''' + +base_datatypes_count = 0 +repository_datatypes_count = 0 +base_sniffers_count = 0 + + +class TestInstallDatatypesSniffers( ShedTwillTestCase ): + '''Test installing a repository that defines datatypes and datatype sniffers.''' + + def test_0000_initiate_users( self ): + """Create necessary user accounts.""" + global base_datatypes_count + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + test_user_1 = test_db_util.get_user( common.test_user_1_email ) + assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % test_user_1_email + test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + self.logout() + self.login( email=common.admin_email, username=common.admin_username ) + admin_user = test_db_util.get_user( common.admin_email ) + assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email + admin_user_private_role = test_db_util.get_private_role( admin_user ) + self.galaxy_logout() + self.galaxy_login( email=common.admin_email, username=common.admin_username ) + galaxy_admin_user = test_db_util.get_galaxy_user( common.admin_email ) + assert galaxy_admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email + galaxy_admin_user_private_role = test_db_util.get_galaxy_private_role( galaxy_admin_user ) + base_datatypes_count = self.get_datatypes_count() + base_sniffers_count = self.get_sniffers_count() + + def test_0005_ensure_repositories_and_categories_exist( self ): + '''Create the 1450 category and proteomics_datatypes_1450 repository.''' + global repository_datatypes_count + category = self.create_category( name=category_name, description=category_description ) + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + repository = self.get_or_create_repository( name=repository_name, + description=repository_description, + long_description=repository_long_description, + owner=common.test_user_1_name, + category_id=self.security.encode_id( category.id ), + strings_displayed=[] ) + if self.repository_is_new( repository ): + self.upload_file( repository, + filename='proteomics_datatypes/proteomics_datatypes.tar', + filepath=None, + valid_tools_only=True, + uncompress_file=False, + remove_repo_files_not_in_tar=False, + commit_message='Uploaded datatype and sniffer definitions.', + strings_displayed=[], + strings_not_displayed=[] ) + repository_datatypes_count = self.get_repository_datatypes_count( repository ) + + def test_0010_install_datatypes_repository( self ): + '''Install the proteomics_datatypes_1450 repository into the Galaxy instance.''' + ''' + This includes steps 1 and 2 - Get a count of datatypes and sniffers. + Store a count of the current datatypes registry and sniffers in global variables, to compare with the updated count + after changing the installation status of the proteomics_datatypes_1450 repository. + ''' + global repository_datatypes_count + global base_datatypes_count + global base_sniffers_count + base_sniffers_count = self.get_sniffers_count() + self.galaxy_logout() + self.galaxy_login( email=common.admin_email, username=common.admin_username ) + strings_displayed = [ 'proteomics' ] + self.install_repository( 'proteomics_datatypes_1450', + common.test_user_1_name, + category_name, + strings_displayed=strings_displayed, + new_tool_panel_section='test_1450' ) + installed_repository = test_db_util.get_installed_repository_by_name_owner( 'proteomics_datatypes_1450', common.test_user_1_name ) + strings_displayed = [ 'user1', + self.url.replace( 'http://', '' ), + installed_repository.installed_changeset_revision ] + self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed ) + + def test_0015_verify_datatypes_count( self ): + '''Verify that datatypes were added in the previous step.''' + ''' + This is step 3 - Verify the count of datatypes and sniffers is the previous count + the datatypes + contained within proteomics_datatypes_1450. + Compare the current datatypes registry and sniffers with the values that were retrieved in the previous step. + ''' + current_datatypes = self.get_datatypes_count() + assert current_datatypes == base_datatypes_count + repository_datatypes_count, \ + 'Found %d datatypes, expected %d.' % ( current_datatypes, base_datatypes_count + repository_datatypes_count ) + current_sniffers = self.get_sniffers_count() + assert current_sniffers > base_sniffers_count, \ + 'Sniffer count after installing proteomics_datatypes_1450 is %d, which is not greater than %d' % \ + ( current_sniffers, base_sniffers_count ) + + def test_0020_deactivate_datatypes_repository( self ): + '''Deactivate the installed proteomics_datatypes_1450 repository.''' + ''' + This is step 4 - Deactivate proteomics_datatypes_1450, verify the count of datatypes and sniffers is equal to + the count determined in step 1. + Deactivate proteomics_datatypes_1450 and check that the in-memory datatypes and sniffers match the base values + determined in the first step. + ''' + repository = test_db_util.get_installed_repository_by_name_owner( repository_name, common.test_user_1_name ) + global repository_datatypes_count + global base_datatypes_count + global base_sniffers_count + self.uninstall_repository( repository, remove_from_disk=False ) + new_datatypes_count = self.get_datatypes_count() + assert new_datatypes_count == base_datatypes_count, 'Expected %d datatypes, got %d' % ( base_datatypes_count, new_datatypes_count ) + current_sniffers = self.get_sniffers_count() + assert current_sniffers == base_sniffers_count, \ + 'Sniffer count after deactivating proteomics_datatypes_1450 is %d, expected %d' % \ + ( current_sniffers, base_sniffers_count ) + + def test_0025_reactivate_datatypes_repository( self ): + '''Reactivate the deactivated proteomics_datatypes_1450 repository.''' + ''' + This is step 5 - Reactivate proteomics_datatypes, verify that the count of datatypes and sniffers has been + increased by the contents of the repository. + ''' + repository = test_db_util.get_installed_repository_by_name_owner( repository_name, common.test_user_1_name ) + global repository_datatypes_count + global base_datatypes_count + global base_sniffers_count + self.reactivate_repository( repository ) + new_datatypes_count = self.get_datatypes_count() + assert new_datatypes_count == base_datatypes_count + repository_datatypes_count, \ + 'Found %d datatypes, expected %d.' % ( new_datatypes_count, base_datatypes_count + repository_datatypes_count ) + current_sniffers = self.get_sniffers_count() + assert current_sniffers > base_sniffers_count, \ + 'Sniffer count after reactivating proteomics_datatypes_1450 is %d, which is not greater than %d' % \ + ( current_sniffers, base_sniffers_count ) + + def test_0030_uninstall_datatypes_repository( self ): + '''Uninstall the installed proteomics_datatypes_1450 repository.''' + ''' + This is step 6 - Uninstall proteomics_datatypes_1450, verify the count of datatypes and sniffers is equal + to the count determined in step 1. + Uninstall proteomics_datatypes_1450 and check that the in-memory datatypes and sniffers match the base values + determined in the first step. + ''' + repository = test_db_util.get_installed_repository_by_name_owner( repository_name, common.test_user_1_name ) + global repository_datatypes_count + global base_datatypes_count + self.uninstall_repository( repository, remove_from_disk=True ) + new_datatypes_count = self.get_datatypes_count() + assert new_datatypes_count == base_datatypes_count, 'Expected %d datatypes, got %d' % ( base_datatypes_count, new_datatypes_count ) + current_sniffers = self.get_sniffers_count() + assert current_sniffers == base_sniffers_count, \ + 'Sniffer count after uninstalling proteomics_datatypes_1450 is %d, expected %d' % \ + ( current_sniffers, base_sniffers_count ) + + def test_0035_reinstall_datatypes_repository( self ): + '''Reinstall the uninstalled proteomics_datatypes_1450 repository.''' + ''' + This is step 7 - Reinstall proteomics_datatypes_1450, verify that the count of datatypes and sniffers has been + increased by the contents of the repository. + ''' + repository = test_db_util.get_installed_repository_by_name_owner( repository_name, common.test_user_1_name ) + global repository_datatypes_count + global base_datatypes_count + self.reinstall_repository( repository ) + new_datatypes_count = self.get_datatypes_count() + assert new_datatypes_count == base_datatypes_count + repository_datatypes_count, \ + 'Found %d datatypes, expected %d.' % ( new_datatypes_count, base_datatypes_count + repository_datatypes_count ) + current_sniffers = self.get_sniffers_count() + assert current_sniffers > base_sniffers_count, \ + 'Sniffer count after reinstalling proteomics_datatypes_1450 is %d, which is not greater than %d' % \ + ( current_sniffers, base_sniffers_count ) + + Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0