2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/01a572fb8b4c/ Changeset: 01a572fb8b4c User: inithello Date: 2014-02-20 17:18:30 Summary: Enable specifying the install database connection when executing the install and test framework. Affected #: 2 files diff -r 952558f0dc0ba52355e87ef2d0ccef37f7f51bec -r 01a572fb8b4c463efa1b2091298fad3ec7ea2958 test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py @@ -412,6 +412,13 @@ database_connection = os.environ[ 'GALAXY_INSTALL_TEST_DBURI' ] else: database_connection = 'sqlite:///' + os.path.join( galaxy_db_path, 'install_and_test_repositories.sqlite' ) + if 'GALAXY_INSTALL_TEST_INSTALL_DBURI' in os.environ: + install_database_connection = os.environ[ 'GALAXY_INSTALL_TEST_INSTALL_DBURI' ] + elif asbool( os.environ.get( 'GALAXY_TEST_INSTALL_DB_MERGED', default_install_db_merged ) ): + install_database_connection = galaxy_database_connection + else: + install_db_path = os.path.join( galaxy_db_path, 'install.sqlite' ) + install_database_connection = 'sqlite:///%s' % install_galaxy_db_path kwargs = {} for dir in [ galaxy_test_tmp_dir ]: try: @@ -419,6 +426,7 @@ except OSError: pass print "Database connection: ", database_connection + print "Install database connection: ", install_database_connection # Generate the shed_tool_data_table_conf.xml file. file( shed_tool_data_table_conf_file, 'w' ).write( install_and_test_base_util.tool_data_table_conf_xml_template ) os.environ[ 'GALAXY_INSTALL_TEST_SHED_TOOL_DATA_TABLE_CONF' ] = shed_tool_data_table_conf_file @@ -446,6 +454,7 @@ datatype_converters_config_file = "datatype_converters_conf.xml.sample", file_path = galaxy_file_path, id_secret = install_and_test_base_util.galaxy_encode_secret, + install_database_connection = install_database_connection, job_config_file = galaxy_job_conf_file, job_queue_workers = 5, log_destination = "stdout", diff -r 952558f0dc0ba52355e87ef2d0ccef37f7f51bec -r 01a572fb8b4c463efa1b2091298fad3ec7ea2958 test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py @@ -264,6 +264,13 @@ database_connection = os.environ[ 'GALAXY_INSTALL_TEST_DBURI' ] else: database_connection = 'sqlite:///' + os.path.join( galaxy_db_path, 'install_and_test_repositories.sqlite' ) + if 'GALAXY_INSTALL_TEST_INSTALL_DBURI' in os.environ: + install_database_connection = os.environ[ 'GALAXY_INSTALL_TEST_INSTALL_DBURI' ] + elif asbool( os.environ.get( 'GALAXY_TEST_INSTALL_DB_MERGED', default_install_db_merged ) ): + install_database_connection = galaxy_database_connection + else: + install_db_path = os.path.join( galaxy_db_path, 'install.sqlite' ) + install_database_connection = 'sqlite:///%s' % install_galaxy_db_path kwargs = {} for dir in [ galaxy_test_tmp_dir ]: try: @@ -271,6 +278,7 @@ except OSError: pass print "Database connection: ", database_connection + print "Install database connection: ", install_database_connection # Generate the shed_tool_data_table_conf.xml file. file( shed_tool_data_table_conf_file, 'w' ).write( install_and_test_base_util.tool_data_table_conf_xml_template ) os.environ[ 'GALAXY_INSTALL_TEST_SHED_TOOL_DATA_TABLE_CONF' ] = shed_tool_data_table_conf_file @@ -298,6 +306,7 @@ datatype_converters_config_file = "datatype_converters_conf.xml.sample", file_path = galaxy_file_path, id_secret = install_and_test_base_util.galaxy_encode_secret, + install_database_connection = install_database_connection, job_config_file = galaxy_job_conf_file, job_queue_workers = 5, log_destination = "stdout", https://bitbucket.org/galaxy/galaxy-central/commits/6674cb6ced26/ Changeset: 6674cb6ced26 Branch: trevorw/specify-thirdparty-cookies-must-not-be-b-1392909850239 User: inithello Date: 2014-02-20 17:19:23 Summary: Close branch trevorw/specify-thirdparty-cookies-must-not-be-b-1392909850239. Affected #: 2 files diff -r c4145b92bc2a9e63778950aef4d53723bcd353ee -r 6674cb6ced267ba83d543f98fe1fe0b7e7de7324 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -4,9 +4,14 @@ import logging import os import shutil +import sys import tempfile -import shutil import td_common_util +import time +import shlex + +from subprocess import PIPE, Popen +from threading import Thread from contextlib import contextmanager from galaxy.util import unicodify from galaxy.util.template import fill_template @@ -16,12 +21,19 @@ eggs.require( 'paramiko' ) eggs.require( 'Fabric' ) +from fabric import state from fabric.api import env from fabric.api import hide from fabric.api import lcd from fabric.api import local from fabric.api import settings from fabric.api import prefix +from fabric.operations import _AttributeString + +try: + from Queue import Queue, Empty +except ImportError: + from queue import Queue, Empty log = logging.getLogger( __name__ ) @@ -120,6 +132,22 @@ if int( version.split( "." )[ 0 ] ) < 1: raise NotImplementedError( "Install Fabric version 1.0 or later." ) +def enqueue_output( stdout, stdout_queue, stderr, stderr_queue ): + stdout_logger = logging.getLogger( 'fabric_util.STDOUT' ) + stderr_logger = logging.getLogger( 'fabric_util.STDERR' ) + for line in iter( stdout.readline, b'' ): + output = line.rstrip() + stdout_logger.debug( output ) + stdout_queue.put( output ) + stdout.close() + stdout_queue.put(None) + for line in iter( stderr.readline, b'' ): + output = line.rstrip() + stderr_logger.debug( output ) + stderr_queue.put( output ) + stderr.close() + stderr_queue.put(None) + def file_append( text, file_path, skip_if_contained=True, make_executable=True ): ''' Append a line to a file unless skip_if_contained is True and the line already exists in the file. This method creates the file @@ -168,10 +196,9 @@ for shell_file_path in shell_file_paths: env_file_builder.append_line( action="source", value=shell_file_path ) -def handle_command( app, tool_dependency, install_dir, cmd, return_output=False ): +def handle_command( app, tool_dependency, install_dir, cmd, return_output=False, use_fabric=False ): context = app.install_model.context - with settings( warn_only=True ): - output = local( cmd, capture=True ) + output = run_local_command( cmd, capture_output=True, stream_output=True ) log_results( cmd, output, os.path.join( install_dir, INSTALLATION_LOG ) ) if output.return_code: tool_dependency.status = app.install_model.ToolDependency.installation_status.ERROR @@ -263,7 +290,7 @@ set_prior_environment_commands.append( 'echo "%s: $%s"' % ( inherited_env_var_name, inherited_env_var_name ) ) command = ' ; '.join( set_prior_environment_commands ) # Run the command and capture the output. - command_return = handle_command( app, tool_dependency, install_dir, command, return_output=True ) + command_return = handle_command( app, tool_dependency, install_dir, command, return_output=True, use_fabric=True ) # And extract anything labeled with the name of the environment variable we're populating here. if '%s: ' % inherited_env_var_name in command_return: environment_variable_value = command_return.split( '\n' ) @@ -370,7 +397,7 @@ filtered_actions = actions[ 1: ] return_code = handle_command( app, tool_dependency, install_dir, action_dict[ 'command' ] ) if return_code: - return tool_dependency + return tool_dependency dir = package_name elif action_type == 'download_file': # <action type="download_file">http://effectors.org/download/version/TTSS_GUI-1.0.1.jar</action> @@ -645,7 +672,7 @@ elif action_type == 'shell_command': with settings( warn_only=True ): cmd = install_environment.build_command( action_dict[ 'command' ] ) - return_code = handle_command( app, tool_dependency, install_dir, cmd ) + return_code = handle_command( app, tool_dependency, install_dir, cmd, use_fabric=False ) if return_code: return tool_dependency elif action_type == 'template_command': @@ -755,6 +782,31 @@ if os.path.exists( work_dir ): local( 'rm -rf %s' % work_dir ) +def run_local_command( command, capture_output=True, stream_output=True ): + wrapped_command = shlex.split( "/bin/sh -c '%s'" % command ) + log.debug( 'Executing command %s in path %s.' % ( command, state.env['lcwd'] ) ) + stdout_queue = Queue() + stderr_queue = Queue() + process_handle = Popen( wrapped_command, stdout=PIPE, stderr=PIPE, bufsize=1, close_fds=False, cwd=state.env['lcwd'] ) + stdio_thread = Thread( target=enqueue_output, args=( process_handle.stdout, stdout_queue, process_handle.stderr, stderr_queue ) ) + stdio_thread.daemon = True + stdio_thread.start() + stdout, stderr = wait_for_process( process_handle, stream_output, stdout_queue, stderr_queue ) + # Handle error condition (deal with stdout being None, too) + output = _AttributeString( stdout.strip() if stdout else "" ) + errors = _AttributeString( stderr.strip() if stderr else "" ) + output.failed = False + output.return_code = process_handle.returncode + output.stderr = errors + if process_handle.returncode not in env.ok_ret_codes: + output.failed = True + message = "local() encountered an error (return code %s) while executing '%s'" % ( process_handle.returncode, command ) + log.error( message ) + output.succeeded = not output.failed + # If we were capturing, this will be a string; otherwise it will be None. + if capture_output: + return output + def set_galaxy_environment( galaxy_user, tool_dependency_dir, host='localhost', shell='/bin/bash -l -c' ): """General Galaxy environment configuration. This method is not currently used.""" env.user = galaxy_user @@ -764,3 +816,21 @@ env.use_sudo = False env.safe_cmd = local return env + +def wait_for_process( process_handle, stream_output, stdout_queue, stderr_queue ): + standard_out = [] + standard_err = [] + process_handle.wait() + while True: + line = stdout_queue.get() + if line is None: + break + standard_out.append(line) + while True: + line = stderr_queue.get() + if line is None: + break + standard_err.append(line) + stdout = '\n'.join( standard_out ) + stderr = '\n'.join( standard_err ) + return stdout, stderr diff -r c4145b92bc2a9e63778950aef4d53723bcd353ee -r 6674cb6ced267ba83d543f98fe1fe0b7e7de7324 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -1719,7 +1719,7 @@ """ if text: if app.name == 'galaxy': - route_to_images = 'admin_toolshed/static/images/%s' % encoded_repository_id + route_to_images = '/admin_toolshed/static/images/%s' % encoded_repository_id else: # We're in the tool shed. route_to_images = '/repository/static/images/%s' % encoded_repository_id Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.