1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/43b42520bd5b/ Changeset: 43b42520bd5b User: Dave Bouvier Date: 2013-11-11 21:06:58 Summary: Handle cases where the installation path for a new tool dependency already exists, but is empty. This may occur when an error has been encountered, but the database has been dropped and recreated, or the Galaxy process was restarted after the installation path was created, but before any files were installed to that path. Affected #: 4 files diff -r 15767ea0e9f064345d35769e2d5b34608fa8877a -r 43b42520bd5b89f5b0f281efcbc5513e501326cd lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -1,8 +1,9 @@ import logging import os -import sys +import shutil import stat import subprocess +import sys import tempfile from string import Template import fabric_util @@ -77,23 +78,6 @@ return tool_shed_repository return None -def get_tool_dependency_install_dir( app, repository_name, repository_owner, repository_changeset_revision, tool_dependency_type, tool_dependency_name, - tool_dependency_version ): - if tool_dependency_type == 'package': - return os.path.abspath( os.path.join( app.config.tool_dependency_dir, - tool_dependency_name, - tool_dependency_version, - repository_owner, - repository_name, - repository_changeset_revision ) ) - if tool_dependency_type == 'set_environment': - return os.path.abspath( os.path.join( app.config.tool_dependency_dir, - 'environment_settings', - tool_dependency_name, - repository_owner, - repository_name, - repository_changeset_revision ) ) - def get_tool_shed_repository_install_dir( app, tool_shed_repository ): return os.path.abspath( tool_shed_repository.repo_files_directory( app ) ) @@ -123,22 +107,22 @@ required_repository_changeset_revision = required_repository.installed_changeset_revision # Define the installation directory for the required tool dependency package in the required repository. required_repository_package_install_dir = \ - get_tool_dependency_install_dir( app=app, - repository_name=required_repository_name, - repository_owner=required_repository_owner, - repository_changeset_revision=required_repository_changeset_revision, - tool_dependency_type='package', - tool_dependency_name=package_name, - tool_dependency_version=package_version ) + tool_dependency_util.get_tool_dependency_install_dir( app=app, + repository_name=required_repository_name, + repository_owner=required_repository_owner, + repository_changeset_revision=required_repository_changeset_revision, + tool_dependency_type='package', + tool_dependency_name=package_name, + tool_dependency_version=package_version ) # Define the this dependent repository's tool dependency installation directory that will contain the env.sh file with a path to the # required repository's installed tool dependency package. - dependent_install_dir = get_tool_dependency_install_dir( app=app, - repository_name=tool_shed_repository.name, - repository_owner=tool_shed_repository.owner, - repository_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dependency_type='package', - tool_dependency_name=package_name, - tool_dependency_version=package_version ) + dependent_install_dir = tool_dependency_util.get_tool_dependency_install_dir( app=app, + repository_name=tool_shed_repository.name, + repository_owner=tool_shed_repository.owner, + repository_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dependency_type='package', + tool_dependency_name=package_name, + tool_dependency_version=package_version ) # Set this dependent repository's tool dependency env.sh file with a path to the required repository's installed tool dependency package. # We can get everything we need from the discovered installed required_repository. if required_repository.status in [ app.model.ToolShedRepository.installation_status.DEACTIVATED, @@ -253,13 +237,13 @@ # We'll read the contents of the received required_repository's env.sh file and replace the 'value' entry of each env_var_dict # with the associated value in the env.sh file. new_env_var_dicts = [] - env_sh_file_dir = get_tool_dependency_install_dir( app=app, - repository_name=required_repository.name, - repository_owner=required_repository.owner, - repository_changeset_revision=required_repository.installed_changeset_revision, - tool_dependency_type='package', - tool_dependency_name=package_name, - tool_dependency_version=package_version ) + env_sh_file_dir = tool_dependency_util.get_tool_dependency_install_dir( app=app, + repository_name=required_repository.name, + repository_owner=required_repository.owner, + repository_changeset_revision=required_repository.installed_changeset_revision, + tool_dependency_type='package', + tool_dependency_name=package_name, + tool_dependency_version=package_version ) env_sh_file_path = os.path.join( env_sh_file_dir, 'env.sh' ) if os.path.exists( env_sh_file_path ): for i, line in enumerate( open( env_sh_file_path, 'r' ) ): @@ -272,12 +256,11 @@ env_var_dict[ 'value' ] = new_value new_env_var_dicts.append( env_var_dict ) else: - tool_dependency.status = app.model.ToolShedRepository.installation_status.ERROR - error_message = 'Invalid file %s specified, ignoring set_environment_for_install action.', env_sh_file_path - tool_dependency.error_message = error_message - sa_session = app.model.context.current - sa_session.add( tool_dependency ) - sa_session.flush() + error_message = 'Invalid file %s specified, ignoring set_environment_for_install action.' % str( env_sh_file_path ) + tool_dependency = tool_dependency_util.handle_tool_dependency_installation_error( app, + tool_dependency, + error_message, + remove_installation_path=False ) action_dict[ 'environment_variable' ] = new_env_var_dicts else: action_dict[ 'environment_variable' ] = env_var_dicts @@ -294,10 +277,13 @@ fabric_util.install_and_build_package( app, tool_dependency, actions_dict ) except Exception, e: log.exception( 'Error installing tool dependency %s version %s.', str( tool_dependency.name ), str( tool_dependency.version ) ) - tool_dependency.status = app.model.ToolDependency.installation_status.ERROR - tool_dependency.error_message = '%s\n%s' % ( td_common_util.format_traceback(), str( e ) ) - sa_session.add( tool_dependency ) - sa_session.flush() + # Since there was an installation error, update the tool dependency status to Error. The remove_installation_path option must + # be left False here. + error_message = '%s\n%s' % ( td_common_util.format_traceback(), str( e ) ) + tool_dependency = tool_dependency_util.handle_tool_dependency_installation_error( app, + tool_dependency, + error_message, + remove_installation_path=False ) if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED sa_session.add( tool_dependency ) @@ -320,24 +306,54 @@ elif package_elem.tag == 'install': # <install version="1.0"> # Get the installation directory for tool dependencies that will be installed for the received tool_shed_repository. - install_dir = get_tool_dependency_install_dir( app=app, - repository_name=tool_shed_repository.name, - repository_owner=tool_shed_repository.owner, - repository_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dependency_type='package', - tool_dependency_name=package_name, - tool_dependency_version=package_version ) + install_dir = tool_dependency_util.get_tool_dependency_install_dir( app=app, + repository_name=tool_shed_repository.name, + repository_owner=tool_shed_repository.owner, + repository_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dependency_type='package', + tool_dependency_name=package_name, + tool_dependency_version=package_version ) + can_install_tool_dependency = True if os.path.exists( install_dir ): - print '\nSkipping installation of tool dependency', package_name, 'version', package_version, 'since it is installed in', install_dir, '\n' + # The tool dependency installation directory exists, check for the presence of fabric_util.INSTALLATION_LOG, + # which indicates a successful installation. This currently assumes that the tool dependency record will + # always exist in the database. tool_dependency = tool_dependency_util.get_tool_dependency_by_name_version_type_repository( app, tool_shed_repository, package_name, package_version, 'package' ) - tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED - sa_session.add( tool_dependency ) - sa_session.flush() + if tool_dependency.status == app.model.ToolDependency.installation_status.INSTALLING: + # The tool dependency is in an Installing state, so we don't want to do anything to it. + log.debug( 'Skipping installation of tool dependency %s version %s because it has a status of %s' % \ + ( tool_dependency.name, tool_dependency.version, tool_dependency.status ) ) + can_install_tool_dependency = False + else: + # If the tool dependency is being installed by someone else, we don't want to interfere with that. + # This assumes the installation by "someone else" is not hung in an Installing state, which is a + # weakness if "someone else" never repaired it. + tool_dependency_installation_directory_contents = os.listdir( install_dir ) + if fabric_util.INSTALLATION_LOG in tool_dependency_installation_directory_contents: + print '\nSkipping installation of tool dependency', package_name, 'version', package_version, \ + 'since it is installed in', install_dir, '\n' + can_install_tool_dependency = False + else: + error_message = '\nInstallation path %s for tool dependency %s version %s exists, but the expected file %s' % \ + ( install_dir, package_name, package_version, fabric_util.INSTALLATION_LOG ) + error_message += ' is missing. This indicates an installation error, so the tool dependency is being' + error_message += ' prepared for reinstallation.' + print error_message + tool_dependency.status = app.model.ToolDependency.installation_status.NEVER_INSTALLED + try: + shutil.rmtree( install_dir ) + except Exception, e: + log.exception( 'Unable to remove installation path %s.', install_dir ) + can_install_tool_dependency = True + sa_session.add( tool_dependency ) + sa_session.flush() else: + can_install_tool_dependency = True + if can_install_tool_dependency: package_install_version = package_elem.get( 'version', '1.0' ) tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app, tool_shed_repository=tool_shed_repository, @@ -436,10 +452,12 @@ error_message = 'Version %s of the %s package cannot be installed because ' % ( str( package_version ), str( package_name ) ) error_message += 'the recipe for installing the package is missing either an <actions> tag set or an <actions_group> ' error_message += 'tag set.' - tool_dependency.status = app.model.ToolDependency.installation_status.ERROR - tool_dependency.error_message = error_message - sa_session.add( tool_dependency ) - sa_session.flush() + # Since there was an installation error, update the tool dependency status to Error. The remove_installation_path option must + # be left False here. + tool_dependency = tool_dependency_util.handle_tool_dependency_installation_error( app, + tool_dependency, + error_message, + remove_installation_path=False ) return tool_dependency else: raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).' ) @@ -878,13 +896,13 @@ # Tool dependencies of type "set_environmnet" always have the version attribute set to None. attr_tup = ( env_var_name, None, 'set_environment' ) if attr_tup in attr_tups_of_dependencies_for_install: - install_dir = get_tool_dependency_install_dir( app=app, - repository_name=tool_shed_repository.name, - repository_owner=tool_shed_repository.owner, - repository_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dependency_type='set_environment', - tool_dependency_name=env_var_name, - tool_dependency_version=None ) + install_dir = tool_dependency_util.get_tool_dependency_install_dir( app=app, + repository_name=tool_shed_repository.name, + repository_owner=tool_shed_repository.owner, + repository_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dependency_type='set_environment', + tool_dependency_name=env_var_name, + tool_dependency_version=None ) tool_shed_repository_install_dir = get_tool_shed_repository_install_dir( app, tool_shed_repository ) env_var_dict = td_common_util.create_env_var_dict( env_var_elem, tool_shed_repository_install_dir=tool_shed_repository_install_dir ) if env_var_dict: diff -r 15767ea0e9f064345d35769e2d5b34608fa8877a -r 43b42520bd5b89f5b0f281efcbc5513e501326cd lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -475,10 +475,12 @@ error_message = "Error installing tool dependency %s version %s: %s" % ( str( package_name ), str( package_version ), str( e ) ) log.exception( error_message ) if tool_dependency: - tool_dependency.status = app.model.ToolDependency.installation_status.ERROR - tool_dependency.error_message = error_message - sa_session.add( tool_dependency ) - sa_session.flush() + # Since there was an installation error, update the tool dependency status to Error. The remove_installation_path option must + # be left False here. + tool_dependency = tool_dependency_util.handle_tool_dependency_installation_error( app, + tool_dependency, + error_message, + remove_installation_path=False ) if tool_dependency and tool_dependency.status in [ app.model.ToolDependency.installation_status.INSTALLED, app.model.ToolDependency.installation_status.ERROR ]: installed_tool_dependencies.append( tool_dependency ) @@ -492,10 +494,12 @@ error_message = "Error setting environment for tool dependency: %s" % str( e ) log.debug( error_message ) if tool_dependency: - tool_dependency.status = app.model.ToolDependency.installation_status.ERROR - tool_dependency.error_message = error_message - sa_session.add( tool_dependency ) - sa_session.flush() + # Since there was an installation error, update the tool dependency status to Error. The remove_installation_path option must + # be left False here. + tool_dependency = tool_dependency_util.handle_tool_dependency_installation_error( app, + tool_dependency, + error_message, + remove_installation_path=False ) if tool_dependency and tool_dependency.status in [ app.model.ToolDependency.installation_status.INSTALLED, app.model.ToolDependency.installation_status.ERROR ]: installed_tool_dependencies.append( tool_dependency ) diff -r 15767ea0e9f064345d35769e2d5b34608fa8877a -r 43b42520bd5b89f5b0f281efcbc5513e501326cd lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -235,7 +235,6 @@ """ # The received value for dist_to_shed will be True if the InstallManager is installing a repository that contains tools or datatypes that used # to be in the Galaxy distribution, but have been moved to the main Galaxy tool shed. - log.debug( "Adding new row (or updating an existing row) for repository '%s' in the tool_shed_repository table, status set to '%s'." % ( str( name ), str( status ) ) ) if current_changeset_revision is None: # The current_changeset_revision is not passed if a repository is being installed for the first time. If a previously installed repository # was later uninstalled, this value should be received as the value of that change set to which the repository had been updated just prior @@ -261,6 +260,7 @@ owner, installed_changeset_revision ) if tool_shed_repository: + log.debug( "Updating an existing row for repository '%s' in the tool_shed_repository table, status set to '%s'." % ( str( name ), str( status ) ) ) tool_shed_repository.description = description tool_shed_repository.changeset_revision = current_changeset_revision tool_shed_repository.ctx_rev = ctx_rev @@ -270,6 +270,7 @@ tool_shed_repository.uninstalled = uninstalled tool_shed_repository.status = status else: + log.debug( "Adding new row for repository '%s' in the tool_shed_repository table, status set to '%s'." % ( str( name ), str( status ) ) ) tool_shed_repository = app.model.ToolShedRepository( tool_shed=tool_shed, name=name, description=description, diff -r 15767ea0e9f064345d35769e2d5b34608fa8877a -r 43b42520bd5b89f5b0f281efcbc5513e501326cd lib/tool_shed/util/tool_dependency_util.py --- a/lib/tool_shed/util/tool_dependency_util.py +++ b/lib/tool_shed/util/tool_dependency_util.py @@ -48,11 +48,15 @@ else: tool_dependency = get_tool_dependency_by_name_type_repository( app, tool_shed_repository, name, type ) if tool_dependency: + log.debug( 'Updating an existing record for tool dependency %s version %s for tool shed repository %s with changeset revision %s.' % + ( str( name ), str( version ), str( tool_shed_repository.name ), str( tool_shed_repository.changeset_revision ) ) ) # In some cases we should not override the current status of an existing tool_dependency, so do so only if set_status is True. if set_status: tool_dependency.status = status else: # Create a new tool_dependency record for the tool_shed_repository. + log.debug( 'Creating a new record for tool dependency %s version %s for tool shed repository %s with changeset revision %s.' % + ( str( name ), str( version ), str( tool_shed_repository.name ), str( tool_shed_repository.changeset_revision ) ) ) tool_dependency = app.model.ToolDependency( tool_shed_repository.id, name, version, type, status ) sa_session.add( tool_dependency ) sa_session.flush() @@ -318,6 +322,51 @@ return ','.join( tool_dependency_ids ) return tool_dependency_ids +def get_tool_dependency_install_dir( app, repository_name, repository_owner, repository_changeset_revision, tool_dependency_type, + tool_dependency_name, tool_dependency_version ): + if tool_dependency_type == 'package': + return os.path.abspath( os.path.join( app.config.tool_dependency_dir, + tool_dependency_name, + tool_dependency_version, + repository_owner, + repository_name, + repository_changeset_revision ) ) + if tool_dependency_type == 'set_environment': + return os.path.abspath( os.path.join( app.config.tool_dependency_dir, + 'environment_settings', + tool_dependency_name, + repository_owner, + repository_name, + repository_changeset_revision ) ) + +def handle_tool_dependency_installation_error( app, tool_dependency, error_message, remove_installation_path=False ): + # Since there was an installation error, remove the installation directory because the install_package method uses + # this: "if os.path.exists( install_dir ):". Setting remove_installation_path to True should rarely occur. It is + # currently set to True only to handle issues with installing tool dependencies into an Amazon S3 bucket. + sa_session = app.model.context.current + tool_shed_repository = tool_dependency.tool_shed_repository + install_dir = get_tool_dependency_install_dir( app=app, + repository_name=tool_shed_repository.name, + repository_owner=tool_shed_repository.owner, + repository_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dependency_type=tool_dependency.type, + tool_dependency_name=tool_dependency.name, + tool_dependency_version=tool_dependency.version ) + if remove_installation_path: + # This will be True only in the case where an exception was encountered during the installation process after the installation + # path was created, but before any information was written to the installation log, and the tool dependency status was not + # set to "Installed" or "Error". + if os.path.exists( install_dir ): + try: + shutil.rmtree( install_dir ) + except Exception, e: + log.exception( 'Error removing existing installation directory %s.', install_dir ) + tool_dependency.status = app.model.ToolDependency.installation_status.ERROR + tool_dependency.error_message = error_message + sa_session.add( tool_dependency ) + sa_session.flush() + return tool_dependency + def merge_missing_tool_dependencies_to_installed_container( containers_dict ): """ Merge the list of missing tool dependencies into the list of installed tool dependencies.""" missing_td_container_root = containers_dict.get( 'missing_tool_dependencies', None ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.