- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: greg: Make sure dir is not None when used to create a path - than Nicolas Lapalu.
by commits-noreply＠bitbucket.org 17 Jul '14

17 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/23f0c41b1cd7/ Changeset: 23f0c41b1cd7 User: greg Date: 2014-07-17 19:38:53 Summary: Make sure dir is not None when used to create a path - than Nicolas Lapalu. Affected #: 1 file diff -r 4d2e744f8efc73dc265c3d3bf33c4c2886888b7a -r 23f0c41b1cd7d7ef2087e1eec387cd3b116923fc lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -100,6 +100,8 @@ # We're in stage 2 of the installation process. The package has been down-loaded, so we can # now perform all of the actions defined for building it. for action_tup in filtered_actions: + if dir is None: + dir = '' current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) with lcd( current_dir ): action_type, action_dict = action_tup Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add classes for managing tool shed repository metadata.
by commits-noreply＠bitbucket.org 17 Jul '14

17 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/4d2e744f8efc/ Changeset: 4d2e744f8efc User: greg Date: 2014-07-17 19:20:49 Summary: Add classes for managing tool shed repository metadata. Affected #: 20 files diff -r 5af80141674f5ac3166b6782385c408258606d0a -r 4d2e744f8efc73dc265c3d3bf33c4c2886888b7a lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -11,6 +11,7 @@ from galaxy.web.base.controller import BaseAPIController from tool_shed.galaxy_install.install_manager import InstallRepositoryManager +from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager from tool_shed.util import common_util from tool_shed.util import encoding_util @@ -417,7 +418,8 @@ for repository in query: repository_id = trans.security.encode_id( repository.id ) try: - invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_installed_repository( trans.app, repository_id ) + irmm = InstalledRepositoryMetadataManager( trans.app ) + invalid_file_tups, metadata_dict = irmm.reset_all_metadata_on_installed_repository( repository_id ) if invalid_file_tups: message = tool_util.generate_message_for_invalid_tools( trans.app, invalid_file_tups, diff -r 5af80141674f5ac3166b6782385c408258606d0a -r 4d2e744f8efc73dc265c3d3bf33c4c2886888b7a lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -16,7 +16,6 @@ from tool_shed.util import datatype_util from tool_shed.util import encoding_util from tool_shed.util import hg_util -from tool_shed.util import metadata_util from tool_shed.util import readme_util from tool_shed.util import repository_maintenance_util from tool_shed.util import shed_util_common as suc @@ -27,8 +26,9 @@ from tool_shed.galaxy_install import dependency_display from tool_shed.galaxy_install import install_manager +from tool_shed.galaxy_install.grids import admin_toolshed_grids +from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager -import tool_shed.galaxy_install.grids.admin_toolshed_grids as admin_toolshed_grids from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager log = logging.getLogger( __name__ ) @@ -1704,7 +1704,8 @@ @web.require_admin def reset_metadata_on_selected_installed_repositories( self, trans, **kwd ): if 'reset_metadata_on_selected_repositories_button' in kwd: - message, status = metadata_util.reset_metadata_on_selected_repositories( trans.app, trans.user, **kwd ) + irmm = InstalledRepositoryMetadataManager( trans.app ) + message, status = irmm.reset_metadata_on_selected_repositories( trans.user, **kwd ) else: message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) @@ -1723,17 +1724,17 @@ tool_path, relative_install_dir = repository.get_tool_relative_path( trans.app ) if relative_install_dir: original_metadata_dict = repository.metadata + irmm = InstalledRepositoryMetadataManager( trans.app ) metadata_dict, invalid_file_tups = \ - metadata_util.generate_metadata_for_changeset_revision( app=trans.app, - repository=repository, - changeset_revision=repository.changeset_revision, - repository_clone_url=repository_clone_url, - shed_config_dict = repository.get_shed_config_dict( trans.app ), - relative_install_dir=relative_install_dir, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=False, - persist=False ) + irmm.generate_metadata_for_changeset_revision( repository=repository, + changeset_revision=repository.changeset_revision, + repository_clone_url=repository_clone_url, + shed_config_dict = repository.get_shed_config_dict( trans.app ), + relative_install_dir=relative_install_dir, + repository_files_dir=None, + resetting_all_metadata_on_repository=False, + updating_installed_repository=False, + persist=False ) repository.metadata = metadata_dict if metadata_dict != original_metadata_dict: suc.update_in_shed_tool_config( trans.app, repository ) @@ -1926,17 +1927,17 @@ if repository.includes_data_managers: data_manager_util.remove_from_data_manager( trans.app, repository ) # Update the repository metadata. + irmm = InstalledRepositoryMetadataManager( trans.app ) metadata_dict, invalid_file_tups = \ - metadata_util.generate_metadata_for_changeset_revision( app=trans.app, - repository=repository, - changeset_revision=latest_changeset_revision, - repository_clone_url=repository_clone_url, - shed_config_dict=repository.get_shed_config_dict( trans.app ), - relative_install_dir=relative_install_dir, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=True, - persist=True ) + irmm.generate_metadata_for_changeset_revision( repository=repository, + changeset_revision=latest_changeset_revision, + repository_clone_url=repository_clone_url, + shed_config_dict=repository.get_shed_config_dict( trans.app ), + relative_install_dir=relative_install_dir, + repository_files_dir=None, + resetting_all_metadata_on_repository=False, + updating_installed_repository=True, + persist=True ) if 'tools' in metadata_dict: tool_panel_dict = metadata_dict.get( 'tool_panel_section', None ) if tool_panel_dict is None: diff -r 5af80141674f5ac3166b6782385c408258606d0a -r 4d2e744f8efc73dc265c3d3bf33c4c2886888b7a lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -4,19 +4,21 @@ import tarfile import tempfile from time import strftime -from galaxy import eggs + from galaxy import util from galaxy import web from galaxy.model.orm import and_ from galaxy.web.base.controller import BaseAPIController from galaxy.web.base.controller import HTTPBadRequest from galaxy.web.framework.helpers import time_ago + from tool_shed.capsule import capsule_manager -import tool_shed.repository_types.util as rt_util +from tool_shed.metadata import repository_metadata_manager +from tool_shed.repository_types import util as rt_util + from tool_shed.util import basic_util from tool_shed.util import encoding_util from tool_shed.util import hg_util -from tool_shed.util import metadata_util from tool_shed.util import repository_maintenance_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_util @@ -428,8 +430,9 @@ log.debug( "Resetting metadata on repository %s" % str( repository.name ) ) repository_id = trans.security.encode_id( repository.id ) try: + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) invalid_file_tups, metadata_dict = \ - metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans.app, trans.user, repository_id ) + rmm.reset_all_metadata_on_repository_in_tool_shed( trans.user, repository_id ) if invalid_file_tups: message = tool_util.generate_message_for_invalid_tools( trans.app, invalid_file_tups, @@ -508,10 +511,10 @@ results = dict( start_time=start_time, repository_status=[] ) try: + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) invalid_file_tups, metadata_dict = \ - metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans.app, - trans.user, - trans.security.encode_id( repository.id ) ) + rmm.reset_all_metadata_on_repository_in_tool_shed( trans.user, + trans.security.encode_id( repository.id ) ) if invalid_file_tups: message = tool_util.generate_message_for_invalid_tools( trans.app, invalid_file_tups, diff -r 5af80141674f5ac3166b6782385c408258606d0a -r 4d2e744f8efc73dc265c3d3bf33c4c2886888b7a lib/galaxy/webapps/tool_shed/controllers/admin.py --- a/lib/galaxy/webapps/tool_shed/controllers/admin.py +++ b/lib/galaxy/webapps/tool_shed/controllers/admin.py @@ -1,13 +1,18 @@ import logging + +from galaxy import util +from galaxy.util import inflector +from galaxy import web + from galaxy.web.base.controller import BaseUIController from galaxy.web.base.controllers.admin import Admin -from galaxy import util -from galaxy import web -from galaxy.util import inflector -import tool_shed.util.shed_util_common as suc -import tool_shed.util.metadata_util as metadata_util + +import tool_shed.grids.admin_grids as admin_grids +from tool_shed.metadata import repository_metadata_manager + +from tool_shed.util import metadata_util from tool_shed.util import repository_maintenance_util -import tool_shed.grids.admin_grids as admin_grids +from tool_shed.util import shed_util_common as suc log = logging.getLogger( __name__ ) @@ -344,7 +349,8 @@ @web.require_admin def reset_metadata_on_selected_repositories_in_tool_shed( self, trans, **kwd ): if 'reset_metadata_on_selected_repositories_button' in kwd: - message, status = metadata_util.reset_metadata_on_selected_repositories( trans.app, trans.user, **kwd ) + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) + message, status = rmm.reset_metadata_on_selected_repositories( trans.user, **kwd ) else: message = util.restore_text( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) diff -r 5af80141674f5ac3166b6782385c408258606d0a -r 4d2e744f8efc73dc265c3d3bf33c4c2886888b7a lib/galaxy/webapps/tool_shed/controllers/hg.py --- a/lib/galaxy/webapps/tool_shed/controllers/hg.py +++ b/lib/galaxy/webapps/tool_shed/controllers/hg.py @@ -3,7 +3,7 @@ from galaxy.web.base.controller import BaseUIController from tool_shed.util.shed_util_common import get_repository_by_name_and_owner from tool_shed.util.hg_util import update_repository -from tool_shed.util.metadata_util import set_repository_metadata +from tool_shed.metadata import repository_metadata_manager from galaxy import eggs eggs.require('mercurial') @@ -28,9 +28,11 @@ return hgwebapp wsgi_app = wsgiapplication( make_web_app ) if hg_version >= '2.2.3' and cmd == 'pushkey': - # When doing an "hg push" from the command line, the following commands, in order, will be retrieved from environ, depending - # upon the mercurial version being used. In mercurial version 2.2.3, section 15.2. Command changes includes a new feature: - # pushkey: add hooks for pushkey/listkeys (see http://mercurial.selenic.com/wiki/WhatsNew#Mercurial_2.2.3_.282012-07-01.29) + # When doing an "hg push" from the command line, the following commands, in order, will be + # retrieved from environ, depending upon the mercurial version being used. In mercurial + # version 2.2.3, section 15.2. Command changes includes a new feature: + # pushkey: add hooks for pushkey/listkeys + # (see http://mercurial.selenic.com/wiki/WhatsNew#Mercurial_2.2.3_.282012-07-01.29) # We require version 2.2.3 since the pushkey hook was added in that version. # If mercurial version >= '2.2.3': capabilities -> batch -> branchmap -> unbundle -> listkeys -> pushkey path_info = kwd.get( 'path_info', None ) @@ -39,13 +41,15 @@ repository = get_repository_by_name_and_owner( trans.app, name, owner ) if repository: if hg_version >= '2.2.3': - # Update the repository on disk to the tip revision, because the web upload form uses the on-disk working - # directory. If the repository is not updated on disk, pushing from the command line and then uploading - # via the web interface will result in a new head being created. + # Update the repository on disk to the tip revision, because the web upload + # form uses the on-disk working directory. If the repository is not updated + # on disk, pushing from the command line and then uploading via the web + # interface will result in a new head being created. repo = hg.repository( ui.ui(), repository.repo_path( trans.app ) ) update_repository( repo, ctx_rev=None ) # Set metadata using the repository files on disk. - error_message, status = set_repository_metadata( trans.app, trans.request.host, trans.user, repository ) + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) + error_message, status = rmm.set_repository_metadata( trans.request.host, trans.user, repository ) if status == 'ok' and error_message: log.debug( "Successfully reset metadata on repository %s owned by %s, but encountered problem: %s" % \ ( str( repository.name ), str( repository.user.username ), error_message ) ) diff -r 5af80141674f5ac3166b6782385c408258606d0a -r 4d2e744f8efc73dc265c3d3bf33c4c2886888b7a lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -19,6 +19,7 @@ from tool_shed.dependencies.repository import relation_builder from tool_shed.galaxy_install import dependency_display +from tool_shed.metadata import repository_metadata_manager from tool_shed.util import basic_util from tool_shed.util import common_util @@ -2610,14 +2611,20 @@ repository_dependencies_dict = metadata[ 'repository_dependencies' ] rd_tups = repository_dependencies_dict.get( 'repository_dependencies', [] ) for rd_tup in rd_tups: - rdtool_shed, rd_name, rd_owner, rd_changeset_revision, rd_prior_installation_required, rd_only_if_compiling_contained_td = \ + rdtool_shed, \ + rd_name, \ + rd_owner, \ + rd_changeset_revision, \ + rd_prior_installation_required, \ + rd_only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple( rd_tup ) if not util.asbool( rd_only_if_compiling_contained_td ): invalid = True break if invalid: - message = metadata_util.generate_message_for_invalid_repository_dependencies( metadata, - error_from_tuple=False ) + dd = dependency_display.DependencyDisplayer( trans.app ) + message = dd.generate_message_for_invalid_repository_dependencies( metadata, + error_from_tuple=False ) status = 'error' else: repository_metadata_id = None @@ -2735,11 +2742,15 @@ def reset_all_metadata( self, trans, id, **kwd ): """Reset all metadata on the complete changelog for a single repository in the tool shed.""" # This method is called only from the ~/templates/webapps/tool_shed/repository/manage_repository.mako template. + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) invalid_file_tups, metadata_dict = \ - metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans.app, trans.user, id, **kwd ) + rmm.reset_all_metadata_on_repository_in_tool_shed( trans.user, id, **kwd ) if invalid_file_tups: repository = suc.get_repository_in_tool_shed( trans.app, id ) - message = tool_util.generate_message_for_invalid_tools( trans.app, invalid_file_tups, repository, metadata_dict ) + message = tool_util.generate_message_for_invalid_tools( trans.app, + invalid_file_tups, + repository, + metadata_dict ) status = 'error' else: message = "All repository metadata has been reset. " @@ -2753,7 +2764,8 @@ @web.expose def reset_metadata_on_my_writable_repositories_in_tool_shed( self, trans, **kwd ): if 'reset_metadata_on_selected_repositories_button' in kwd: - message, status = metadata_util.reset_metadata_on_selected_repositories( trans.app, trans.user, **kwd ) + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) + message, status = rmm.reset_metadata_on_selected_repositories( trans.user, **kwd ) else: message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) @@ -2817,11 +2829,11 @@ if tip == repository.tip( trans.app ): message += 'No changes to repository. ' else: - status, error_message = metadata_util.set_repository_metadata_due_to_new_tip( trans.app, - trans.request.host, - trans.user, - repository, - **kwd ) + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) + status, error_message = rmm.set_repository_metadata_due_to_new_tip( trans.request.host, + trans.user, + repository, + **kwd ) if error_message: message = error_message else: diff -r 5af80141674f5ac3166b6782385c408258606d0a -r 4d2e744f8efc73dc265c3d3bf33c4c2886888b7a lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -12,12 +12,12 @@ from tool_shed.dependencies import attribute_handlers from tool_shed.galaxy_install import dependency_display +from tool_shed.metadata import repository_metadata_manager import tool_shed.repository_types.util as rt_util from tool_shed.util import basic_util from tool_shed.util import commit_util from tool_shed.util import hg_util -from tool_shed.util import metadata_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_util from tool_shed.util import xml_util @@ -262,12 +262,13 @@ ( len( files_to_remove ), upload_point ) else: message += " %d files were removed from the repository root. " % len( files_to_remove ) - status, error_message = metadata_util.set_repository_metadata_due_to_new_tip( trans.app, - trans.request.host, - trans.user, - repository, - content_alert_str=content_alert_str, - **kwd ) + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) + status, error_message = \ + rmm.set_repository_metadata_due_to_new_tip( trans.request.host, + trans.user, + repository, + content_alert_str=content_alert_str, + **kwd ) if error_message: message = error_message kwd[ 'message' ] = message @@ -303,8 +304,8 @@ status = 'error' # Handle messaging for invalid repository dependencies. invalid_repository_dependencies_message = \ - metadata_util.generate_message_for_invalid_repository_dependencies( metadata_dict, - error_from_tuple=True ) + dd.generate_message_for_invalid_repository_dependencies( metadata_dict, + error_from_tuple=True ) if invalid_repository_dependencies_message: message += invalid_repository_dependencies_message status = 'error' diff -r 5af80141674f5ac3166b6782385c408258606d0a -r 4d2e744f8efc73dc265c3d3bf33c4c2886888b7a lib/tool_shed/capsule/capsule_manager.py --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -7,23 +7,26 @@ import urllib from time import gmtime from time import strftime -import tool_shed.repository_types.util as rt_util + from galaxy import web from galaxy.util import asbool from galaxy.util import CHUNK_SIZE from galaxy.util.odict import odict + from tool_shed.dependencies.repository.relation_builder import RelationBuilder from tool_shed.dependencies import attribute_handlers +from tool_shed.galaxy_install.repository_dependencies.repository_dependency_manager import RepositoryDependencyInstallManager +from tool_shed.metadata import repository_metadata_manager +import tool_shed.repository_types.util as rt_util + from tool_shed.util import basic_util from tool_shed.util import commit_util from tool_shed.util import common_util from tool_shed.util import encoding_util from tool_shed.util import hg_util -from tool_shed.util import metadata_util from tool_shed.util import repository_maintenance_util from tool_shed.util import shed_util_common as suc from tool_shed.util import xml_util -from tool_shed.galaxy_install.repository_dependencies.repository_dependency_manager import RepositoryDependencyInstallManager log = logging.getLogger( __name__ ) @@ -33,6 +36,7 @@ def __init__( self ): self.exported_repository_elems = [] + class ExportRepositoryManager( object ): def __init__( self, app, user, tool_shed_url, repository, changeset_revision, export_repository_dependencies, using_api ): @@ -723,11 +727,11 @@ results_dict[ 'ok' ] = False results_dict[ 'error_message' ] += error_message try: - status, error_message = metadata_util.set_repository_metadata_due_to_new_tip( self.app, - self.host, - self.user, - repository, - content_alert_str=content_alert_str ) + rmm = repository_metadata_manager.RepositoryMetadataManager( self.app ) + status, error_message = rmm.set_repository_metadata_due_to_new_tip( self.host, + self.user, + repository, + content_alert_str=content_alert_str ) if error_message: results_dict[ 'ok' ] = False results_dict[ 'error_message' ] += error_message diff -r 5af80141674f5ac3166b6782385c408258606d0a -r 4d2e744f8efc73dc265c3d3bf33c4c2886888b7a lib/tool_shed/galaxy_install/dependency_display.py --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -48,6 +48,57 @@ tool_dependencies[ dependency_key ] = requirements_dict return tool_dependencies + def generate_message_for_invalid_repository_dependencies( self, metadata_dict, error_from_tuple=False ): + """ + Get or generate and return an error message associated with an invalid repository dependency. + """ + message = '' + if metadata_dict: + if error_from_tuple: + # Return the error messages associated with a set of one or more invalid repository + # dependency tuples. + invalid_repository_dependencies_dict = metadata_dict.get( 'invalid_repository_dependencies', None ) + if invalid_repository_dependencies_dict is not None: + invalid_repository_dependencies = \ + invalid_repository_dependencies_dict.get( 'invalid_repository_dependencies', [] ) + for repository_dependency_tup in invalid_repository_dependencies: + toolshed, \ + name, \ + owner, \ + changeset_revision, \ + prior_installation_required, \ + only_if_compiling_contained_td, error = \ + common_util.parse_repository_dependency_tuple( repository_dependency_tup, contains_error=True ) + if error: + message += '%s ' % str( error ) + else: + # The complete dependency hierarchy could not be determined for a repository being installed into + # Galaxy. This is likely due to invalid repository dependency definitions, so we'll get them from + # the metadata and parse them for display in an error message. This will hopefully communicate the + # problem to the user in such a way that a resolution can be determined. + message += 'The complete dependency hierarchy could not be determined for this repository, so no required ' + message += 'repositories will not be installed. This is likely due to invalid repository dependency definitions. ' + repository_dependencies_dict = metadata_dict.get( 'repository_dependencies', None ) + if repository_dependencies_dict is not None: + rd_tups = repository_dependencies_dict.get( 'repository_dependencies', None ) + if rd_tups is not None: + message += 'Here are the attributes of the dependencies defined for this repository to help determine the ' + message += 'cause of this problem. ' + message += '<table cellpadding="2" cellspacing="2">' + message += '<tr><th>Tool shed</th><th>Repository name</th><th>Owner</th><th>Changeset revision</th>' + message += '<th>Prior install required</th></tr>' + for rd_tup in rd_tups: + tool_shed, name, owner, changeset_revision, pir, oicct = \ + common_util.parse_repository_dependency_tuple( rd_tup ) + if util.asbool( pir ): + pir_str = 'True' + else: + pir_str = '' + message += '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % \ + ( tool_shed, name, owner, changeset_revision, pir_str ) + message += '</table>' + return message + def generate_message_for_invalid_tool_dependencies( self, metadata_dict ): """ Tool dependency definitions can only be invalid if they include a definition for a complex diff -r 5af80141674f5ac3166b6782385c408258606d0a -r 4d2e744f8efc73dc265c3d3bf33c4c2886888b7a lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -22,17 +22,17 @@ from tool_shed.util import datatype_util from tool_shed.util import encoding_util from tool_shed.util import hg_util -from tool_shed.util import metadata_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util from tool_shed.util import xml_util +from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager +from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager from tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import StepManager from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import TagManager -from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager log = logging.getLogger( __name__ ) @@ -499,17 +499,17 @@ """ install_model = self.app.install_model shed_config_dict = self.app.toolbox.get_shed_config_dict_by_filename( shed_tool_conf ) + irmm = InstalledRepositoryMetadataManager( self.app ) metadata_dict, invalid_file_tups = \ - metadata_util.generate_metadata_for_changeset_revision( app=self.app, - repository=tool_shed_repository, - changeset_revision=tool_shed_repository.changeset_revision, - repository_clone_url=repository_clone_url, - shed_config_dict=shed_config_dict, - relative_install_dir=relative_install_dir, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=False, - persist=True ) + irmm.generate_metadata_for_changeset_revision( repository=tool_shed_repository, + changeset_revision=tool_shed_repository.changeset_revision, + repository_clone_url=repository_clone_url, + shed_config_dict=shed_config_dict, + relative_install_dir=relative_install_dir, + repository_files_dir=None, + resetting_all_metadata_on_repository=False, + updating_installed_repository=False, + persist=True ) tool_shed_repository.metadata = metadata_dict # Update the tool_shed_repository.tool_shed_status column in the database. tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( self.app, tool_shed_repository ) diff -r 5af80141674f5ac3166b6782385c408258606d0a -r 4d2e744f8efc73dc265c3d3bf33c4c2886888b7a lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py --- /dev/null +++ b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -0,0 +1,92 @@ +import logging +import os + +from galaxy import util +from galaxy.util import inflector + +from tool_shed.metadata import metadata_generator + +from tool_shed.util import common_util +from tool_shed.util import shed_util_common as suc +from tool_shed.util import tool_util + +log = logging.getLogger( __name__ ) + + +class InstalledRepositoryMetadataManager( metadata_generator.MetadataGenerator ): + + def __init__( self, app ): + super( InstalledRepositoryMetadataManager, self ).__init__( app ) + self.app = app + + def reset_all_metadata_on_installed_repository( self, id ): + """Reset all metadata on a single tool shed repository installed into a Galaxy instance.""" + invalid_file_tups = [] + metadata_dict = {} + repository = suc.get_installed_tool_shed_repository( self.app, id ) + repository_clone_url = common_util.generate_clone_url_for_installed_repository( self.app, repository ) + tool_path, relative_install_dir = repository.get_tool_relative_path( self.app ) + if relative_install_dir: + original_metadata_dict = repository.metadata + metadata_dict, invalid_file_tups = \ + self.generate_metadata_for_changeset_revision( repository=repository, + changeset_revision=repository.changeset_revision, + repository_clone_url=repository_clone_url, + shed_config_dict = repository.get_shed_config_dict( self.app ), + relative_install_dir=relative_install_dir, + repository_files_dir=None, + resetting_all_metadata_on_repository=False, + updating_installed_repository=False, + persist=False ) + repository.metadata = metadata_dict + if metadata_dict != original_metadata_dict: + suc.update_in_shed_tool_config( self.app, repository ) + self.app.install_model.context.add( repository ) + self.app.install_model.context.flush() + log.debug( 'Metadata has been reset on repository %s.' % repository.name ) + else: + log.debug( 'Metadata did not need to be reset on repository %s.' % repository.name ) + else: + log.debug( 'Error locating installation directory for repository %s.' % repository.name ) + return invalid_file_tups, metadata_dict + + def reset_metadata_on_selected_repositories( self, user, **kwd ): + """ + Inspect the repository changelog to reset metadata for all appropriate changeset revisions. + This method is called from both Galaxy and the Tool Shed. + """ + repository_ids = util.listify( kwd.get( 'repository_ids', None ) ) + message = '' + status = 'done' + if repository_ids: + successful_count = 0 + unsuccessful_count = 0 + for repository_id in repository_ids: + try: + repository = suc.get_installed_tool_shed_repository( self.app, repository_id ) + owner = str( repository.owner ) + invalid_file_tups, metadata_dict = \ + self.reset_all_metadata_on_installed_repository( repository_id ) + if invalid_file_tups: + message = tool_util.generate_message_for_invalid_tools( self.app, + invalid_file_tups, + repository, + None, + as_html=False ) + log.debug( message ) + unsuccessful_count += 1 + else: + log.debug( "Successfully reset metadata on repository %s owned by %s" % ( str( repository.name ), owner ) ) + successful_count += 1 + except: + log.exception( "Error attempting to reset metadata on repository %s", str( repository.name ) ) + unsuccessful_count += 1 + message = "Successfully reset metadata on %d %s. " % \ + ( successful_count, inflector.cond_plural( successful_count, "repository" ) ) + if unsuccessful_count: + message += "Error setting metadata on %d %s - see the paster log for details. " % \ + ( unsuccessful_count, inflector.cond_plural( unsuccessful_count, "repository" ) ) + else: + message = 'Select at least one repository to on which to reset all metadata.' + status = 'error' + return message, status diff -r 5af80141674f5ac3166b6782385c408258606d0a -r 4d2e744f8efc73dc265c3d3bf33c4c2886888b7a lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -8,15 +8,18 @@ import tempfile import threading import logging + from galaxy import util from galaxy.tools import ToolSection + from tool_shed.galaxy_install import install_manager -import tool_shed.util.shed_util_common as suc +from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager + from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import datatype_util from tool_shed.util import hg_util -from tool_shed.util import metadata_util +from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util from tool_shed.util import xml_util @@ -381,17 +384,17 @@ log.exception( "Exception attempting to filter and persist non-shed-related tool panel configs:\n%s" % str( e ) ) finally: lock.release() + irmm = InstalledRepositoryMetadataManager( self.app ) metadata_dict, invalid_file_tups = \ - metadata_util.generate_metadata_for_changeset_revision( app=self.app, - repository=tool_shed_repository, - changeset_revision=tool_shed_repository.changeset_revision, - repository_clone_url=repository_clone_url, - shed_config_dict = self.shed_config_dict, - relative_install_dir=relative_install_dir, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=False, - persist=True ) + irmm.generate_metadata_for_changeset_revision( repository=tool_shed_repository, + changeset_revision=tool_shed_repository.changeset_revision, + repository_clone_url=repository_clone_url, + shed_config_dict = self.shed_config_dict, + relative_install_dir=relative_install_dir, + repository_files_dir=None, + resetting_all_metadata_on_repository=False, + updating_installed_repository=False, + persist=True ) tool_shed_repository.metadata = metadata_dict self.app.install_model.context.add( tool_shed_repository ) self.app.install_model.context.flush() diff -r 5af80141674f5ac3166b6782385c408258606d0a -r 4d2e744f8efc73dc265c3d3bf33c4c2886888b7a lib/tool_shed/metadata/metadata_generator.py --- /dev/null +++ b/lib/tool_shed/metadata/metadata_generator.py @@ -0,0 +1,1038 @@ +import json +import logging +import os +import tempfile + +from galaxy import util +from galaxy.datatypes import checkers +from galaxy.model.orm import and_ +from galaxy.tools.data_manager.manager import DataManager +from galaxy.web import url_for + +from tool_shed.repository_types import util as rt_util + +from tool_shed.util import basic_util +from tool_shed.util import common_util +from tool_shed.util import hg_util +from tool_shed.util import readme_util +from tool_shed.util import shed_util_common as suc +from tool_shed.util import tool_dependency_util +from tool_shed.util import tool_util +from tool_shed.util import xml_util + +log = logging.getLogger( __name__ ) + + +class MetadataGenerator( object ): + + def __init__( self, app ): + self.app = app + self.sa_session = app.model.context.current + self.NOT_TOOL_CONFIGS = [ suc.DATATYPES_CONFIG_FILENAME, + rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME, + rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME, + suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME ] + + def generate_data_manager_metadata( self, repository, repo_dir, data_manager_config_filename, metadata_dict, + shed_config_dict=None ): + """ + Update the received metadata_dict with information from the parsed data_manager_config_filename. + """ + if data_manager_config_filename is None: + return metadata_dict + repo_path = repository.repo_path( self.app ) + try: + # Galaxy Side. + repo_files_directory = repository.repo_files_directory( self.app ) + repo_dir = repo_files_directory + repository_clone_url = common_util.generate_clone_url_for_installed_repository( self.app, repository ) + except AttributeError: + # Tool Shed side. + repo_files_directory = repo_path + repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( None, repository ) + relative_data_manager_dir = util.relpath( os.path.split( data_manager_config_filename )[0], repo_dir ) + rel_data_manager_config_filename = os.path.join( relative_data_manager_dir, + os.path.split( data_manager_config_filename )[1] ) + data_managers = {} + invalid_data_managers = [] + data_manager_metadata = { 'config_filename': rel_data_manager_config_filename, + 'data_managers': data_managers, + 'invalid_data_managers': invalid_data_managers, + 'error_messages': [] } + metadata_dict[ 'data_manager' ] = data_manager_metadata + tree, error_message = xml_util.parse_xml( data_manager_config_filename ) + if tree is None: + # We are not able to load any data managers. + data_manager_metadata[ 'error_messages' ].append( error_message ) + return metadata_dict + tool_path = None + if shed_config_dict: + tool_path = shed_config_dict.get( 'tool_path', None ) + tools = {} + for tool in metadata_dict.get( 'tools', [] ): + tool_conf_name = tool[ 'tool_config' ] + if tool_path: + tool_conf_name = os.path.join( tool_path, tool_conf_name ) + tools[ tool_conf_name ] = tool + root = tree.getroot() + data_manager_tool_path = root.get( 'tool_path', None ) + if data_manager_tool_path: + relative_data_manager_dir = os.path.join( relative_data_manager_dir, data_manager_tool_path ) + for i, data_manager_elem in enumerate( root.findall( 'data_manager' ) ): + tool_file = data_manager_elem.get( 'tool_file', None ) + data_manager_id = data_manager_elem.get( 'id', None ) + if data_manager_id is None: + log.error( 'Data Manager entry is missing id attribute in "%s".' % ( data_manager_config_filename ) ) + invalid_data_managers.append( { 'index': i, + 'error_message': 'Data Manager entry is missing id attribute' } ) + continue + # FIXME: default behavior is to fall back to tool.name. + data_manager_name = data_manager_elem.get( 'name', data_manager_id ) + version = data_manager_elem.get( 'version', DataManager.DEFAULT_VERSION ) + guid = self.generate_guid_for_object( repository_clone_url, DataManager.GUID_TYPE, data_manager_id, version ) + data_tables = [] + if tool_file is None: + log.error( 'Data Manager entry is missing tool_file attribute in "%s".' % ( data_manager_config_filename ) ) + invalid_data_managers.append( { 'index': i, + 'error_message': 'Data Manager entry is missing tool_file attribute' } ) + continue + else: + bad_data_table = False + for data_table_elem in data_manager_elem.findall( 'data_table' ): + data_table_name = data_table_elem.get( 'name', None ) + if data_table_name is None: + log.error( 'Data Manager data_table entry is missing name attribute in "%s".' % ( data_manager_config_filename ) ) + invalid_data_managers.append( { 'index': i, + 'error_message': 'Data Manager entry is missing name attribute' } ) + bad_data_table = True + break + else: + data_tables.append( data_table_name ) + if bad_data_table: + continue + data_manager_metadata_tool_file = os.path.normpath( os.path.join( relative_data_manager_dir, tool_file ) ) + tool_metadata_tool_file = os.path.join( repo_files_directory, data_manager_metadata_tool_file ) + tool = tools.get( tool_metadata_tool_file, None ) + if tool is None: + log.error( "Unable to determine tools metadata for '%s'." % ( data_manager_metadata_tool_file ) ) + invalid_data_managers.append( { 'index': i, + 'error_message': 'Unable to determine tools metadata' } ) + continue + data_managers[ data_manager_id ] = { 'id': data_manager_id, + 'name': data_manager_name, + 'guid': guid, + 'version': version, + 'tool_config_file': data_manager_metadata_tool_file, + 'data_tables': data_tables, + 'tool_guid': tool[ 'guid' ] } + log.debug( 'Loaded Data Manager tool_files: %s' % ( tool_file ) ) + return metadata_dict + + def generate_datatypes_metadata( self, repository, repository_clone_url, repository_files_dir, datatypes_config, + metadata_dict ): + """Update the received metadata_dict with information from the parsed datatypes_config.""" + tree, error_message = xml_util.parse_xml( datatypes_config ) + if tree is None: + return metadata_dict + root = tree.getroot() + repository_datatype_code_files = [] + datatype_files = root.find( 'datatype_files' ) + if datatype_files is not None: + for elem in datatype_files.findall( 'datatype_file' ): + name = elem.get( 'name', None ) + repository_datatype_code_files.append( name ) + metadata_dict[ 'datatype_files' ] = repository_datatype_code_files + datatypes = [] + registration = root.find( 'registration' ) + if registration is not None: + for elem in registration.findall( 'datatype' ): + converters = [] + display_app_containers = [] + datatypes_dict = {} + # Handle defined datatype attributes. + display_in_upload = elem.get( 'display_in_upload', None ) + if display_in_upload: + datatypes_dict[ 'display_in_upload' ] = display_in_upload + dtype = elem.get( 'type', None ) + if dtype: + datatypes_dict[ 'dtype' ] = dtype + extension = elem.get( 'extension', None ) + if extension: + datatypes_dict[ 'extension' ] = extension + max_optional_metadata_filesize = elem.get( 'max_optional_metadata_filesize', None ) + if max_optional_metadata_filesize: + datatypes_dict[ 'max_optional_metadata_filesize' ] = max_optional_metadata_filesize + mimetype = elem.get( 'mimetype', None ) + if mimetype: + datatypes_dict[ 'mimetype' ] = mimetype + subclass = elem.get( 'subclass', None ) + if subclass: + datatypes_dict[ 'subclass' ] = subclass + # Handle defined datatype converters and display applications. + for sub_elem in elem: + if sub_elem.tag == 'converter': + # <converter file="bed_to_gff_converter.xml" target_datatype="gff"/> + tool_config = sub_elem.attrib[ 'file' ] + target_datatype = sub_elem.attrib[ 'target_datatype' ] + # Parse the tool_config to get the guid. + tool_config_path = hg_util.get_config_from_disk( tool_config, repository_files_dir ) + full_path = os.path.abspath( tool_config_path ) + tool, valid, error_message = \ + tool_util.load_tool_from_config( self.app, self.app.security.encode_id( repository.id ), full_path ) + if tool is None: + guid = None + else: + guid = suc.generate_tool_guid( repository_clone_url, tool ) + converter_dict = dict( tool_config=tool_config, + guid=guid, + target_datatype=target_datatype ) + converters.append( converter_dict ) + elif sub_elem.tag == 'display': + # <display file="ucsc/bigwig.xml" /> + # Should we store more than this? + display_file = sub_elem.attrib[ 'file' ] + display_app_dict = dict( display_file=display_file ) + display_app_containers.append( display_app_dict ) + if converters: + datatypes_dict[ 'converters' ] = converters + if display_app_containers: + datatypes_dict[ 'display_app_containers' ] = display_app_containers + if datatypes_dict: + datatypes.append( datatypes_dict ) + if datatypes: + metadata_dict[ 'datatypes' ] = datatypes + return metadata_dict + + def generate_environment_dependency_metadata( self, elem, valid_tool_dependencies_dict ): + """ + The value of env_var_name must match the value of the "set_environment" type + in the tool config's <requirements> tag set, or the tool dependency will be + considered an orphan. + """ + # The value of the received elem looks something like this: + # <set_environment version="1.0"> + # <environment_variable name="JAVA_JAR_PATH" action="set_to">$INSTALL_DIR</environment_variable> + # </set_environment> + for env_elem in elem: + # <environment_variable name="JAVA_JAR_PATH" action="set_to">$INSTALL_DIR</environment_variable> + env_name = env_elem.get( 'name', None ) + if env_name: + requirements_dict = dict( name=env_name, type='set_environment' ) + if 'set_environment' in valid_tool_dependencies_dict: + valid_tool_dependencies_dict[ 'set_environment' ].append( requirements_dict ) + else: + valid_tool_dependencies_dict[ 'set_environment' ] = [ requirements_dict ] + return valid_tool_dependencies_dict + + def generate_guid_for_object( self, repository_clone_url, guid_type, obj_id, version ): + tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) + return '%s/%s/%s/%s' % ( tmp_url, guid_type, obj_id, version ) + + def generate_metadata_for_changeset_revision( self, repository, changeset_revision, repository_clone_url, + shed_config_dict=None, relative_install_dir=None, repository_files_dir=None, + resetting_all_metadata_on_repository=False, updating_installed_repository=False, + persist=False ): + """ + Generate metadata for a repository using its files on disk. To generate metadata + for changeset revisions older than the repository tip, the repository will have been + cloned to a temporary location and updated to a specified changeset revision to access + that changeset revision's disk files, so the value of repository_files_dir will not + always be repository.repo_path( self.app ) (it could be an absolute path to a temporary + directory containing a clone). If it is an absolute path, the value of relative_install_dir + must contain repository.repo_path( self.app ). + + The value of persist will be True when the installed repository contains a valid + tool_data_table_conf.xml.sample file, in which case the entries should ultimately be + persisted to the file referred to by self.app.config.shed_tool_data_table_config. + """ + if shed_config_dict is None: + shed_config_dict = {} + if updating_installed_repository: + # Keep the original tool shed repository metadata if setting metadata on a repository + # installed into a local Galaxy instance for which we have pulled updates. + original_repository_metadata = repository.metadata + else: + original_repository_metadata = None + readme_file_names = readme_util.get_readme_file_names( str( repository.name ) ) + if self.app.name == 'galaxy': + # Shed related tool panel configs are only relevant to Galaxy. + metadata_dict = { 'shed_config_filename' : shed_config_dict.get( 'config_filename' ) } + else: + metadata_dict = {} + readme_files = [] + invalid_file_tups = [] + invalid_tool_configs = [] + tool_dependencies_config = None + original_tool_data_path = self.app.config.tool_data_path + original_tool_data_table_config_path = self.app.config.tool_data_table_config_path + if resetting_all_metadata_on_repository: + if not relative_install_dir: + raise Exception( "The value of repository.repo_path must be sent when resetting all metadata on a repository." ) + # Keep track of the location where the repository is temporarily cloned so that we can + # strip the path when setting metadata. The value of repository_files_dir is the full + # path to the temporary directory to which the repository was cloned. + work_dir = repository_files_dir + files_dir = repository_files_dir + # Since we're working from a temporary directory, we can safely copy sample files included + # in the repository to the repository root. + self.app.config.tool_data_path = repository_files_dir + self.app.config.tool_data_table_config_path = repository_files_dir + else: + # Use a temporary working directory to copy all sample files. + work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-gmfcr" ) + # All other files are on disk in the repository's repo_path, which is the value of + # relative_install_dir. + files_dir = relative_install_dir + if shed_config_dict.get( 'tool_path' ): + files_dir = os.path.join( shed_config_dict[ 'tool_path' ], files_dir ) + self.app.config.tool_data_path = work_dir #FIXME: Thread safe? + self.app.config.tool_data_table_config_path = work_dir + # Handle proprietary datatypes, if any. + datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, files_dir ) + if datatypes_config: + metadata_dict = self.generate_datatypes_metadata( repository, + repository_clone_url, + files_dir, + datatypes_config, + metadata_dict ) + # Get the relative path to all sample files included in the repository for storage in + # the repository's metadata. + sample_file_metadata_paths, sample_file_copy_paths = \ + self.get_sample_files_from_disk( repository_files_dir=files_dir, + tool_path=shed_config_dict.get( 'tool_path' ), + relative_install_dir=relative_install_dir, + resetting_all_metadata_on_repository=resetting_all_metadata_on_repository ) + if sample_file_metadata_paths: + metadata_dict[ 'sample_files' ] = sample_file_metadata_paths + # Copy all sample files included in the repository to a single directory location so we + # can load tools that depend on them. + for sample_file in sample_file_copy_paths: + tool_util.copy_sample_file( self.app, sample_file, dest_path=work_dir ) + # If the list of sample files includes a tool_data_table_conf.xml.sample file, load + # its table elements into memory. + relative_path, filename = os.path.split( sample_file ) + if filename == 'tool_data_table_conf.xml.sample': + new_table_elems, error_message = \ + self.app.tool_data_tables.add_new_entries_from_config_file( config_filename=sample_file, + tool_data_path=self.app.config.tool_data_path, + shed_tool_data_table_config=self.app.config.shed_tool_data_table_config, + persist=False ) + if error_message: + invalid_file_tups.append( ( filename, error_message ) ) + for root, dirs, files in os.walk( files_dir ): + if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0: + if '.hg' in dirs: + dirs.remove( '.hg' ) + for name in files: + # See if we have a repository dependencies defined. + if name == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: + path_to_repository_dependencies_config = os.path.join( root, name ) + metadata_dict, error_message = \ + self.generate_repository_dependency_metadata( path_to_repository_dependencies_config, + metadata_dict, + updating_installed_repository=updating_installed_repository ) + if error_message: + invalid_file_tups.append( ( name, error_message ) ) + # See if we have one or more READ_ME files. + elif name.lower() in readme_file_names: + relative_path_to_readme = self.get_relative_path_to_repository_file( root, + name, + relative_install_dir, + work_dir, + shed_config_dict, + resetting_all_metadata_on_repository ) + readme_files.append( relative_path_to_readme ) + # See if we have a tool config. + elif name not in self.NOT_TOOL_CONFIGS and name.endswith( '.xml' ): + full_path = str( os.path.abspath( os.path.join( root, name ) ) ) + if os.path.getsize( full_path ) > 0: + if not ( checkers.check_binary( full_path ) or + checkers.check_image( full_path ) or + checkers.check_gzip( full_path )[ 0 ] or + checkers.check_bz2( full_path )[ 0 ] or + checkers.check_zip( full_path ) ): + # Make sure we're looking at a tool config and not a display application + # config or something else. + element_tree, error_message = xml_util.parse_xml( full_path ) + if element_tree is None: + is_tool = False + else: + element_tree_root = element_tree.getroot() + is_tool = element_tree_root.tag == 'tool' + if is_tool: + tool, valid, error_message = \ + tool_util.load_tool_from_config( self.app, + self.app.security.encode_id( repository.id ), + full_path ) + if tool is None: + if not valid: + invalid_tool_configs.append( name ) + invalid_file_tups.append( ( name, error_message ) ) + else: + invalid_files_and_errors_tups = \ + tool_util.check_tool_input_params( self.app, + files_dir, + name, + tool, + sample_file_copy_paths ) + can_set_metadata = True + for tup in invalid_files_and_errors_tups: + if name in tup: + can_set_metadata = False + invalid_tool_configs.append( name ) + break + if can_set_metadata: + relative_path_to_tool_config = \ + self.get_relative_path_to_repository_file( root, + name, + relative_install_dir, + work_dir, + shed_config_dict, + resetting_all_metadata_on_repository ) + metadata_dict = self.generate_tool_metadata( relative_path_to_tool_config, + tool, + repository_clone_url, + metadata_dict ) + else: + for tup in invalid_files_and_errors_tups: + invalid_file_tups.append( tup ) + # Find all exported workflows. + elif name.endswith( '.ga' ): + relative_path = os.path.join( root, name ) + if os.path.getsize( os.path.abspath( relative_path ) ) > 0: + fp = open( relative_path, 'rb' ) + workflow_text = fp.read() + fp.close() + if workflow_text: + valid_exported_galaxy_workflow = True + try: + exported_workflow_dict = json.loads( workflow_text ) + except Exception, e: + log.exception( "Skipping file %s since it does not seem to be a valid exported Galaxy workflow: %s" \ + % str( relative_path ), str( e ) ) + valid_exported_galaxy_workflow = False + if valid_exported_galaxy_workflow and \ + 'a_galaxy_workflow' in exported_workflow_dict and \ + exported_workflow_dict[ 'a_galaxy_workflow' ] == 'true': + metadata_dict = self.generate_workflow_metadata( relative_path, + exported_workflow_dict, + metadata_dict ) + # Handle any data manager entries + data_manager_config = hg_util.get_config_from_disk( suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME, files_dir ) + metadata_dict = self.generate_data_manager_metadata( repository, + files_dir, + data_manager_config, + metadata_dict, + shed_config_dict=shed_config_dict ) + + if readme_files: + metadata_dict[ 'readme_files' ] = readme_files + # This step must be done after metadata for tools has been defined. + tool_dependencies_config = hg_util.get_config_from_disk( rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME, files_dir ) + if tool_dependencies_config: + metadata_dict, error_message = \ + self.generate_tool_dependency_metadata( repository, + changeset_revision, + repository_clone_url, + tool_dependencies_config, + metadata_dict, + original_repository_metadata=original_repository_metadata ) + if error_message: + invalid_file_tups.append( ( rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME, error_message ) ) + if invalid_tool_configs: + metadata_dict [ 'invalid_tools' ] = invalid_tool_configs + # Reset the value of the app's tool_data_path and tool_data_table_config_path to their respective original values. + self.app.config.tool_data_path = original_tool_data_path + self.app.config.tool_data_table_config_path = original_tool_data_table_config_path + basic_util.remove_dir( work_dir ) + return metadata_dict, invalid_file_tups + + def generate_package_dependency_metadata( self, elem, valid_tool_dependencies_dict, invalid_tool_dependencies_dict ): + """ + Generate the metadata for a tool dependencies package defined for a repository. The + value of package_name must match the value of the "package" type in the tool config's + <requirements> tag set. + """ + # TODO: make this function a class. + repository_dependency_is_valid = True + repository_dependency_tup = [] + requirements_dict = {} + error_message = '' + package_name = elem.get( 'name', None ) + package_version = elem.get( 'version', None ) + if package_name and package_version: + requirements_dict[ 'name' ] = package_name + requirements_dict[ 'version' ] = package_version + requirements_dict[ 'type' ] = 'package' + for sub_elem in elem: + if sub_elem.tag == 'readme': + requirements_dict[ 'readme' ] = sub_elem.text + elif sub_elem.tag == 'repository': + # We have a complex repository dependency. If the returned value of repository_dependency_is_valid + # is True, the tool dependency definition will be set as invalid. This is currently the only case + # where a tool dependency definition is considered invalid. + repository_dependency_tup, repository_dependency_is_valid, error_message = \ + self.handle_repository_elem( repository_elem=sub_elem, + only_if_compiling_contained_td=False, + updating_installed_repository=False ) + elif sub_elem.tag == 'install': + package_install_version = sub_elem.get( 'version', '1.0' ) + if package_install_version == '1.0': + # Complex repository dependencies can be defined within the last <actions> tag set contained in an + # <actions_group> tag set. Comments, <repository> tag sets and <readme> tag sets will be skipped + # in tool_dependency_util.parse_package_elem(). + actions_elem_tuples = tool_dependency_util.parse_package_elem( sub_elem, + platform_info_dict=None, + include_after_install_actions=False ) + if actions_elem_tuples: + # We now have a list of a single tuple that looks something like: + # [(True, <Element 'actions' at 0x104017850>)] + actions_elem_tuple = actions_elem_tuples[ 0 ] + in_actions_group, actions_elem = actions_elem_tuple + if in_actions_group: + # Since we're inside an <actions_group> tag set, inspect the actions_elem to see if a complex + # repository dependency is defined. By definition, complex repository dependency definitions + # contained within the last <actions> tag set within an <actions_group> tag set will have the + # value of "only_if_compiling_contained_td" set to True in + for action_elem in actions_elem: + if action_elem.tag == 'package': + # <package name="libgtextutils" version="0.6"> + # <repository name="package_libgtextutils_0_6" owner="test" prior_installation_required="True" /> + # </package> + ae_package_name = action_elem.get( 'name', None ) + ae_package_version = action_elem.get( 'version', None ) + if ae_package_name and ae_package_version: + for sub_action_elem in action_elem: + if sub_action_elem.tag == 'repository': + # We have a complex repository dependency. + repository_dependency_tup, repository_dependency_is_valid, error_message = \ + self.handle_repository_elem( repository_elem=sub_action_elem, + only_if_compiling_contained_td=True, + updating_installed_repository=False ) + elif action_elem.tag == 'action': + # <action type="set_environment_for_install"> + # <repository changeset_revision="b107b91b3574" name="package_readline_6_2" owner="devteam" prior_installation_required="True" toolshed="http://localhost:9009"> + # <package name="readline" version="6.2" /> + # </repository> + # </action> + for sub_action_elem in action_elem: + if sub_action_elem.tag == 'repository': + # We have a complex repository dependency. + repository_dependency_tup, repository_dependency_is_valid, error_message = \ + self.handle_repository_elem( repository_elem=sub_action_elem, + only_if_compiling_contained_td=True, + updating_installed_repository=False ) + if requirements_dict: + dependency_key = '%s/%s' % ( package_name, package_version ) + if repository_dependency_is_valid: + valid_tool_dependencies_dict[ dependency_key ] = requirements_dict + else: + # Append the error message to the requirements_dict. + requirements_dict[ 'error' ] = error_message + invalid_tool_dependencies_dict[ dependency_key ] = requirements_dict + return valid_tool_dependencies_dict, \ + invalid_tool_dependencies_dict, \ + repository_dependency_tup, \ + repository_dependency_is_valid, \ + error_message + + def generate_repository_dependency_metadata( self, repository_dependencies_config, metadata_dict, + updating_installed_repository=False ): + """ + Generate a repository dependencies dictionary based on valid information defined in the received + repository_dependencies_config. This method is called from the tool shed as well as from Galaxy. + """ + error_message = '' + # Make sure we're looking at a valid repository_dependencies.xml file. + tree, error_message = xml_util.parse_xml( repository_dependencies_config ) + if tree is None: + xml_is_valid = False + else: + root = tree.getroot() + xml_is_valid = root.tag == 'repositories' + if xml_is_valid: + invalid_repository_dependencies_dict = dict( description=root.get( 'description' ) ) + invalid_repository_dependency_tups = [] + valid_repository_dependencies_dict = dict( description=root.get( 'description' ) ) + valid_repository_dependency_tups = [] + for repository_elem in root.findall( 'repository' ): + repository_dependency_tup, repository_dependency_is_valid, err_msg = \ + self.handle_repository_elem( repository_elem, + only_if_compiling_contained_td=False, + updating_installed_repository=updating_installed_repository ) + if repository_dependency_is_valid: + valid_repository_dependency_tups.append( repository_dependency_tup ) + else: + # Append the error_message to the repository dependencies tuple. + toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + repository_dependency_tup + repository_dependency_tup = ( toolshed, + name, + owner, + changeset_revision, + prior_installation_required, + only_if_compiling_contained_td, + err_msg ) + invalid_repository_dependency_tups.append( repository_dependency_tup ) + error_message += err_msg + if invalid_repository_dependency_tups: + invalid_repository_dependencies_dict[ 'repository_dependencies' ] = invalid_repository_dependency_tups + metadata_dict[ 'invalid_repository_dependencies' ] = invalid_repository_dependencies_dict + if valid_repository_dependency_tups: + valid_repository_dependencies_dict[ 'repository_dependencies' ] = valid_repository_dependency_tups + metadata_dict[ 'repository_dependencies' ] = valid_repository_dependencies_dict + return metadata_dict, error_message + + def generate_tool_metadata( self, tool_config, tool, repository_clone_url, metadata_dict ): + """Update the received metadata_dict with changes that have been applied to the received tool.""" + # Generate the guid. + guid = suc.generate_tool_guid( repository_clone_url, tool ) + # Handle tool.requirements. + tool_requirements = [] + for tool_requirement in tool.requirements: + name = str( tool_requirement.name ) + type = str( tool_requirement.type ) + version = str( tool_requirement.version ) if tool_requirement.version else None + requirement_dict = dict( name=name, + type=type, + version=version ) + tool_requirements.append( requirement_dict ) + # Handle tool.tests. + tool_tests = [] + if tool.tests: + for ttb in tool.tests: + required_files = [] + for required_file in ttb.required_files: + value, extra = required_file + required_files.append( ( value ) ) + inputs = [] + for param_name, values in ttb.inputs.iteritems(): + # Handle improperly defined or strange test parameters and values. + if param_name is not None: + if values is None: + # An example is the 3rd test in http://testtoolshed.g2.bx.psu.edu/view/devteam/samtools_rmdup + # which is defined as: + # <test> + # <param name="input1" value="1.bam" ftype="bam" /> + # <param name="bam_paired_end_type_selector" value="PE" /> + # <param name="force_se" /> + # <output name="output1" file="1.bam" ftype="bam" sort="True" /> + # </test> + inputs.append( ( param_name, values ) ) + else: + if len( values ) == 1: + inputs.append( ( param_name, values[ 0 ] ) ) + else: + inputs.append( ( param_name, values ) ) + outputs = [] + for output in ttb.outputs: + name, file_name, extra = output + outputs.append( ( name, basic_util.strip_path( file_name ) if file_name else None ) ) + if file_name not in required_files and file_name is not None: + required_files.append( file_name ) + test_dict = dict( name=str( ttb.name ), + required_files=required_files, + inputs=inputs, + outputs=outputs ) + tool_tests.append( test_dict ) + # Determine if the tool should be loaded into the tool panel. Examples of valid tools that + # should not be displayed in the tool panel are datatypes converters and DataManager tools + # (which are of type 'manage_data'). + datatypes = metadata_dict.get( 'datatypes', None ) + add_to_tool_panel_attribute = self.set_add_to_tool_panel_attribute_for_tool( tool=tool, + guid=guid, + datatypes=datatypes ) + tool_dict = dict( id=tool.id, + guid=guid, + name=tool.name, + version=tool.version, + description=tool.description, + version_string_cmd = tool.version_string_cmd, + tool_config=tool_config, + tool_type=tool.tool_type, + requirements=tool_requirements, + tests=tool_tests, + add_to_tool_panel=add_to_tool_panel_attribute ) + if 'tools' in metadata_dict: + metadata_dict[ 'tools' ].append( tool_dict ) + else: + metadata_dict[ 'tools' ] = [ tool_dict ] + return metadata_dict + + def generate_tool_dependency_metadata( self, repository, changeset_revision, repository_clone_url, tool_dependencies_config, + metadata_dict, original_repository_metadata=None ): + """ + If the combination of name, version and type of each element is defined in the <requirement> tag for + at least one tool in the repository, then update the received metadata_dict with information from the + parsed tool_dependencies_config. + """ + error_message = '' + if original_repository_metadata: + # Keep a copy of the original tool dependencies dictionary and the list of tool + # dictionaries in the metadata. + original_valid_tool_dependencies_dict = original_repository_metadata.get( 'tool_dependencies', None ) + original_invalid_tool_dependencies_dict = original_repository_metadata.get( 'invalid_tool_dependencies', None ) + else: + original_valid_tool_dependencies_dict = None + original_invalid_tool_dependencies_dict = None + tree, error_message = xml_util.parse_xml( tool_dependencies_config ) + if tree is None: + return metadata_dict, error_message + root = tree.getroot() + tool_dependency_is_valid = True + valid_tool_dependencies_dict = {} + invalid_tool_dependencies_dict = {} + valid_repository_dependency_tups = [] + invalid_repository_dependency_tups = [] + tools_metadata = metadata_dict.get( 'tools', None ) + description = root.get( 'description' ) + for elem in root: + if elem.tag == 'package': + valid_tool_dependencies_dict, \ + invalid_tool_dependencies_dict, \ + repository_dependency_tup, \ + repository_dependency_is_valid, \ + message = self.generate_package_dependency_metadata( elem, + valid_tool_dependencies_dict, + invalid_tool_dependencies_dict ) + if repository_dependency_is_valid: + if repository_dependency_tup and repository_dependency_tup not in valid_repository_dependency_tups: + # We have a valid complex repository dependency. + valid_repository_dependency_tups.append( repository_dependency_tup ) + else: + if repository_dependency_tup and repository_dependency_tup not in invalid_repository_dependency_tups: + # We have an invalid complex repository dependency, so mark the tool dependency as invalid. + tool_dependency_is_valid = False + # Append the error message to the invalid repository dependency tuple. + toolshed, \ + name, \ + owner, \ + changeset_revision, \ + prior_installation_required, \ + only_if_compiling_contained_td \ + = repository_dependency_tup + repository_dependency_tup = \ + ( toolshed, \ + name, \ + owner, \ + changeset_revision, \ + prior_installation_required, \ + only_if_compiling_contained_td, \ + message ) + invalid_repository_dependency_tups.append( repository_dependency_tup ) + error_message = '%s %s' % ( error_message, message ) + elif elem.tag == 'set_environment': + valid_tool_dependencies_dict = \ + self.generate_environment_dependency_metadata( elem, valid_tool_dependencies_dict ) + if valid_tool_dependencies_dict: + if original_valid_tool_dependencies_dict: + # We're generating metadata on an update pulled to a tool shed repository installed + # into a Galaxy instance, so handle changes to tool dependencies appropriately. + irm = self.app.installed_repository_manager + updated_tool_dependency_names, deleted_tool_dependency_names = \ + irm.handle_existing_tool_dependencies_that_changed_in_update( repository, + original_valid_tool_dependencies_dict, + valid_tool_dependencies_dict ) + metadata_dict[ 'tool_dependencies' ] = valid_tool_dependencies_dict + if invalid_tool_dependencies_dict: + metadata_dict[ 'invalid_tool_dependencies' ] = invalid_tool_dependencies_dict + if valid_repository_dependency_tups: + metadata_dict = \ + self.update_repository_dependencies_metadata( metadata=metadata_dict, + repository_dependency_tups=valid_repository_dependency_tups, + is_valid=True, + description=description ) + if invalid_repository_dependency_tups: + metadata_dict = \ + self.update_repository_dependencies_metadata( metadata=metadata_dict, + repository_dependency_tups=invalid_repository_dependency_tups, + is_valid=False, + description=description ) + return metadata_dict, error_message + + def generate_workflow_metadata( self, relative_path, exported_workflow_dict, metadata_dict ): + """ + Update the received metadata_dict with changes that have been applied to the + received exported_workflow_dict. + """ + if 'workflows' in metadata_dict: + metadata_dict[ 'workflows' ].append( ( relative_path, exported_workflow_dict ) ) + else: + metadata_dict[ 'workflows' ] = [ ( relative_path, exported_workflow_dict ) ] + return metadata_dict + + def get_relative_path_to_repository_file( self, root, name, relative_install_dir, work_dir, shed_config_dict, + resetting_all_metadata_on_repository ): + if resetting_all_metadata_on_repository: + full_path_to_file = os.path.join( root, name ) + stripped_path_to_file = full_path_to_file.replace( work_dir, '' ) + if stripped_path_to_file.startswith( '/' ): + stripped_path_to_file = stripped_path_to_file[ 1: ] + relative_path_to_file = os.path.join( relative_install_dir, stripped_path_to_file ) + else: + relative_path_to_file = os.path.join( root, name ) + if relative_install_dir and \ + shed_config_dict.get( 'tool_path' ) and \ + relative_path_to_file.startswith( os.path.join( shed_config_dict.get( 'tool_path' ), relative_install_dir ) ): + relative_path_to_file = relative_path_to_file[ len( shed_config_dict.get( 'tool_path' ) ) + 1: ] + return relative_path_to_file + + def get_sample_files_from_disk( self, repository_files_dir, tool_path=None, relative_install_dir=None, + resetting_all_metadata_on_repository=False ): + if resetting_all_metadata_on_repository: + # Keep track of the location where the repository is temporarily cloned so that we can strip + # it when setting metadata. + work_dir = repository_files_dir + sample_file_metadata_paths = [] + sample_file_copy_paths = [] + for root, dirs, files in os.walk( repository_files_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name.endswith( '.sample' ): + if resetting_all_metadata_on_repository: + full_path_to_sample_file = os.path.join( root, name ) + stripped_path_to_sample_file = full_path_to_sample_file.replace( work_dir, '' ) + if stripped_path_to_sample_file.startswith( '/' ): + stripped_path_to_sample_file = stripped_path_to_sample_file[ 1: ] + relative_path_to_sample_file = os.path.join( relative_install_dir, stripped_path_to_sample_file ) + if os.path.exists( relative_path_to_sample_file ): + sample_file_copy_paths.append( relative_path_to_sample_file ) + else: + sample_file_copy_paths.append( full_path_to_sample_file ) + else: + relative_path_to_sample_file = os.path.join( root, name ) + sample_file_copy_paths.append( relative_path_to_sample_file ) + if tool_path and relative_install_dir: + if relative_path_to_sample_file.startswith( os.path.join( tool_path, relative_install_dir ) ): + relative_path_to_sample_file = relative_path_to_sample_file[ len( tool_path ) + 1 :] + sample_file_metadata_paths.append( relative_path_to_sample_file ) + return sample_file_metadata_paths, sample_file_copy_paths + + def handle_repository_elem( self, repository_elem, only_if_compiling_contained_td=False, updating_installed_repository=False ): + """ + Process the received repository_elem which is a <repository> tag either from a + repository_dependencies.xml file or a tool_dependencies.xml file. If the former, + we're generating repository dependencies metadata for a repository in the Tool Shed. + If the latter, we're generating package dependency metadata within Galaxy or the + Tool Shed. + """ + is_valid = True + error_message = '' + toolshed = repository_elem.get( 'toolshed', None ) + name = repository_elem.get( 'name', None ) + owner = repository_elem.get( 'owner', None ) + changeset_revision = repository_elem.get( 'changeset_revision', None ) + prior_installation_required = str( repository_elem.get( 'prior_installation_required', False ) ) + if self.app.name == 'galaxy': + if updating_installed_repository: + pass + else: + # We're installing a repository into Galaxy, so make sure its contained repository + # dependency definition is valid. + if toolshed is None or name is None or owner is None or changeset_revision is None: + # Raise an exception here instead of returning an error_message to keep the + # installation from proceeding. Reaching here implies a bug in the Tool Shed + # framework. + error_message = 'Installation halted because the following repository dependency definition is invalid:\n' + error_message += xml_util.xml_to_string( repository_elem, use_indent=True ) + raise Exception( error_message ) + if not toolshed: + # Default to the current tool shed. + toolshed = str( url_for( '/', qualified=True ) ).rstrip( '/' ) + repository_dependency_tup = [ toolshed, + name, + owner, + changeset_revision, + prior_installation_required, + str( only_if_compiling_contained_td ) ] + user = None + repository = None + toolshed = common_util.remove_protocol_from_tool_shed_url( toolshed ) + if self.app.name == 'galaxy': + # We're in Galaxy. We reach here when we're generating the metadata for a tool + # dependencies package defined for a repository or when we're generating metadata + # for an installed repository. See if we can locate the installed repository via + # the changeset_revision defined in the repository_elem (it may be outdated). If + # we're successful in locating an installed repository with the attributes defined + # in the repository_elem, we know it is valid. + repository = suc.get_repository_for_dependency_relationship( self.app, + toolshed, + name, + owner, + changeset_revision ) + if repository: + return repository_dependency_tup, is_valid, error_message + else: + # Send a request to the tool shed to retrieve appropriate additional changeset + # revisions with which the repository + # may have been installed. + text = suc.get_updated_changeset_revisions_from_tool_shed( self.app, + toolshed, + name, + owner, + changeset_revision ) + if text: + updated_changeset_revisions = util.listify( text ) + for updated_changeset_revision in updated_changeset_revisions: + repository = suc.get_repository_for_dependency_relationship( self.app, + toolshed, + name, + owner, + updated_changeset_revision ) + if repository: + return repository_dependency_tup, is_valid, error_message + if updating_installed_repository: + # The repository dependency was included in an update to the installed + # repository, so it will not yet be installed. Return the tuple for later + # installation. + return repository_dependency_tup, is_valid, error_message + if updating_installed_repository: + # The repository dependency was included in an update to the installed repository, + # so it will not yet be installed. Return the tuple for later installation. + return repository_dependency_tup, is_valid, error_message + # Don't generate an error message for missing repository dependencies that are required + # only if compiling the dependent repository's tool dependency. + if not only_if_compiling_contained_td: + # We'll currently default to setting the repository dependency definition as invalid + # if an installed repository cannot be found. This may not be ideal because the tool + # shed may have simply been inaccessible when metadata was being generated for the + # installed tool shed repository. + error_message = "Ignoring invalid repository dependency definition for tool shed %s, name %s, owner %s, " % \ + ( toolshed, name, owner ) + error_message += "changeset revision %s." % changeset_revision + log.debug( error_message ) + is_valid = False + return repository_dependency_tup, is_valid, error_message + else: + # We're in the tool shed. + if suc.tool_shed_is_this_tool_shed( toolshed ): + try: + user = self.sa_session.query( self.app.model.User ) \ + .filter( self.app.model.User.table.c.username == owner ) \ + .one() + except Exception, e: + error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, " % \ + ( toolshed, name, owner ) + error_message += "changeset revision %s because the owner is invalid. " % changeset_revision + log.debug( error_message ) + is_valid = False + return repository_dependency_tup, is_valid, error_message + try: + repository = self.sa_session.query( self.app.model.Repository ) \ + .filter( and_( self.app.model.Repository.table.c.name == name, + self.app.model.Repository.table.c.user_id == user.id ) ) \ + .one() + except: + error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, " % \ + ( toolshed, name, owner ) + error_message += "changeset revision %s because the name is invalid. " % changeset_revision + log.debug( error_message ) + is_valid = False + return repository_dependency_tup, is_valid, error_message + repo = hg_util.get_repo_for_repository( self.app, repository=repository, repo_path=None, create=False ) + + # The received changeset_revision may be None since defining it in the dependency definition is optional. + # If this is the case, the default will be to set its value to the repository dependency tip revision. + # This probably occurs only when handling circular dependency definitions. + tip_ctx = repo.changectx( repo.changelog.tip() ) + # Make sure the repo.changlog includes at least 1 revision. + if changeset_revision is None and tip_ctx.rev() >= 0: + changeset_revision = str( tip_ctx ) + repository_dependency_tup = [ toolshed, + name, + owner, + changeset_revision, + prior_installation_required, + str( only_if_compiling_contained_td ) ] + return repository_dependency_tup, is_valid, error_message + else: + # Find the specified changeset revision in the repository's changelog to see if it's valid. + found = False + for changeset in repo.changelog: + changeset_hash = str( repo.changectx( changeset ) ) + if changeset_hash == changeset_revision: + found = True + break + if not found: + error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, " % \ + ( toolshed, name, owner ) + error_message += "changeset revision %s because the changeset revision is invalid. " % changeset_revision + log.debug( error_message ) + is_valid = False + return repository_dependency_tup, is_valid, error_message + else: + # Repository dependencies are currently supported within a single tool shed. + error_message = "Repository dependencies are currently supported only within the same tool shed. Ignoring " + error_message += "repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s. " % \ + ( toolshed, name, owner, changeset_revision ) + log.debug( error_message ) + is_valid = False + return repository_dependency_tup, is_valid, error_message + return repository_dependency_tup, is_valid, error_message + + def set_add_to_tool_panel_attribute_for_tool( self, tool, guid, datatypes ): + """ + Determine if a tool should be loaded into the Galaxy tool panel. Examples of valid tools that + should not be displayed in the tool panel are datatypes converters and DataManager tools. + """ + if hasattr( tool, 'tool_type' ): + if tool.tool_type in [ 'manage_data' ]: + # We have a DataManager tool. + return False + if datatypes: + for datatype_dict in datatypes: + converters = datatype_dict.get( 'converters', None ) + # [{"converters": + # [{"target_datatype": "gff", + # "tool_config": "bed_to_gff_converter.xml", + # "guid": "localhost:9009/repos/test/bed_to_gff_converter/CONVERTER_bed_to_gff_0/2.0.0"}], + # "display_in_upload": "true", + # "dtype": "galaxy.datatypes.interval:Bed", + # "extension": "bed"}] + if converters: + for converter_dict in converters: + converter_guid = converter_dict.get( 'guid', None ) + if converter_guid: + if converter_guid == guid: + # We have a datatypes converter. + return False + return True + + def update_repository_dependencies_metadata( self, metadata, repository_dependency_tups, is_valid, description ): + if is_valid: + repository_dependencies_dict = metadata.get( 'repository_dependencies', None ) + else: + repository_dependencies_dict = metadata.get( 'invalid_repository_dependencies', None ) + for repository_dependency_tup in repository_dependency_tups: + if is_valid: + tool_shed, \ + name, \ + owner, \ + changeset_revision, \ + prior_installation_required, \ + only_if_compiling_contained_td \ + = repository_dependency_tup + else: + tool_shed, \ + name, \ + owner, \ + changeset_revision, \ + prior_installation_required, \ + only_if_compiling_contained_td, \ + error_message \ + = repository_dependency_tup + if repository_dependencies_dict: + repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] ) + for repository_dependency_tup in repository_dependency_tups: + if repository_dependency_tup not in repository_dependencies: + repository_dependencies.append( repository_dependency_tup ) + repository_dependencies_dict[ 'repository_dependencies' ] = repository_dependencies + else: + repository_dependencies_dict = dict( description=description, + repository_dependencies=repository_dependency_tups ) + if repository_dependencies_dict: + if is_valid: + metadata[ 'repository_dependencies' ] = repository_dependencies_dict + else: + metadata[ 'invalid_repository_dependencies' ] = repository_dependencies_dict + return metadata This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add a DependencyDisplayer class for Galaxy installs from the Tool Shed.
by commits-noreply＠bitbucket.org 16 Jul '14

16 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/5af80141674f/ Changeset: 5af80141674f User: greg Date: 2014-07-16 20:30:36 Summary: Add a DependencyDisplayer class for Galaxy installs from the Tool Shed. Affected #: 9 files diff -r 3232e68101a46276f07c1f545ccf37c55552f66a -r 5af80141674f5ac3166b6782385c408258606d0a lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -3,16 +3,12 @@ import shutil from admin import AdminGalaxy -from galaxy import eggs from galaxy import web from galaxy import util from galaxy.web.form_builder import CheckboxField -from galaxy.web.framework.helpers import grids -from galaxy.web.framework.helpers import iff from galaxy.util import json from galaxy.model.orm import or_ -import tool_shed.util.shed_util_common as suc import tool_shed.repository_types.util as rt_util from tool_shed.util import common_util @@ -23,11 +19,14 @@ from tool_shed.util import metadata_util from tool_shed.util import readme_util from tool_shed.util import repository_maintenance_util +from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util from tool_shed.util import workflow_util -from tool_shed.util import xml_util + +from tool_shed.galaxy_install import dependency_display from tool_shed.galaxy_install import install_manager + from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager import tool_shed.galaxy_install.grids.admin_toolshed_grids as admin_toolshed_grids from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager @@ -221,17 +220,20 @@ @web.require_admin def deactivate_or_uninstall_repository( self, trans, **kwd ): """ - Handle all changes when a tool shed repository is being deactivated or uninstalled. Notice that if the repository contents include - a file named tool_data_table_conf.xml.sample, its entries are not removed from the defined config.shed_tool_data_table_config. This - is because it becomes a bit complex to determine if other installed repositories include tools that require the same entry. For now - we'll never delete entries from config.shed_tool_data_table_config, but we may choose to do so in the future if it becomes necessary. + Handle all changes when a tool shed repository is being deactivated or uninstalled. Notice + that if the repository contents include a file named tool_data_table_conf.xml.sample, its + entries are not removed from the defined config.shed_tool_data_table_config. This is because + it becomes a bit complex to determine if other installed repositories include tools that + require the same entry. For now we'll never delete entries from config.shed_tool_data_table_config, + but we may choose to do so in the future if it becomes necessary. """ message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) remove_from_disk = kwd.get( 'remove_from_disk', '' ) remove_from_disk_checked = CheckboxField.is_checked( remove_from_disk ) tool_shed_repository = suc.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) - shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository ) + shed_tool_conf, tool_path, relative_install_dir = \ + suc.get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository ) if relative_install_dir: if tool_path: relative_install_dir = os.path.join( tool_path, relative_install_dir ) @@ -247,7 +249,10 @@ data_manager_util.remove_from_data_manager( trans.app, tool_shed_repository ) if tool_shed_repository.includes_datatypes: # Deactivate proprietary datatypes. - installed_repository_dict = datatype_util.load_installed_datatypes( trans.app, tool_shed_repository, repository_install_dir, deactivate=True ) + installed_repository_dict = datatype_util.load_installed_datatypes( trans.app, + tool_shed_repository, + repository_install_dir, + deactivate=True ) if installed_repository_dict: converter_path = installed_repository_dict.get( 'converter_path' ) if converter_path is not None: @@ -779,12 +784,12 @@ trans.install_model.context.add( repository ) trans.install_model.context.flush() message = "The repository information has been updated." - containers_dict = metadata_util.populate_containers_dict_from_repository_metadata( app=trans.app, - tool_shed_url=tool_shed_url, - tool_path=tool_path, - repository=repository, - reinstalling=False, - required_repo_info_dicts=None ) + dd = dependency_display.DependencyDisplayer( trans.app ) + containers_dict = dd.populate_containers_dict_from_repository_metadata( tool_shed_url=tool_shed_url, + tool_path=tool_path, + repository=repository, + reinstalling=False, + required_repo_info_dicts=None ) return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako', repository=repository, description=description, @@ -1049,6 +1054,7 @@ includes_tool_dependencies = util.string_as_bool( repo_information_dict.get( 'includes_tool_dependencies', False ) ) encoded_repo_info_dicts = util.listify( repo_information_dict.get( 'repo_info_dicts', [] ) ) repo_info_dicts = [ encoding_util.tool_shed_decode( encoded_repo_info_dict ) for encoded_repo_info_dict in encoded_repo_info_dicts ] + dd = dependency_display.DependencyDisplayer( trans.app ) install_repository_manager = install_manager.InstallRepositoryManager( trans.app ) if ( not includes_tools_for_display_in_tool_panel and kwd.get( 'select_shed_tool_panel_config_button', False ) ) or \ ( includes_tools_for_display_in_tool_panel and kwd.get( 'select_tool_panel_section_button', False ) ): @@ -1149,14 +1155,14 @@ # defined repository (and possibly tool) dependencies. In this case, merging will result in newly defined # dependencies to be lost. We pass the updating parameter to make sure merging occurs only when appropriate. containers_dict = \ - install_repository_manager.populate_containers_dict_for_new_install( tool_shed_url=tool_shed_url, - tool_path=tool_path, - readme_files_dict=readme_files_dict, - installed_repository_dependencies=installed_repository_dependencies, - missing_repository_dependencies=missing_repository_dependencies, - installed_tool_dependencies=installed_tool_dependencies, - missing_tool_dependencies=missing_tool_dependencies, - updating=updating ) + dd.populate_containers_dict_for_new_install( tool_shed_url=tool_shed_url, + tool_path=tool_path, + readme_files_dict=readme_files_dict, + installed_repository_dependencies=installed_repository_dependencies, + missing_repository_dependencies=missing_repository_dependencies, + installed_tool_dependencies=installed_tool_dependencies, + missing_tool_dependencies=missing_tool_dependencies, + updating=updating ) else: # We're installing a list of repositories, each of which may have tool dependencies or repository dependencies. containers_dicts = [] @@ -1184,17 +1190,17 @@ name = dependencies_for_repository_dict.get( 'name', None ) repository_owner = dependencies_for_repository_dict.get( 'repository_owner', None ) containers_dict = \ - install_repository_manager.populate_containers_dict_for_new_install( tool_shed_url=tool_shed_url, - tool_path=tool_path, - readme_files_dict=None, - installed_repository_dependencies=installed_repository_dependencies, - missing_repository_dependencies=missing_repository_dependencies, - installed_tool_dependencies=installed_tool_dependencies, - missing_tool_dependencies=missing_tool_dependencies, - updating=updating ) + dd.populate_containers_dict_for_new_install( tool_shed_url=tool_shed_url, + tool_path=tool_path, + readme_files_dict=None, + installed_repository_dependencies=installed_repository_dependencies, + missing_repository_dependencies=missing_repository_dependencies, + installed_tool_dependencies=installed_tool_dependencies, + missing_tool_dependencies=missing_tool_dependencies, + updating=updating ) containers_dicts.append( containers_dict ) # Merge all containers into a single container. - containers_dict = install_repository_manager.merge_containers_dicts_for_new_install( containers_dicts ) + containers_dict = dd.merge_containers_dicts_for_new_install( containers_dicts ) # Handle tool dependencies check box. if trans.app.config.tool_dependency_dir is None: if includes_tool_dependencies: @@ -1647,19 +1653,20 @@ original_section_name = '' tool_panel_section_select_field = None shed_tool_conf_select_field = tool_util.build_shed_tool_conf_select_field( trans ) - irm = install_manager.InstallRepositoryManager( trans.app ) + dd = dependency_display.DependencyDisplayer( trans.app ) containers_dict = \ - irm.populate_containers_dict_for_new_install( tool_shed_url=tool_shed_url, - tool_path=tool_path, - readme_files_dict=readme_files_dict, - installed_repository_dependencies=installed_repository_dependencies, - missing_repository_dependencies=missing_repository_dependencies, - installed_tool_dependencies=installed_tool_dependencies, - missing_tool_dependencies=missing_tool_dependencies, - updating=False ) - # Since we're reinstalling we'll merge the list of missing repository dependencies into the list of installed repository dependencies since each displayed - # repository dependency will display a status, whether installed or missing. - containers_dict = irm.merge_missing_repository_dependencies_to_installed_container( containers_dict ) + dd.populate_containers_dict_for_new_install( tool_shed_url=tool_shed_url, + tool_path=tool_path, + readme_files_dict=readme_files_dict, + installed_repository_dependencies=installed_repository_dependencies, + missing_repository_dependencies=missing_repository_dependencies, + installed_tool_dependencies=installed_tool_dependencies, + missing_tool_dependencies=missing_tool_dependencies, + updating=False ) + # Since we're reinstalling we'll merge the list of missing repository dependencies into the list of + # installed repository dependencies since each displayed repository dependency will display a status, + # whether installed or missing. + containers_dict = dd.merge_missing_repository_dependencies_to_installed_container( containers_dict ) # Handle repository dependencies check box. install_repository_dependencies_check_box = CheckboxField( 'install_repository_dependencies', checked=True ) # Handle tool dependencies check box. @@ -1796,12 +1803,12 @@ status = 'error' shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) ) - containers_dict = metadata_util.populate_containers_dict_from_repository_metadata( app=trans.app, - tool_shed_url=tool_shed_url, - tool_path=tool_path, - repository=repository, - reinstalling=False, - required_repo_info_dicts=None ) + dd = dependency_display.DependencyDisplayer( trans.app ) + containers_dict = dd.populate_containers_dict_from_repository_metadata( tool_shed_url=tool_shed_url, + tool_path=tool_path, + repository=repository, + reinstalling=False, + required_repo_info_dicts=None ) return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako', repository=repository, description=repository.description, @@ -1860,7 +1867,8 @@ message = "Error attempting to uninstall tool dependencies: %s" % message status = 'error' else: - message = "These tool dependencies have been uninstalled: %s" % ','.join( td.name for td in tool_dependencies_for_uninstallation ) + message = "These tool dependencies have been uninstalled: %s" % \ + ','.join( td.name for td in tool_dependencies_for_uninstallation ) td_ids = [ trans.security.encode_id( td.id ) for td in tool_shed_repository.tool_dependencies ] return trans.response.send_redirect( web.url_for( controller='admin_toolshed', action='manage_repository_tool_dependencies', diff -r 3232e68101a46276f07c1f545ccf37c55552f66a -r 5af80141674f5ac3166b6782385c408258606d0a lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -6,16 +6,20 @@ from time import strftime from datetime import date from datetime import datetime + from galaxy import util from galaxy import web -from galaxy.util.odict import odict from galaxy.web.base.controller import BaseUIController from galaxy.web.form_builder import CheckboxField from galaxy.web.framework.helpers import grids from galaxy.util import json from galaxy.model.orm import and_ + from tool_shed.capsule import capsule_manager from tool_shed.dependencies.repository import relation_builder + +from tool_shed.galaxy_install import dependency_display + from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import container_util @@ -24,17 +28,16 @@ from tool_shed.util import metadata_util from tool_shed.util import readme_util from tool_shed.util import repository_maintenance_util -from tool_shed.util import review_util from tool_shed.util import search_util from tool_shed.util import shed_util_common as suc -from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util from tool_shed.util import workflow_util -from tool_shed.galaxy_install import install_manager + from galaxy.webapps.tool_shed.util import ratings_util -import galaxy.tools + import tool_shed.grids.repository_grids as repository_grids import tool_shed.grids.util as grids_util + import tool_shed.repository_types.util as rt_util from galaxy import eggs @@ -2421,9 +2424,8 @@ status = 'warning' else: # Handle messaging for orphan tool dependency definitions. - orphan_message = tool_dependency_util.generate_message_for_orphan_tool_dependencies( trans, - repository, - metadata ) + dd = dependency_display.DependencyDisplayer( trans.app ) + orphan_message = dd.generate_message_for_orphan_tool_dependencies( repository, metadata ) if orphan_message: message += orphan_message status = 'warning' @@ -3313,7 +3315,8 @@ repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() if str( repository.type ) != rt_util.TOOL_DEPENDENCY_DEFINITION: # Handle messaging for orphan tool dependency definitions. - orphan_message = tool_dependency_util.generate_message_for_orphan_tool_dependencies( trans, repository, metadata ) + dd = dependency_display.DependencyDisplayer( trans.app ) + orphan_message = dd.generate_message_for_orphan_tool_dependencies( repository, metadata ) if orphan_message: message += orphan_message status = 'warning' diff -r 3232e68101a46276f07c1f545ccf37c55552f66a -r 5af80141674f5ac3166b6782385c408258606d0a lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -4,18 +4,21 @@ import tarfile import tempfile import urllib -from galaxy.web.base.controller import BaseUIController + from galaxy import util from galaxy import web from galaxy.datatypes import checkers +from galaxy.web.base.controller import BaseUIController + from tool_shed.dependencies import attribute_handlers +from tool_shed.galaxy_install import dependency_display import tool_shed.repository_types.util as rt_util + from tool_shed.util import basic_util from tool_shed.util import commit_util from tool_shed.util import hg_util from tool_shed.util import metadata_util from tool_shed.util import shed_util_common as suc -from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util from tool_shed.util import xml_util @@ -274,6 +277,7 @@ metadata_dict = repository.metadata_revisions[ 0 ].metadata else: metadata_dict = {} + dd = dependency_display.DependencyDisplayer( trans.app ) if str( repository.type ) not in [ rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION ]: change_repository_type_message = rt_util.generate_message_for_repository_type_change( trans.app, @@ -288,15 +292,12 @@ # repository), so warning messages are important because orphans are always valid. The repository # owner must be warned in case they did not intend to define an orphan dependency, but simply # provided incorrect information (tool shed, name owner, changeset_revision) for the definition. - orphan_message = tool_dependency_util.generate_message_for_orphan_tool_dependencies( trans, - repository, - metadata_dict ) + orphan_message = dd.generate_message_for_orphan_tool_dependencies( repository, metadata_dict ) if orphan_message: message += orphan_message status = 'warning' # Handle messaging for invalid tool dependencies. - invalid_tool_dependencies_message = \ - tool_dependency_util.generate_message_for_invalid_tool_dependencies( metadata_dict ) + invalid_tool_dependencies_message = dd.generate_message_for_invalid_tool_dependencies( metadata_dict ) if invalid_tool_dependencies_message: message += invalid_tool_dependencies_message status = 'error' diff -r 3232e68101a46276f07c1f545ccf37c55552f66a -r 5af80141674f5ac3166b6782385c408258606d0a lib/tool_shed/galaxy_install/dependency_display.py --- /dev/null +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -0,0 +1,575 @@ +import logging +import os +import threading + +from tool_shed.util import common_util +from tool_shed.util import container_util +from tool_shed.util import readme_util +from tool_shed.util import shed_util_common as suc +from tool_shed.util import tool_dependency_util + +log = logging.getLogger( __name__ ) + + +class DependencyDisplayer( object ): + + def __init__( self, app ): + self.app = app + + def add_installation_directories_to_tool_dependencies( self, tool_dependencies ): + """ + Determine the path to the installation directory for each of the received + tool dependencies. This path will be displayed within the tool dependencies + container on the select_tool_panel_section or reselect_tool_panel_section + pages when installing or reinstalling repositories that contain tools with + the defined tool dependencies. The list of tool dependencies may be associated + with more than a single repository. + """ + for dependency_key, requirements_dict in tool_dependencies.items(): + if dependency_key in [ 'set_environment' ]: + continue + repository_name = requirements_dict.get( 'repository_name', 'unknown' ) + repository_owner = requirements_dict.get( 'repository_owner', 'unknown' ) + changeset_revision = requirements_dict.get( 'changeset_revision', 'unknown' ) + dependency_name = requirements_dict[ 'name' ] + version = requirements_dict[ 'version' ] + type = requirements_dict[ 'type' ] + if self.app.config.tool_dependency_dir: + root_dir = self.app.config.tool_dependency_dir + else: + root_dir = '<set your tool_dependency_dir in your Galaxy configuration file>' + install_dir = os.path.join( root_dir, + dependency_name, + version, + repository_owner, + repository_name, + changeset_revision ) + requirements_dict[ 'install_dir' ] = install_dir + tool_dependencies[ dependency_key ] = requirements_dict + return tool_dependencies + + def generate_message_for_invalid_tool_dependencies( self, metadata_dict ): + """ + Tool dependency definitions can only be invalid if they include a definition for a complex + repository dependency and the repository dependency definition is invalid. This method + retrieves the error message associated with the invalid tool dependency for display in the + caller. + """ + message = '' + if metadata_dict: + invalid_tool_dependencies = metadata_dict.get( 'invalid_tool_dependencies', None ) + if invalid_tool_dependencies: + for td_key, requirement_dict in invalid_tool_dependencies.items(): + error = requirement_dict.get( 'error', None ) + if error: + message = '%s ' % str( error ) + return message + + def generate_message_for_orphan_tool_dependencies( self, repository, metadata_dict ): + """ + The designation of a ToolDependency into the "orphan" category has evolved over time, + and is significantly restricted since the introduction of the TOOL_DEPENDENCY_DEFINITION + repository type. This designation is still critical, however, in that it handles the + case where a repository contains both tools and a tool_dependencies.xml file, but the + definition in the tool_dependencies.xml file is in no way related to anything defined + by any of the contained tool's requirements tag sets. This is important in that it is + often a result of a typo (e.g., dependency name or version) that differs between the tool + dependency definition within the tool_dependencies.xml file and what is defined in the + tool config's <requirements> tag sets. In these cases, the user should be presented with + a warning message, and this warning message is is in fact displayed if the following + is_orphan attribute is True. This is tricky because in some cases it may be intentional, + and tool dependencies that are categorized as "orphan" are in fact valid. + """ + has_orphan_package_dependencies = False + has_orphan_set_environment_dependencies = False + message = '' + package_orphans_str = '' + set_environment_orphans_str = '' + # Tool dependencies are categorized as orphan only if the repository contains tools. + if metadata_dict: + tools = metadata_dict.get( 'tools', [] ) + invalid_tools = metadata_dict.get( 'invalid_tools', [] ) + tool_dependencies = metadata_dict.get( 'tool_dependencies', {} ) + # The use of the orphan_tool_dependencies category in metadata has been deprecated, + # but we still need to check in case the metadata is out of date. + orphan_tool_dependencies = metadata_dict.get( 'orphan_tool_dependencies', {} ) + # Updating should cause no problems here since a tool dependency cannot be included + # in both dictionaries. + tool_dependencies.update( orphan_tool_dependencies ) + if tool_dependencies and ( tools or invalid_tools ): + for td_key, requirements_dict in tool_dependencies.items(): + if td_key == 'set_environment': + # "set_environment": [{"name": "R_SCRIPT_PATH", "type": "set_environment"}] + for env_requirements_dict in requirements_dict: + name = env_requirements_dict[ 'name' ] + type = env_requirements_dict[ 'type' ] + if self.tool_dependency_is_orphan( type, name, None, tools ): + if not has_orphan_set_environment_dependencies: + has_orphan_set_environment_dependencies = True + set_environment_orphans_str += "* name: %s, type: %s " % \ + ( str( name ), str( type ) ) + else: + # "R/2.15.1": {"name": "R", "readme": "some string", "type": "package", "version": "2.15.1"} + name = requirements_dict[ 'name' ] + type = requirements_dict[ 'type' ] + version = requirements_dict[ 'version' ] + if self.tool_dependency_is_orphan( type, name, version, tools ): + if not has_orphan_package_dependencies: + has_orphan_package_dependencies = True + package_orphans_str += "* name: %s, type: %s, version: %s " % \ + ( str( name ), str( type ), str( version ) ) + if has_orphan_package_dependencies: + message += "The settings for name, version and type from a " + message += "contained tool configuration file's requirement tag does not match " + message += "the information for the following tool dependency definitions in the " + message += "tool_dependencies.xml file, so these tool dependencies have no " + message += "relationship with any tools within this repository. " + message += package_orphans_str + if has_orphan_set_environment_dependencies: + message += "The settings for name and type from a contained tool " + message += "configuration file's requirement tag does not match the information " + message += "for the following tool dependency definitions in the tool_dependencies.xml " + message += "file, so these tool dependencies have no relationship with any tools within " + message += "this repository. " + message += set_environment_orphans_str + return message + + def get_installed_and_missing_tool_dependencies_for_installed_repository( self, repository, all_tool_dependencies ): + """ + Return the lists of installed tool dependencies and missing tool dependencies for a Tool Shed + repository that has been installed into Galaxy. + """ + if all_tool_dependencies: + tool_dependencies = {} + missing_tool_dependencies = {} + for td_key, val in all_tool_dependencies.items(): + if td_key in [ 'set_environment' ]: + for index, td_info_dict in enumerate( val ): + name = td_info_dict[ 'name' ] + version = None + type = td_info_dict[ 'type' ] + tool_dependency = tool_dependency_util.get_tool_dependency_by_name_type_repository( self.app, + repository, + name, + type ) + if tool_dependency: + td_info_dict[ 'repository_id' ] = repository.id + td_info_dict[ 'tool_dependency_id' ] = tool_dependency.id + if tool_dependency.status: + tool_dependency_status = str( tool_dependency.status ) + else: + tool_dependency_status = 'Never installed' + td_info_dict[ 'status' ] = tool_dependency_status + val[ index ] = td_info_dict + if tool_dependency.status == self.app.install_model.ToolDependency.installation_status.INSTALLED: + tool_dependencies[ td_key ] = val + else: + missing_tool_dependencies[ td_key ] = val + else: + name = val[ 'name' ] + version = val[ 'version' ] + type = val[ 'type' ] + tool_dependency = tool_dependency_util.get_tool_dependency_by_name_version_type_repository( self.app, + repository, + name, + version, + type ) + if tool_dependency: + val[ 'repository_id' ] = repository.id + val[ 'tool_dependency_id' ] = tool_dependency.id + if tool_dependency.status: + tool_dependency_status = str( tool_dependency.status ) + else: + tool_dependency_status = 'Never installed' + val[ 'status' ] = tool_dependency_status + if tool_dependency.status == self.app.install_model.ToolDependency.installation_status.INSTALLED: + tool_dependencies[ td_key ] = val + else: + missing_tool_dependencies[ td_key ] = val + else: + tool_dependencies = None + missing_tool_dependencies = None + return tool_dependencies, missing_tool_dependencies + + def merge_containers_dicts_for_new_install( self, containers_dicts ): + """ + When installing one or more tool shed repositories for the first time, the received list of + containers_dicts contains a containers_dict for each repository being installed. Since the + repositories are being installed for the first time, all entries are None except the repository + dependencies and tool dependencies. The entries for missing dependencies are all None since + they have previously been merged into the installed dependencies. This method will merge the + dependencies entries into a single container and return it for display. + """ + new_containers_dict = dict( readme_files=None, + datatypes=None, + missing_repository_dependencies=None, + repository_dependencies=None, + missing_tool_dependencies=None, + tool_dependencies=None, + invalid_tools=None, + valid_tools=None, + workflows=None ) + if containers_dicts: + lock = threading.Lock() + lock.acquire( True ) + try: + repository_dependencies_root_folder = None + tool_dependencies_root_folder = None + # Use a unique folder id (hopefully the following is). + folder_id = 867 + for old_container_dict in containers_dicts: + # Merge repository_dependencies. + old_container_repository_dependencies_root = old_container_dict[ 'repository_dependencies' ] + if old_container_repository_dependencies_root: + if repository_dependencies_root_folder is None: + repository_dependencies_root_folder = container_util.Folder( id=folder_id, + key='root', + label='root', + parent=None ) + folder_id += 1 + repository_dependencies_folder = container_util.Folder( id=folder_id, + key='merged', + label='Repository dependencies', + parent=repository_dependencies_root_folder ) + folder_id += 1 + # The old_container_repository_dependencies_root will be a root folder containing a single sub_folder. + old_container_repository_dependencies_folder = old_container_repository_dependencies_root.folders[ 0 ] + # Change the folder id so it won't confict with others being merged. + old_container_repository_dependencies_folder.id = folder_id + folder_id += 1 + repository_components_tuple = container_util.get_components_from_key( old_container_repository_dependencies_folder.key ) + components_list = suc.extract_components_from_tuple( repository_components_tuple ) + name = components_list[ 1 ] + # Generate the label by retrieving the repository name. + old_container_repository_dependencies_folder.label = str( name ) + repository_dependencies_folder.folders.append( old_container_repository_dependencies_folder ) + # Merge tool_dependencies. + old_container_tool_dependencies_root = old_container_dict[ 'tool_dependencies' ] + if old_container_tool_dependencies_root: + if tool_dependencies_root_folder is None: + tool_dependencies_root_folder = container_util.Folder( id=folder_id, + key='root', + label='root', + parent=None ) + folder_id += 1 + tool_dependencies_folder = container_util.Folder( id=folder_id, + key='merged', + label='Tool dependencies', + parent=tool_dependencies_root_folder ) + folder_id += 1 + else: + td_list = [ td.listify for td in tool_dependencies_folder.tool_dependencies ] + # The old_container_tool_dependencies_root will be a root folder containing a single sub_folder. + old_container_tool_dependencies_folder = old_container_tool_dependencies_root.folders[ 0 ] + for td in old_container_tool_dependencies_folder.tool_dependencies: + if td.listify not in td_list: + tool_dependencies_folder.tool_dependencies.append( td ) + if repository_dependencies_root_folder: + repository_dependencies_root_folder.folders.append( repository_dependencies_folder ) + new_containers_dict[ 'repository_dependencies' ] = repository_dependencies_root_folder + if tool_dependencies_root_folder: + tool_dependencies_root_folder.folders.append( tool_dependencies_folder ) + new_containers_dict[ 'tool_dependencies' ] = tool_dependencies_root_folder + except Exception, e: + log.debug( "Exception in merge_containers_dicts_for_new_install: %s" % str( e ) ) + finally: + lock.release() + return new_containers_dict + + def merge_missing_repository_dependencies_to_installed_container( self, containers_dict ): + """ + Merge the list of missing repository dependencies into the list of installed + repository dependencies. + """ + missing_rd_container_root = containers_dict.get( 'missing_repository_dependencies', None ) + if missing_rd_container_root: + # The missing_rd_container_root will be a root folder containing a single sub_folder. + missing_rd_container = missing_rd_container_root.folders[ 0 ] + installed_rd_container_root = containers_dict.get( 'repository_dependencies', None ) + # The installed_rd_container_root will be a root folder containing a single sub_folder. + if installed_rd_container_root: + installed_rd_container = installed_rd_container_root.folders[ 0 ] + installed_rd_container.label = 'Repository dependencies' + for index, rd in enumerate( missing_rd_container.repository_dependencies ): + # Skip the header row. + if index == 0: + continue + installed_rd_container.repository_dependencies.append( rd ) + installed_rd_container_root.folders = [ installed_rd_container ] + containers_dict[ 'repository_dependencies' ] = installed_rd_container_root + else: + # Change the folder label from 'Missing repository dependencies' to be + # 'Repository dependencies' for display. + root_container = containers_dict[ 'missing_repository_dependencies' ] + for sub_container in root_container.folders: + # There should only be 1 sub-folder. + sub_container.label = 'Repository dependencies' + containers_dict[ 'repository_dependencies' ] = root_container + containers_dict[ 'missing_repository_dependencies' ] = None + return containers_dict + + def merge_missing_tool_dependencies_to_installed_container( self, containers_dict ): + """ + Merge the list of missing tool dependencies into the list of installed tool + dependencies. + """ + missing_td_container_root = containers_dict.get( 'missing_tool_dependencies', None ) + if missing_td_container_root: + # The missing_td_container_root will be a root folder containing a single sub_folder. + missing_td_container = missing_td_container_root.folders[ 0 ] + installed_td_container_root = containers_dict.get( 'tool_dependencies', None ) + # The installed_td_container_root will be a root folder containing a single sub_folder. + if installed_td_container_root: + installed_td_container = installed_td_container_root.folders[ 0 ] + installed_td_container.label = 'Tool dependencies' + for index, td in enumerate( missing_td_container.tool_dependencies ): + # Skip the header row. + if index == 0: + continue + installed_td_container.tool_dependencies.append( td ) + installed_td_container_root.folders = [ installed_td_container ] + containers_dict[ 'tool_dependencies' ] = installed_td_container_root + else: + # Change the folder label from 'Missing tool dependencies' to be + # 'Tool dependencies' for display. + root_container = containers_dict[ 'missing_tool_dependencies' ] + for sub_container in root_container.folders: + # There should only be 1 subfolder. + sub_container.label = 'Tool dependencies' + containers_dict[ 'tool_dependencies' ] = root_container + containers_dict[ 'missing_tool_dependencies' ] = None + return containers_dict + + def populate_containers_dict_for_new_install( self, tool_shed_url, tool_path, readme_files_dict, + installed_repository_dependencies, missing_repository_dependencies, + installed_tool_dependencies, missing_tool_dependencies, + updating=False ): + """ + Return the populated containers for a repository being installed for the first time + or for an installed repository that is being updated and the updates include newly + defined repository (and possibly tool) dependencies. + """ + installed_tool_dependencies, missing_tool_dependencies = \ + self.populate_tool_dependencies_dicts( tool_shed_url=tool_shed_url, + tool_path=tool_path, + repository_installed_tool_dependencies=installed_tool_dependencies, + repository_missing_tool_dependencies=missing_tool_dependencies, + required_repo_info_dicts=None ) + # Most of the repository contents are set to None since we don't yet know what they are. + containers_dict = \ + container_util.build_repository_containers_for_galaxy( app=self.app, + repository=None, + datatypes=None, + invalid_tools=None, + missing_repository_dependencies=missing_repository_dependencies, + missing_tool_dependencies=missing_tool_dependencies, + readme_files_dict=readme_files_dict, + repository_dependencies=installed_repository_dependencies, + tool_dependencies=installed_tool_dependencies, + valid_tools=None, + workflows=None, + valid_data_managers=None, + invalid_data_managers=None, + data_managers_errors=None, + new_install=True, + reinstalling=False ) + if not updating: + # If we installing a new repository and not updaing an installed repository, we can merge + # the missing_repository_dependencies container contents to the installed_repository_dependencies + # container. When updating an installed repository, merging will result in losing newly defined + # dependencies included in the updates. + containers_dict = self.merge_missing_repository_dependencies_to_installed_container( containers_dict ) + # Merge the missing_tool_dependencies container contents to the installed_tool_dependencies container. + containers_dict = self.merge_missing_tool_dependencies_to_installed_container( containers_dict ) + return containers_dict + + def populate_containers_dict_from_repository_metadata( self, tool_shed_url, tool_path, repository, reinstalling=False, + required_repo_info_dicts=None ): + """ + Retrieve necessary information from the received repository's metadata to populate the + containers_dict for display. This method is called only from Galaxy (not the tool shed) + when displaying repository dependencies for installed repositories and when displaying + them for uninstalled repositories that are being reinstalled. + """ + metadata = repository.metadata + if metadata: + # Handle proprietary datatypes. + datatypes = metadata.get( 'datatypes', None ) + # Handle invalid tools. + invalid_tools = metadata.get( 'invalid_tools', None ) + # Handle README files. + if repository.has_readme_files: + if reinstalling or repository.status not in \ + [ self.app.install_model.ToolShedRepository.installation_status.DEACTIVATED, + self.app.install_model.ToolShedRepository.installation_status.INSTALLED ]: + # Since we're reinstalling, we need to send a request to the tool shed to get the README files. + tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed_url ) + params = '?name=%s&owner=%s&changeset_revision=%s' % ( str( repository.name ), + str( repository.owner ), + str( repository.installed_changeset_revision ) ) + url = common_util.url_join( tool_shed_url, + 'repository/get_readme_files%s' % params ) + raw_text = common_util.tool_shed_get( self.app, tool_shed_url, url ) + readme_files_dict = json.from_json_string( raw_text ) + else: + readme_files_dict = readme_util.build_readme_files_dict( self.app, + repository, + repository.changeset_revision, + repository.metadata, tool_path ) + else: + readme_files_dict = None + # Handle repository dependencies. + installed_repository_dependencies, missing_repository_dependencies = \ + self.app.installed_repository_manager.get_installed_and_missing_repository_dependencies( repository ) + # Handle the current repository's tool dependencies. + repository_tool_dependencies = metadata.get( 'tool_dependencies', None ) + # Make sure to display missing tool dependencies as well. + repository_invalid_tool_dependencies = metadata.get( 'invalid_tool_dependencies', None ) + if repository_invalid_tool_dependencies is not None: + if repository_tool_dependencies is None: + repository_tool_dependencies = {} + repository_tool_dependencies.update( repository_invalid_tool_dependencies ) + repository_installed_tool_dependencies, repository_missing_tool_dependencies = \ + self.get_installed_and_missing_tool_dependencies_for_installed_repository( repository, + repository_tool_dependencies ) + if reinstalling: + installed_tool_dependencies, missing_tool_dependencies = \ + self.populate_tool_dependencies_dicts( tool_shed_url, + tool_path, + repository_installed_tool_dependencies, + repository_missing_tool_dependencies, + required_repo_info_dicts ) + else: + installed_tool_dependencies = repository_installed_tool_dependencies + missing_tool_dependencies = repository_missing_tool_dependencies + # Handle valid tools. + valid_tools = metadata.get( 'tools', None ) + # Handle workflows. + workflows = metadata.get( 'workflows', None ) + # Handle Data Managers + valid_data_managers = None + invalid_data_managers = None + data_managers_errors = None + if 'data_manager' in metadata: + valid_data_managers = metadata['data_manager'].get( 'data_managers', None ) + invalid_data_managers = metadata['data_manager'].get( 'invalid_data_managers', None ) + data_managers_errors = metadata['data_manager'].get( 'messages', None ) + containers_dict = \ + container_util.build_repository_containers_for_galaxy( app=self.app, + repository=repository, + datatypes=datatypes, + invalid_tools=invalid_tools, + missing_repository_dependencies=missing_repository_dependencies, + missing_tool_dependencies=missing_tool_dependencies, + readme_files_dict=readme_files_dict, + repository_dependencies=installed_repository_dependencies, + tool_dependencies=installed_tool_dependencies, + valid_tools=valid_tools, + workflows=workflows, + valid_data_managers=valid_data_managers, + invalid_data_managers=invalid_data_managers, + data_managers_errors=data_managers_errors, + new_install=False, + reinstalling=reinstalling ) + else: + containers_dict = dict( datatypes=None, + invalid_tools=None, + readme_files_dict=None, + repository_dependencies=None, + tool_dependencies=None, + valid_tools=None, + workflows=None ) + return containers_dict + + def populate_tool_dependencies_dicts( self, tool_shed_url, tool_path, repository_installed_tool_dependencies, + repository_missing_tool_dependencies, required_repo_info_dicts ): + """ + Return the populated installed_tool_dependencies and missing_tool_dependencies dictionaries + for all repositories defined by entries in the received required_repo_info_dicts. + """ + installed_tool_dependencies = None + missing_tool_dependencies = None + if repository_installed_tool_dependencies is None: + repository_installed_tool_dependencies = {} + else: + # Add the install_dir attribute to the tool_dependencies. + repository_installed_tool_dependencies = \ + self.add_installation_directories_to_tool_dependencies( repository_installed_tool_dependencies ) + if repository_missing_tool_dependencies is None: + repository_missing_tool_dependencies = {} + else: + # Add the install_dir attribute to the tool_dependencies. + repository_missing_tool_dependencies = \ + self.add_installation_directories_to_tool_dependencies( repository_missing_tool_dependencies ) + if required_repo_info_dicts: + # Handle the tool dependencies defined for each of the repository's repository dependencies. + for rid in required_repo_info_dicts: + for name, repo_info_tuple in rid.items(): + description, \ + repository_clone_url, \ + changeset_revision, \ + ctx_rev, \ + repository_owner, \ + repository_dependencies, \ + tool_dependencies = \ + suc.get_repo_info_tuple_contents( repo_info_tuple ) + if tool_dependencies: + # Add the install_dir attribute to the tool_dependencies. + tool_dependencies = self.add_installation_directories_to_tool_dependencies( tool_dependencies ) + # The required_repository may have been installed with a different changeset revision. + required_repository, installed_changeset_revision = \ + suc.repository_was_previously_installed( self.app, + tool_shed_url, + name, + repo_info_tuple, + from_tip=False ) + if required_repository: + required_repository_installed_tool_dependencies, required_repository_missing_tool_dependencies = \ + self.get_installed_and_missing_tool_dependencies_for_installed_repository( required_repository, + tool_dependencies ) + if required_repository_installed_tool_dependencies: + # Add the install_dir attribute to the tool_dependencies. + required_repository_installed_tool_dependencies = \ + self.add_installation_directories_to_tool_dependencies( required_repository_installed_tool_dependencies ) + for td_key, td_dict in required_repository_installed_tool_dependencies.items(): + if td_key not in repository_installed_tool_dependencies: + repository_installed_tool_dependencies[ td_key ] = td_dict + if required_repository_missing_tool_dependencies: + # Add the install_dir attribute to the tool_dependencies. + required_repository_missing_tool_dependencies = \ + self.add_installation_directories_to_tool_dependencies( required_repository_missing_tool_dependencies ) + for td_key, td_dict in required_repository_missing_tool_dependencies.items(): + if td_key not in repository_missing_tool_dependencies: + repository_missing_tool_dependencies[ td_key ] = td_dict + if repository_installed_tool_dependencies: + installed_tool_dependencies = repository_installed_tool_dependencies + if repository_missing_tool_dependencies: + missing_tool_dependencies = repository_missing_tool_dependencies + return installed_tool_dependencies, missing_tool_dependencies + + def tool_dependency_is_orphan( self, type, name, version, tools ): + """ + Determine if the combination of the received type, name and version is defined in the <requirement> + tag for at least one tool in the received list of tools. If not, the tool dependency defined by the + combination is considered an orphan in its repository in the tool shed. + """ + if type == 'package': + if name and version: + for tool_dict in tools: + requirements = tool_dict.get( 'requirements', [] ) + for requirement_dict in requirements: + req_name = requirement_dict.get( 'name', None ) + req_version = requirement_dict.get( 'version', None ) + req_type = requirement_dict.get( 'type', None ) + if req_name == name and req_version == version and req_type == type: + return False + elif type == 'set_environment': + if name: + for tool_dict in tools: + requirements = tool_dict.get( 'requirements', [] ) + for requirement_dict in requirements: + req_name = requirement_dict.get( 'name', None ) + req_type = requirement_dict.get( 'type', None ) + if req_name == name and req_type == type: + return False + return True diff -r 3232e68101a46276f07c1f545ccf37c55552f66a -r 5af80141674f5ac3166b6782385c408258606d0a lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -3,7 +3,6 @@ import os import sys import tempfile -import threading import traceback from galaxy import exceptions @@ -19,7 +18,6 @@ from tool_shed.util import basic_util from tool_shed.util import common_util -from tool_shed.util import container_util from tool_shed.util import data_manager_util from tool_shed.util import datatype_util from tool_shed.util import encoding_util @@ -225,7 +223,8 @@ installed_packages.append( tool_dependency ) if self.app.config.manage_dependency_relationships: # Add the tool_dependency to the in-memory dictionaries in the installed_repository_manager. - self.app.installed_repository_manager.handle_tool_dependency_install( tool_shed_repository, tool_dependency ) + self.app.installed_repository_manager.handle_tool_dependency_install( tool_shed_repository, + tool_dependency ) return installed_packages def install_via_fabric( self, tool_shed_repository, tool_dependency, install_dir, package_name=None, custom_fabfile_path=None, @@ -371,8 +370,8 @@ # Delete contents of installation directory if attempt at binary installation failed. installation_directory_contents = os.listdir( installation_directory ) if installation_directory_contents: - removed, error_message = tool_dependency_util.remove_tool_dependency( self.app, - tool_dependency ) + removed, error_message = \ + tool_dependency_util.remove_tool_dependency( self.app, tool_dependency ) if removed: can_install_from_source = True else: @@ -894,7 +893,8 @@ tool_shed_repository, install_model.ToolShedRepository.installation_status.INSTALLED ) if self.app.config.manage_dependency_relationships: - # Add the installed repository and any tool dependencies to the in-memory dictionaries in the installed_repository_manager. + # Add the installed repository and any tool dependencies to the in-memory dictionaries + # in the installed_repository_manager. self.app.installed_repository_manager.handle_repository_install( tool_shed_repository ) else: # An error occurred while cloning the repository, so reset everything necessary to enable another attempt. @@ -906,147 +906,6 @@ uninstalled=False, remove_from_disk=True ) - def merge_containers_dicts_for_new_install( self, containers_dicts ): - """ - When installing one or more tool shed repositories for the first time, the received list of - containers_dicts contains a containers_dict for each repository being installed. Since the - repositories are being installed for the first time, all entries are None except the repository - dependencies and tool dependencies. The entries for missing dependencies are all None since - they have previously been merged into the installed dependencies. This method will merge the - dependencies entries into a single container and return it for display. - """ - new_containers_dict = dict( readme_files=None, - datatypes=None, - missing_repository_dependencies=None, - repository_dependencies=None, - missing_tool_dependencies=None, - tool_dependencies=None, - invalid_tools=None, - valid_tools=None, - workflows=None ) - if containers_dicts: - lock = threading.Lock() - lock.acquire( True ) - try: - repository_dependencies_root_folder = None - tool_dependencies_root_folder = None - # Use a unique folder id (hopefully the following is). - folder_id = 867 - for old_container_dict in containers_dicts: - # Merge repository_dependencies. - old_container_repository_dependencies_root = old_container_dict[ 'repository_dependencies' ] - if old_container_repository_dependencies_root: - if repository_dependencies_root_folder is None: - repository_dependencies_root_folder = container_util.Folder( id=folder_id, - key='root', - label='root', - parent=None ) - folder_id += 1 - repository_dependencies_folder = container_util.Folder( id=folder_id, - key='merged', - label='Repository dependencies', - parent=repository_dependencies_root_folder ) - folder_id += 1 - # The old_container_repository_dependencies_root will be a root folder containing a single sub_folder. - old_container_repository_dependencies_folder = old_container_repository_dependencies_root.folders[ 0 ] - # Change the folder id so it won't confict with others being merged. - old_container_repository_dependencies_folder.id = folder_id - folder_id += 1 - repository_components_tuple = container_util.get_components_from_key( old_container_repository_dependencies_folder.key ) - components_list = suc.extract_components_from_tuple( repository_components_tuple ) - name = components_list[ 1 ] - # Generate the label by retrieving the repository name. - old_container_repository_dependencies_folder.label = str( name ) - repository_dependencies_folder.folders.append( old_container_repository_dependencies_folder ) - # Merge tool_dependencies. - old_container_tool_dependencies_root = old_container_dict[ 'tool_dependencies' ] - if old_container_tool_dependencies_root: - if tool_dependencies_root_folder is None: - tool_dependencies_root_folder = container_util.Folder( id=folder_id, - key='root', - label='root', - parent=None ) - folder_id += 1 - tool_dependencies_folder = container_util.Folder( id=folder_id, - key='merged', - label='Tool dependencies', - parent=tool_dependencies_root_folder ) - folder_id += 1 - else: - td_list = [ td.listify for td in tool_dependencies_folder.tool_dependencies ] - # The old_container_tool_dependencies_root will be a root folder containing a single sub_folder. - old_container_tool_dependencies_folder = old_container_tool_dependencies_root.folders[ 0 ] - for td in old_container_tool_dependencies_folder.tool_dependencies: - if td.listify not in td_list: - tool_dependencies_folder.tool_dependencies.append( td ) - if repository_dependencies_root_folder: - repository_dependencies_root_folder.folders.append( repository_dependencies_folder ) - new_containers_dict[ 'repository_dependencies' ] = repository_dependencies_root_folder - if tool_dependencies_root_folder: - tool_dependencies_root_folder.folders.append( tool_dependencies_folder ) - new_containers_dict[ 'tool_dependencies' ] = tool_dependencies_root_folder - except Exception, e: - log.debug( "Exception in merge_containers_dicts_for_new_install: %s" % str( e ) ) - finally: - lock.release() - return new_containers_dict - - def merge_missing_repository_dependencies_to_installed_container( self, containers_dict ): - """Merge the list of missing repository dependencies into the list of installed repository dependencies.""" - missing_rd_container_root = containers_dict.get( 'missing_repository_dependencies', None ) - if missing_rd_container_root: - # The missing_rd_container_root will be a root folder containing a single sub_folder. - missing_rd_container = missing_rd_container_root.folders[ 0 ] - installed_rd_container_root = containers_dict.get( 'repository_dependencies', None ) - # The installed_rd_container_root will be a root folder containing a single sub_folder. - if installed_rd_container_root: - installed_rd_container = installed_rd_container_root.folders[ 0 ] - installed_rd_container.label = 'Repository dependencies' - for index, rd in enumerate( missing_rd_container.repository_dependencies ): - # Skip the header row. - if index == 0: - continue - installed_rd_container.repository_dependencies.append( rd ) - installed_rd_container_root.folders = [ installed_rd_container ] - containers_dict[ 'repository_dependencies' ] = installed_rd_container_root - else: - # Change the folder label from 'Missing repository dependencies' to be 'Repository dependencies' for display. - root_container = containers_dict[ 'missing_repository_dependencies' ] - for sub_container in root_container.folders: - # There should only be 1 sub-folder. - sub_container.label = 'Repository dependencies' - containers_dict[ 'repository_dependencies' ] = root_container - containers_dict[ 'missing_repository_dependencies' ] = None - return containers_dict - - def merge_missing_tool_dependencies_to_installed_container( self, containers_dict ): - """ Merge the list of missing tool dependencies into the list of installed tool dependencies.""" - missing_td_container_root = containers_dict.get( 'missing_tool_dependencies', None ) - if missing_td_container_root: - # The missing_td_container_root will be a root folder containing a single sub_folder. - missing_td_container = missing_td_container_root.folders[ 0 ] - installed_td_container_root = containers_dict.get( 'tool_dependencies', None ) - # The installed_td_container_root will be a root folder containing a single sub_folder. - if installed_td_container_root: - installed_td_container = installed_td_container_root.folders[ 0 ] - installed_td_container.label = 'Tool dependencies' - for index, td in enumerate( missing_td_container.tool_dependencies ): - # Skip the header row. - if index == 0: - continue - installed_td_container.tool_dependencies.append( td ) - installed_td_container_root.folders = [ installed_td_container ] - containers_dict[ 'tool_dependencies' ] = installed_td_container_root - else: - # Change the folder label from 'Missing tool dependencies' to be 'Tool dependencies' for display. - root_container = containers_dict[ 'missing_tool_dependencies' ] - for sub_container in root_container.folders: - # There should only be 1 subfolder. - sub_container.label = 'Tool dependencies' - containers_dict[ 'tool_dependencies' ] = root_container - containers_dict[ 'missing_tool_dependencies' ] = None - return containers_dict - def order_components_for_installation( self, tsr_ids, repo_info_dicts, tool_panel_section_keys ): """ Some repositories may have repository dependencies that are required to be installed @@ -1096,45 +955,3 @@ ordered_repo_info_dicts.append( repo_info_dict ) ordered_tool_panel_section_keys.append( tool_panel_section_key ) return ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys - - def populate_containers_dict_for_new_install( self, tool_shed_url, tool_path, readme_files_dict, installed_repository_dependencies, - missing_repository_dependencies, installed_tool_dependencies, missing_tool_dependencies, - updating=False ): - """ - Return the populated containers for a repository being installed for the first time or for an installed repository - that is being updated and the updates include newly defined repository (and possibly tool) dependencies. - """ - installed_tool_dependencies, missing_tool_dependencies = \ - tool_dependency_util.populate_tool_dependencies_dicts( app=self.app, - tool_shed_url=tool_shed_url, - tool_path=tool_path, - repository_installed_tool_dependencies=installed_tool_dependencies, - repository_missing_tool_dependencies=missing_tool_dependencies, - required_repo_info_dicts=None ) - # Most of the repository contents are set to None since we don't yet know what they are. - containers_dict = \ - container_util.build_repository_containers_for_galaxy( app=self.app, - repository=None, - datatypes=None, - invalid_tools=None, - missing_repository_dependencies=missing_repository_dependencies, - missing_tool_dependencies=missing_tool_dependencies, - readme_files_dict=readme_files_dict, - repository_dependencies=installed_repository_dependencies, - tool_dependencies=installed_tool_dependencies, - valid_tools=None, - workflows=None, - valid_data_managers=None, - invalid_data_managers=None, - data_managers_errors=None, - new_install=True, - reinstalling=False ) - if not updating: - # If we installing a new repository and not updaing an installed repository, we can merge - # the missing_repository_dependencies container contents to the installed_repository_dependencies - # container. When updating an installed repository, merging will result in losing newly defined - # dependencies included in the updates. - containers_dict = self.merge_missing_repository_dependencies_to_installed_container( containers_dict ) - # Merge the missing_tool_dependencies container contents to the installed_tool_dependencies container. - containers_dict = self.merge_missing_tool_dependencies_to_installed_container( containers_dict ) - return containers_dict diff -r 3232e68101a46276f07c1f545ccf37c55552f66a -r 5af80141674f5ac3166b6782385c408258606d0a lib/tool_shed/galaxy_install/installed_repository_manager.py --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -395,7 +395,13 @@ installed_repository_dependencies = {} missing_rd_tups = [] installed_rd_tups = [] - description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \ + description, \ + repository_clone_url, \ + changeset_revision, \ + ctx_rev, \ + repository_owner, \ + repository_dependencies, \ + tool_dependencies = \ suc.get_repo_info_tuple_contents( repo_info_tuple ) if repository_dependencies: description = repository_dependencies[ 'description' ] @@ -614,6 +620,30 @@ type = str( tool_dependency.type ) return ( tool_dependency.tool_shed_repository_id, str( tool_dependency.name ), str( tool_dependency.version ), type ) + def handle_existing_tool_dependencies_that_changed_in_update( self, repository, original_dependency_dict, + new_dependency_dict ): + """ + This method is called when a Galaxy admin is getting updates for an installed tool shed + repository in order to cover the case where an existing tool dependency was changed (e.g., + the version of the dependency was changed) but the tool version for which it is a dependency + was not changed. In this case, we only want to determine if any of the dependency information + defined in original_dependency_dict was changed in new_dependency_dict. We don't care if new + dependencies were added in new_dependency_dict since they will just be treated as missing + dependencies for the tool. + """ + updated_tool_dependency_names = [] + deleted_tool_dependency_names = [] + for original_dependency_key, original_dependency_val_dict in original_dependency_dict.items(): + if original_dependency_key not in new_dependency_dict: + updated_tool_dependency = self.update_existing_tool_dependency( repository, + original_dependency_val_dict, + new_dependency_dict ) + if updated_tool_dependency: + updated_tool_dependency_names.append( updated_tool_dependency.name ) + else: + deleted_tool_dependency_names.append( original_dependency_val_dict[ 'name' ] ) + return updated_tool_dependency_names, deleted_tool_dependency_names + def handle_repository_install( self, repository ): """Load the dependency relationships for a repository that was just installed or reinstalled.""" # Populate self.repository_dependencies_of_installed_repositories. @@ -924,3 +954,68 @@ repository_dependency.changeset_revision == changeset_revision ): return True return False + + def update_existing_tool_dependency( self, repository, original_dependency_dict, new_dependencies_dict ): + """ + Update an exsiting tool dependency whose definition was updated in a change set + pulled by a Galaxy administrator when getting updates to an installed tool shed + repository. The original_dependency_dict is a single tool dependency definition, + an example of which is:: + + {"name": "bwa", + "readme": "\\nCompiling BWA requires zlib and libpthread to be present on your system.\\n ", + "type": "package", + "version": "0.6.2"} + + The new_dependencies_dict is the dictionary generated by the metadata_util.generate_tool_dependency_metadata method. + """ + new_tool_dependency = None + original_name = original_dependency_dict[ 'name' ] + original_type = original_dependency_dict[ 'type' ] + original_version = original_dependency_dict[ 'version' ] + # Locate the appropriate tool_dependency associated with the repository. + tool_dependency = None + for tool_dependency in repository.tool_dependencies: + if tool_dependency.name == original_name and \ + tool_dependency.type == original_type and \ + tool_dependency.version == original_version: + break + if tool_dependency and tool_dependency.can_update: + dependency_install_dir = tool_dependency.installation_directory( self.app ) + removed_from_disk, error_message = \ + tool_dependency_util.remove_tool_dependency_installation_directory( dependency_install_dir ) + if removed_from_disk: + context = self.app.install_model.context + new_dependency_name = None + new_dependency_type = None + new_dependency_version = None + for new_dependency_key, new_dependency_val_dict in new_dependencies_dict.items(): + # Match on name only, hopefully this will be enough! + if original_name == new_dependency_val_dict[ 'name' ]: + new_dependency_name = new_dependency_val_dict[ 'name' ] + new_dependency_type = new_dependency_val_dict[ 'type' ] + new_dependency_version = new_dependency_val_dict[ 'version' ] + break + if new_dependency_name and new_dependency_type and new_dependency_version: + # Update all attributes of the tool_dependency record in the database. + log.debug( "Updating version %s of tool dependency %s %s to have new version %s and type %s." % \ + ( str( tool_dependency.version ), + str( tool_dependency.type ), + str( tool_dependency.name ), + str( new_dependency_version ), + str( new_dependency_type ) ) ) + tool_dependency.type = new_dependency_type + tool_dependency.version = new_dependency_version + tool_dependency.status = self.app.install_model.ToolDependency.installation_status.UNINSTALLED + tool_dependency.error_message = None + context.add( tool_dependency ) + context.flush() + new_tool_dependency = tool_dependency + else: + # We have no new tool dependency definition based on a matching dependency name, so remove + # the existing tool dependency record from the database. + log.debug( "Deleting version %s of tool dependency %s %s from the database since it is no longer defined." % \ + ( str( tool_dependency.version ), str( tool_dependency.type ), str( tool_dependency.name ) ) ) + context.delete( tool_dependency ) + context.flush() + return new_tool_dependency diff -r 3232e68101a46276f07c1f545ccf37c55552f66a -r 5af80141674f5ac3166b6782385c408258606d0a lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py @@ -25,7 +25,89 @@ raise "Unimplemented Method" -class Install( RecipeTag ): +class SyncDatabase( object ): + + def sync_database_with_file_system( self, app, tool_shed_repository, tool_dependency_name, tool_dependency_version, + tool_dependency_install_dir, tool_dependency_type='package' ): + """ + The installation directory defined by the received tool_dependency_install_dir exists, so check for + the presence of INSTALLATION_LOG. If the files exists, we'll assume the tool dependency is installed, + but not necessarily successfully (it could be in an error state on disk. However, we can justifiably + assume here that no matter the state, an associated database record will exist. + """ + # This method should be reached very rarely. It implies that either the Galaxy environment + # became corrupted (i.e., the database records for installed tool dependencies is not synchronized + # with tool dependencies on disk) or the Tool Shed's install and test framework is running. The Tool + # Shed's install and test framework installs repositories in 2 stages, those of type tool_dependency_definition + # followed by those containing valid tools and tool functional test components. + log.debug( "Synchronizing the database with the file system..." ) + try: + log.debug( "The value of app.config.running_functional_tests is: %s" % \ + str( app.config.running_functional_tests ) ) + except: + pass + sa_session = app.install_model.context + can_install_tool_dependency = False + tool_dependency = get_tool_dependency_by_name_version_type_repository( app, + tool_shed_repository, + tool_dependency_name, + tool_dependency_version, + tool_dependency_type ) + if tool_dependency.status == app.install_model.ToolDependency.installation_status.INSTALLING: + # The tool dependency is in an Installing state, so we don't want to do anything to it. If the tool + # dependency is being installed by someone else, we don't want to interfere with that. This assumes + # the installation by "someone else" is not hung in an Installing state, which is a weakness if that + # "someone else" never repaired it. + log.debug( 'Skipping installation of tool dependency %s version %s because it has a status of %s' % \ + ( str( tool_dependency.name ), str( tool_dependency.version ), str( tool_dependency.status ) ) ) + else: + # We have a pre-existing installation directory on the file system, but our associated database record is + # in a state that allowed us to arrive here. At this point, we'll inspect the installation directory to + # see if we have a "proper installation" and if so, synchronize the database record rather than reinstalling + # the dependency if we're "running_functional_tests". If we're not "running_functional_tests, we'll set + # the tool dependency's installation status to ERROR. + tool_dependency_installation_directory_contents = os.listdir( tool_dependency_install_dir ) + if basic_util.INSTALLATION_LOG in tool_dependency_installation_directory_contents: + # Since this tool dependency's installation directory contains an installation log, we consider it to be + # installed. In some cases the record may be missing from the database due to some activity outside of + # the control of the Tool Shed. Since a new record was created for it and we don't know the state of the + # files on disk, we will set it to an error state (unless we are running Tool Shed functional tests - see + # below). + log.debug( 'Skipping installation of tool dependency %s version %s because it is installed in %s' % \ + ( str( tool_dependency.name ), str( tool_dependency.version ), str( tool_dependency_install_dir ) ) ) + if app.config.running_functional_tests: + # If we are running functional tests, the state will be set to Installed because previously compiled + # tool dependencies are not deleted by default, from the "install and test" framework.. + tool_dependency.status = app.install_model.ToolDependency.installation_status.INSTALLED + else: + error_message = 'The installation directory for this tool dependency had contents but the database had no record. ' + error_message += 'The installation log may show this tool dependency to be correctly installed, but due to the ' + error_message += 'missing database record it is now being set to Error.' + tool_dependency.status = app.install_model.ToolDependency.installation_status.ERROR + tool_dependency.error_message = error_message + else: + error_message = '\nInstallation path %s for tool dependency %s version %s exists, but the expected file %s' % \ + ( str( tool_dependency_install_dir ), + str( tool_dependency_name ), + str( tool_dependency_version ), + str( basic_util.INSTALLATION_LOG ) ) + error_message += ' is missing. This indicates an installation error so the tool dependency is being' + error_message += ' prepared for re-installation.' + print error_message + tool_dependency.status = app.install_model.ToolDependency.installation_status.NEVER_INSTALLED + basic_util.remove_dir( tool_dependency_install_dir ) + can_install_tool_dependency = True + sa_session.add( tool_dependency ) + sa_session.flush() + try: + log.debug( "Returning from sync_database_with_file_system with tool_dependency %s, can_install_tool_dependency %s." % \ + ( str( tool_dependency.name ), str( can_install_tool_dependency ) ) ) + except Exception, e: + log.debug( str( e ) ) + return tool_dependency, can_install_tool_dependency + + +class Install( RecipeTag, SyncDatabase ): def __init__( self, app ): self.app = app @@ -57,13 +139,12 @@ proceed_with_install = True else: # Notice that we'll throw away the following tool_dependency if it can be installed. - tool_dependency, proceed_with_install = \ - tool_dependency_util.sync_database_with_file_system( self.app, - tool_shed_repository, - package_name, - package_version, - install_dir, - tool_dependency_type='package' ) + tool_dependency, proceed_with_install = self.sync_database_with_file_system( self.app, + tool_shed_repository, + package_name, + package_version, + install_dir, + tool_dependency_type='package' ) if not proceed_with_install: log.debug( "Tool dependency %s version %s cannot be installed (it was probably previously installed), so returning it." % \ ( str( tool_dependency.name ), str( tool_dependency.version ) ) ) @@ -156,7 +237,7 @@ return tool_dependency, proceed_with_install, action_elem_tuples -class Repository( RecipeTag ): +class Repository( RecipeTag, SyncDatabase ): def __init__( self, app ): self.app = app @@ -190,7 +271,8 @@ return None def create_tool_dependency_with_initialized_env_sh_file( self, dependent_install_dir, tool_shed_repository, - required_repository, package_name, package_version, tool_dependencies_config ): + required_repository, package_name, package_version, + tool_dependencies_config ): """ Create or get a tool_dependency record that is defined by the received package_name and package_version. An env.sh file will be created for the tool_dependency in the received dependent_install_dir. @@ -238,10 +320,9 @@ # the path defined by required_tool_dependency_env_file_path. It doesn't matter if the required env.sh # file currently exists.. required_tool_dependency_env_file_path = \ - tool_dependency_util.get_required_repository_package_env_sh_path( self.app, - package_name, - package_version, - required_repository ) + self.get_required_repository_package_env_sh_path( package_name, + package_version, + required_repository ) env_file_builder = EnvFileBuilder( tool_dependency.installation_directory( self.app ) ) env_file_builder.append_line( action="source", value=required_tool_dependency_env_file_path ) return_code = env_file_builder.return_code @@ -269,6 +350,19 @@ tool_dependencies.append( tool_dependency ) return tool_dependencies + def get_required_repository_package_env_sh_path( self, package_name, package_version, required_repository ): + """Return path to env.sh file in required repository if the required repository has been installed.""" + env_sh_file_dir = \ + tool_dependency_util.get_tool_dependency_install_dir( app=self.app, + repository_name=required_repository.name, + repository_owner=required_repository.owner, + repository_changeset_revision=required_repository.installed_changeset_revision, + tool_dependency_type='package', + tool_dependency_name=package_name, + tool_dependency_version=package_version ) + env_sh_file_path = os.path.join( env_sh_file_dir, 'env.sh' ) + return env_sh_file_path + def get_tool_shed_repository_by_tool_shed_name_owner_changeset_revision( self, tool_shed_url, name, owner, changeset_revision ): sa_session = self.app.install_model.context # The protocol is not stored, but the port is if it exists. @@ -352,13 +446,12 @@ can_install_tool_dependency = True else: # Notice that we'll throw away the following tool_dependency if it can be installed. - tool_dependency, can_install_tool_dependency = \ - tool_dependency_util.sync_database_with_file_system( self.app, - tool_shed_repository, - package_name, - package_version, - dependent_install_dir, - tool_dependency_type='package' ) + tool_dependency, can_install_tool_dependency = self.sync_database_with_file_system( self.app, + tool_shed_repository, + package_name, + package_version, + dependent_install_dir, + tool_dependency_type='package' ) if not can_install_tool_dependency: log.debug( "Tool dependency %s version %s cannot be installed (it was probably previously installed), " % \ ( str( tool_dependency.name, str( tool_dependency.version ) ) ) ) diff -r 3232e68101a46276f07c1f545ccf37c55552f66a -r 5af80141674f5ac3166b6782385c408258606d0a lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -3,7 +3,6 @@ import tempfile from galaxy import util -from galaxy import web from galaxy.datatypes import checkers from galaxy.model.orm import and_ from galaxy.tools.data_manager.manager import DataManager @@ -11,17 +10,17 @@ from galaxy.util import json from galaxy.web import url_for -import tool_shed.util.shed_util_common as suc from tool_shed.repository_types.metadata import TipOnly +import tool_shed.repository_types.util as rt_util + from tool_shed.util import basic_util from tool_shed.util import common_util -from tool_shed.util import container_util from tool_shed.util import hg_util from tool_shed.util import readme_util +from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util from tool_shed.util import xml_util -import tool_shed.repository_types.util as rt_util log = logging.getLogger( __name__ ) @@ -1013,7 +1012,11 @@ description = root.get( 'description' ) for elem in root: if elem.tag == 'package': - valid_tool_dependencies_dict, invalid_tool_dependencies_dict, repository_dependency_tup, repository_dependency_is_valid, message = \ + valid_tool_dependencies_dict, \ + invalid_tool_dependencies_dict, \ + repository_dependency_tup, \ + repository_dependency_is_valid, \ + message = \ generate_package_dependency_metadata( app, elem, valid_tool_dependencies_dict, invalid_tool_dependencies_dict ) if repository_dependency_is_valid: if repository_dependency_tup and repository_dependency_tup not in valid_repository_dependency_tups: @@ -1024,7 +1027,13 @@ # We have an invalid complex repository dependency, so mark the tool dependency as invalid. tool_dependency_is_valid = False # Append the error message to the invalid repository dependency tuple. - toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = repository_dependency_tup + toolshed, \ + name, \ + owner, \ + changeset_revision, \ + prior_installation_required, \ + only_if_compiling_contained_td \ + = repository_dependency_tup repository_dependency_tup = \ ( toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, message ) invalid_repository_dependency_tups.append( repository_dependency_tup ) @@ -1033,9 +1042,13 @@ valid_tool_dependencies_dict = generate_environment_dependency_metadata( elem, valid_tool_dependencies_dict ) if valid_tool_dependencies_dict: if original_valid_tool_dependencies_dict: - # We're generating metadata on an update pulled to a tool shed repository installed into a Galaxy instance, so handle changes to - # tool dependencies appropriately. - handle_existing_tool_dependencies_that_changed_in_update( app, repository, original_valid_tool_dependencies_dict, valid_tool_dependencies_dict ) + # We're generating metadata on an update pulled to a tool shed repository installed + # into a Galaxy instance, so handle changes to tool dependencies appropriately. + irm = app.installed_repository_manager + updated_tool_dependency_names, deleted_tool_dependency_names = \ + irm.handle_existing_tool_dependencies_that_changed_in_update( repository, + original_valid_tool_dependencies_dict, + valid_tool_dependencies_dict ) metadata_dict[ 'tool_dependencies' ] = valid_tool_dependencies_dict if invalid_tool_dependencies_dict: metadata_dict[ 'invalid_tool_dependencies' ] = invalid_tool_dependencies_dict @@ -1296,25 +1309,6 @@ sample_file_metadata_paths.append( relative_path_to_sample_file ) return sample_file_metadata_paths, sample_file_copy_paths -def handle_existing_tool_dependencies_that_changed_in_update( app, repository, original_dependency_dict, new_dependency_dict ): - """ - This method is called when a Galaxy admin is getting updates for an installed tool shed repository in order to cover the case where an - existing tool dependency was changed (e.g., the version of the dependency was changed) but the tool version for which it is a dependency - was not changed. In this case, we only want to determine if any of the dependency information defined in original_dependency_dict was - changed in new_dependency_dict. We don't care if new dependencies were added in new_dependency_dict since they will just be treated as - missing dependencies for the tool. - """ - updated_tool_dependency_names = [] - deleted_tool_dependency_names = [] - for original_dependency_key, original_dependency_val_dict in original_dependency_dict.items(): - if original_dependency_key not in new_dependency_dict: - updated_tool_dependency = update_existing_tool_dependency( app, repository, original_dependency_val_dict, new_dependency_dict ) - if updated_tool_dependency: - updated_tool_dependency_names.append( updated_tool_dependency.name ) - else: - deleted_tool_dependency_names.append( original_dependency_val_dict[ 'name' ] ) - return updated_tool_dependency_names, deleted_tool_dependency_names - def handle_repository_elem( app, repository_elem, only_if_compiling_contained_td=False, updating_installed_repository=False ): """ Process the received repository_elem which is a <repository> tag either from a repository_dependencies.xml @@ -1737,104 +1731,6 @@ # The received metadata_dict includes no metadata for workflows, so a new repository_metadata table record is not needed. return False -def populate_containers_dict_from_repository_metadata( app, tool_shed_url, tool_path, repository, - reinstalling=False, required_repo_info_dicts=None ): - """ - Retrieve necessary information from the received repository's metadata to populate the - containers_dict for display. This method is called only from Galaxy (not the tool shed) - when displaying repository dependencies for installed repositories and when displaying - them for uninstalled repositories that are being reinstalled. - """ - metadata = repository.metadata - if metadata: - # Handle proprietary datatypes. - datatypes = metadata.get( 'datatypes', None ) - # Handle invalid tools. - invalid_tools = metadata.get( 'invalid_tools', None ) - # Handle README files. - if repository.has_readme_files: - if reinstalling or repository.status not in [ app.install_model.ToolShedRepository.installation_status.DEACTIVATED, - app.install_model.ToolShedRepository.installation_status.INSTALLED ]: - # Since we're reinstalling, we need to send a request to the tool shed to get the README files. - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) - params = '?name=%s&owner=%s&changeset_revision=%s' % ( str( repository.name ), - str( repository.owner ), - str( repository.installed_changeset_revision ) ) - url = common_util.url_join( tool_shed_url, - 'repository/get_readme_files%s' % params ) - raw_text = common_util.tool_shed_get( app, tool_shed_url, url ) - readme_files_dict = json.from_json_string( raw_text ) - else: - readme_files_dict = readme_util.build_readme_files_dict( app, - repository, - repository.changeset_revision, - repository.metadata, tool_path ) - else: - readme_files_dict = None - # Handle repository dependencies. - installed_repository_dependencies, missing_repository_dependencies = \ - app.installed_repository_manager.get_installed_and_missing_repository_dependencies( repository ) - # Handle the current repository's tool dependencies. - repository_tool_dependencies = metadata.get( 'tool_dependencies', None ) - # Make sure to display missing tool dependencies as well. - repository_invalid_tool_dependencies = metadata.get( 'invalid_tool_dependencies', None ) - if repository_invalid_tool_dependencies is not None: - if repository_tool_dependencies is None: - repository_tool_dependencies = {} - repository_tool_dependencies.update( repository_invalid_tool_dependencies ) - repository_installed_tool_dependencies, repository_missing_tool_dependencies = \ - tool_dependency_util.get_installed_and_missing_tool_dependencies_for_installed_repository( app, - repository, - repository_tool_dependencies ) - if reinstalling: - installed_tool_dependencies, missing_tool_dependencies = \ - tool_dependency_util.populate_tool_dependencies_dicts( app, - tool_shed_url, - tool_path, - repository_installed_tool_dependencies, - repository_missing_tool_dependencies, - required_repo_info_dicts ) - else: - installed_tool_dependencies = repository_installed_tool_dependencies - missing_tool_dependencies = repository_missing_tool_dependencies - # Handle valid tools. - valid_tools = metadata.get( 'tools', None ) - # Handle workflows. - workflows = metadata.get( 'workflows', None ) - # Handle Data Managers - valid_data_managers = None - invalid_data_managers = None - data_managers_errors = None - if 'data_manager' in metadata: - valid_data_managers = metadata['data_manager'].get( 'data_managers', None ) - invalid_data_managers = metadata['data_manager'].get( 'invalid_data_managers', None ) - data_managers_errors = metadata['data_manager'].get( 'messages', None ) - containers_dict = container_util.build_repository_containers_for_galaxy( app=app, - repository=repository, - datatypes=datatypes, - invalid_tools=invalid_tools, - missing_repository_dependencies=missing_repository_dependencies, - missing_tool_dependencies=missing_tool_dependencies, - readme_files_dict=readme_files_dict, - repository_dependencies=installed_repository_dependencies, - tool_dependencies=installed_tool_dependencies, - valid_tools=valid_tools, - workflows=workflows, - valid_data_managers=valid_data_managers, - invalid_data_managers=invalid_data_managers, - data_managers_errors=data_managers_errors, - new_install=False, - reinstalling=reinstalling ) - else: - containers_dict = dict( datatypes=None, - invalid_tools=None, - readme_files_dict=None, - repository_dependencies=None, - tool_dependencies=None, - valid_tools=None, - workflows=None ) - return containers_dict - def reset_all_metadata_on_installed_repository( app, id ): """Reset all metadata on a single tool shed repository installed into a Galaxy instance.""" invalid_file_tups = [] @@ -2201,68 +2097,6 @@ error_message, status = set_repository_metadata( app, host, user, repository, content_alert_str=content_alert_str, **kwd ) return status, error_message -def update_existing_tool_dependency( app, repository, original_dependency_dict, new_dependencies_dict ): - """ - Update an exsiting tool dependency whose definition was updated in a change set - pulled by a Galaxy administrator when getting updates to an installed tool shed - repository. The original_dependency_dict is a single tool dependency definition, - an example of which is:: - - {"name": "bwa", - "readme": "\\nCompiling BWA requires zlib and libpthread to be present on your system.\\n ", - "type": "package", - "version": "0.6.2"} - - The new_dependencies_dict is the dictionary generated by the metadata_util.generate_tool_dependency_metadata method. - """ - new_tool_dependency = None - original_name = original_dependency_dict[ 'name' ] - original_type = original_dependency_dict[ 'type' ] - original_version = original_dependency_dict[ 'version' ] - # Locate the appropriate tool_dependency associated with the repository. - tool_dependency = None - for tool_dependency in repository.tool_dependencies: - if tool_dependency.name == original_name and tool_dependency.type == original_type and tool_dependency.version == original_version: - break - if tool_dependency and tool_dependency.can_update: - dependency_install_dir = tool_dependency.installation_directory( app ) - removed_from_disk, error_message = tool_dependency_util.remove_tool_dependency_installation_directory( dependency_install_dir ) - if removed_from_disk: - context = app.install_model.context - new_dependency_name = None - new_dependency_type = None - new_dependency_version = None - for new_dependency_key, new_dependency_val_dict in new_dependencies_dict.items(): - # Match on name only, hopefully this will be enough! - if original_name == new_dependency_val_dict[ 'name' ]: - new_dependency_name = new_dependency_val_dict[ 'name' ] - new_dependency_type = new_dependency_val_dict[ 'type' ] - new_dependency_version = new_dependency_val_dict[ 'version' ] - break - if new_dependency_name and new_dependency_type and new_dependency_version: - # Update all attributes of the tool_dependency record in the database. - log.debug( "Updating version %s of tool dependency %s %s to have new version %s and type %s." % \ - ( str( tool_dependency.version ), - str( tool_dependency.type ), - str( tool_dependency.name ), - str( new_dependency_version ), - str( new_dependency_type ) ) ) - tool_dependency.type = new_dependency_type - tool_dependency.version = new_dependency_version - tool_dependency.status = app.install_model.ToolDependency.installation_status.UNINSTALLED - tool_dependency.error_message = None - context.add( tool_dependency ) - context.flush() - new_tool_dependency = tool_dependency - else: - # We have no new tool dependency definition based on a matching dependency name, so remove - # the existing tool dependency record from the database. - log.debug( "Deleting version %s of tool dependency %s %s from the database since it is no longer defined." % \ - ( str( tool_dependency.version ), str( tool_dependency.type ), str( tool_dependency.name ) ) ) - context.delete( tool_dependency ) - context.flush() - return new_tool_dependency - def update_repository_dependencies_metadata( metadata, repository_dependency_tups, is_valid, description ): if is_valid: repository_dependencies_dict = metadata.get( 'repository_dependencies', None ) This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: davebgx: Make the tool shed's database migrations compatible with MySQL.
by commits-noreply＠bitbucket.org 16 Jul '14

16 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3232e68101a4/ Changeset: 3232e68101a4 User: davebgx Date: 2014-07-16 18:50:15 Summary: Make the tool shed's database migrations compatible with MySQL. Affected #: 1 file diff -r 10c58feed93f26d8247be96baa111b3408970784 -r 3232e68101a46276f07c1f545ccf37c55552f66a lib/galaxy/webapps/tool_shed/model/migrate/versions/0001_initial_tables.py --- a/lib/galaxy/webapps/tool_shed/model/migrate/versions/0001_initial_tables.py +++ b/lib/galaxy/webapps/tool_shed/model/migrate/versions/0001_initial_tables.py @@ -143,12 +143,13 @@ Column( "id", Integer, primary_key=True ), Column( "tool_id", Integer, ForeignKey( "tool.id" ), index=True ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), - Column( "annotation", TEXT, index=True) ) + Column( "annotation", TEXT ) ) def upgrade( migrate_engine ): print __doc__ metadata.bind = migrate_engine metadata.create_all() + Index( 'ix_tool_annotation_association_annotation', ToolAnnotationAssociation_table.c.annotation, mysql_length=767 ).create() def downgrade( migrate_engine ): # Operations to reverse the above upgrade go here. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Rename the ~/tool_shed/dependencies/dependency_manager.py module to be attribute_handlers.py
by commits-noreply＠bitbucket.org 16 Jul '14

16 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/10c58feed93f/ Changeset: 10c58feed93f User: greg Date: 2014-07-16 15:16:35 Summary: Rename the ~/tool_shed/dependencies/dependency_manager.py module to be attribute_handlers.py Affected #: 4 files diff -r 6a548e4e607f57942bc874ab10eb738522d5e9d9 -r 10c58feed93f26d8247be96baa111b3408970784 lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -8,7 +8,7 @@ from galaxy import util from galaxy import web from galaxy.datatypes import checkers -from tool_shed.dependencies import dependency_manager +from tool_shed.dependencies import attribute_handlers import tool_shed.repository_types.util as rt_util from tool_shed.util import basic_util from tool_shed.util import commit_util @@ -95,8 +95,8 @@ uploaded_file_filename = os.path.split( file_data.filename )[ -1 ] isempty = os.path.getsize( os.path.abspath( uploaded_file_name ) ) == 0 if uploaded_file or uploaded_directory: - rdah = dependency_manager.RepositoryDependencyAttributeHandler( trans.app, unpopulate=False ) - tdah = dependency_manager.ToolDependencyAttributeHandler( trans.app, unpopulate=False ) + rdah = attribute_handlers.RepositoryDependencyAttributeHandler( trans.app, unpopulate=False ) + tdah = attribute_handlers.ToolDependencyAttributeHandler( trans.app, unpopulate=False ) ok = True isgzip = False isbz2 = False diff -r 6a548e4e607f57942bc874ab10eb738522d5e9d9 -r 10c58feed93f26d8247be96baa111b3408970784 lib/tool_shed/capsule/capsule_manager.py --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -13,8 +13,7 @@ from galaxy.util import CHUNK_SIZE from galaxy.util.odict import odict from tool_shed.dependencies.repository.relation_builder import RelationBuilder -from tool_shed.dependencies.dependency_manager import RepositoryDependencyAttributeHandler -from tool_shed.dependencies.dependency_manager import ToolDependencyAttributeHandler +from tool_shed.dependencies import attribute_handlers from tool_shed.util import basic_util from tool_shed.util import commit_util from tool_shed.util import common_util @@ -130,8 +129,8 @@ return sub_elements def generate_repository_archive( self, repository, changeset_revision, work_dir ): - rdah = RepositoryDependencyAttributeHandler( self.app, unpopulate=True ) - tdah = ToolDependencyAttributeHandler( self.app, unpopulate=True ) + rdah = attribute_handlers.RepositoryDependencyAttributeHandler( self.app, unpopulate=True ) + tdah = attribute_handlers.ToolDependencyAttributeHandler( self.app, unpopulate=True ) file_type_str = basic_util.get_file_type_str( changeset_revision, self.file_type ) file_name = '%s-%s' % ( repository.name, file_type_str ) return_code, error_message = hg_util.archive_repository_revision( self.app, @@ -646,8 +645,8 @@ def import_repository_archive( self, repository, repository_archive_dict ): """Import a repository archive contained within a repository capsule.""" - rdah = RepositoryDependencyAttributeHandler( self.app, unpopulate=False ) - tdah = ToolDependencyAttributeHandler( self.app, unpopulate=False ) + rdah = attribute_handlers.RepositoryDependencyAttributeHandler( self.app, unpopulate=False ) + tdah = attribute_handlers.ToolDependencyAttributeHandler( self.app, unpopulate=False ) archive_file_name = repository_archive_dict.get( 'archive_file_name', None ) capsule_file_name = repository_archive_dict[ 'capsule_file_name' ] encoded_file_path = repository_archive_dict[ 'encoded_file_path' ] diff -r 6a548e4e607f57942bc874ab10eb738522d5e9d9 -r 10c58feed93f26d8247be96baa111b3408970784 lib/tool_shed/dependencies/attribute_handlers.py --- /dev/null +++ b/lib/tool_shed/dependencies/attribute_handlers.py @@ -0,0 +1,203 @@ +import copy +import logging + +from galaxy.util import asbool +from galaxy.util.odict import odict +from galaxy.web import url_for + +from tool_shed.dependencies.tool import tag_attribute_handler +from tool_shed.repository_types.util import REPOSITORY_DEPENDENCY_DEFINITION_FILENAME +from tool_shed.repository_types.util import TOOL_DEPENDENCY_DEFINITION_FILENAME +from tool_shed.util import hg_util +from tool_shed.util import shed_util_common as suc +from tool_shed.util import xml_util +log = logging.getLogger( __name__ ) + + +class RepositoryDependencyAttributeHandler( object ): + + def __init__( self, app, unpopulate ): + self.app = app + self.file_name = REPOSITORY_DEPENDENCY_DEFINITION_FILENAME + self.unpopulate = unpopulate + + def check_tag_attributes( self, elem ): + # <repository name="molecule_datatypes" owner="test" /> + error_message = '' + name = elem.get( 'name' ) + if not name: + error_message += 'The tag is missing the required name attribute. ' + owner = elem.get( 'owner' ) + if not owner: + error_message += 'The tag is missing the required owner attribute. ' + log.debug( error_message ) + return error_message + + def handle_complex_dependency_elem( self, parent_elem, elem_index, elem ): + """ + Populate or unpopulate the toolshed and changeset_revision attributes of a + <repository> tag that defines a complex repository dependency. + """ + # <repository name="package_eigen_2_0" owner="test" prior_installation_required="True" /> + altered, new_elem, error_message = self.handle_elem( elem ) + if error_message: + error_message += ' The %s file contains an invalid <repository> tag.' % TOOL_DEPENDENCY_DEFINITION_FILENAME + return altered, new_elem, error_message + + def handle_elem( self, elem ): + """Populate or unpopulate the changeset_revision and toolshed attributes of repository tags.""" + # <repository name="molecule_datatypes" owner="test" changeset_revision="1a070566e9c6" /> + # <repository changeset_revision="xxx" name="package_xorg_macros_1_17_1" owner="test" toolshed="yyy"> + # <package name="xorg_macros" version="1.17.1" /> + # </repository> + error_message = '' + name = elem.get( 'name' ) + owner = elem.get( 'owner' ) + # The name and owner attributes are always required, so if either are missing, return the error message. + if not name or not owner: + error_message = self.check_tag_attributes( elem ) + return False, elem, error_message + altered = False + toolshed = elem.get( 'toolshed' ) + changeset_revision = elem.get( 'changeset_revision' ) + # Over a short period of time a bug existed which caused the prior_installation_required attribute + # to be set to False and included in the <repository> tag when a repository was exported along with + # its dependencies. The following will eliminate this problematic attribute upon import. + prior_installation_required = elem.get( 'prior_installation_required' ) + if prior_installation_required is not None and not asbool( prior_installation_required ): + del elem.attrib[ 'prior_installation_required' ] + sub_elems = [ child_elem for child_elem in list( elem ) ] + if len( sub_elems ) > 0: + # At this point, a <repository> tag will point only to a package. + # <package name="xorg_macros" version="1.17.1" /> + # Coerce the list to an odict(). + sub_elements = odict() + packages = [] + for sub_elem in sub_elems: + sub_elem_type = sub_elem.tag + sub_elem_name = sub_elem.get( 'name' ) + sub_elem_version = sub_elem.get( 'version' ) + if sub_elem_type and sub_elem_name and sub_elem_version: + packages.append( ( sub_elem_name, sub_elem_version ) ) + sub_elements[ 'packages' ] = packages + else: + # Set to None. + sub_elements = None + if self.unpopulate: + # We're exporting the repository, so eliminate all toolshed and changeset_revision attributes + # from the <repository> tag. + if toolshed or changeset_revision: + attributes = odict() + attributes[ 'name' ] = name + attributes[ 'owner' ] = owner + prior_installation_required = elem.get( 'prior_installation_required' ) + if asbool( prior_installation_required ): + attributes[ 'prior_installation_required' ] = 'True' + new_elem = xml_util.create_element( 'repository', attributes=attributes, sub_elements=sub_elements ) + altered = True + return altered, new_elem, error_message + # From here on we're populating the toolshed and changeset_revision attributes if necessary. + if not toolshed: + # Default the setting to the current tool shed. + toolshed = str( url_for( '/', qualified=True ) ).rstrip( '/' ) + elem.attrib[ 'toolshed' ] = toolshed + altered = True + if not changeset_revision: + # Populate the changeset_revision attribute with the latest installable metadata revision for + # the defined repository. We use the latest installable revision instead of the latest metadata + # revision to ensure that the contents of the revision are valid. + repository = suc.get_repository_by_name_and_owner( self.app, name, owner ) + if repository: + repo = hg_util.get_repo_for_repository( self.app, + repository=repository, + repo_path=None, + create=False ) + lastest_installable_changeset_revision = \ + suc.get_latest_downloadable_changeset_revision( self.app, repository, repo ) + if lastest_installable_changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: + elem.attrib[ 'changeset_revision' ] = lastest_installable_changeset_revision + altered = True + else: + error_message = 'Invalid latest installable changeset_revision %s ' % \ + str( lastest_installable_changeset_revision ) + error_message += 'retrieved for repository %s owned by %s. ' % ( str( name ), str( owner ) ) + else: + error_message = 'Unable to locate repository with name %s and owner %s. ' % ( str( name ), str( owner ) ) + return altered, elem, error_message + + def handle_sub_elem( self, parent_elem, elem_index, elem ): + """ + Populate or unpopulate the toolshed and changeset_revision attributes for each of + the following tag sets. + <action type="set_environment_for_install"> + <action type="setup_r_environment"> + <action type="setup_ruby_environment"> + """ + sub_elem_altered = False + error_message = '' + for sub_index, sub_elem in enumerate( elem ): + # Make sure to skip comments and tags that are not <repository>. + if sub_elem.tag == 'repository': + altered, new_sub_elem, message = self.handle_elem( sub_elem ) + if message: + error_message += 'The %s file contains an invalid <repository> tag. %s' % \ + ( TOOL_DEPENDENCY_DEFINITION_FILENAME, message ) + if altered: + if not sub_elem_altered: + sub_elem_altered = True + elem[ sub_index ] = new_sub_elem + if sub_elem_altered: + parent_elem[ elem_index ] = elem + return sub_elem_altered, parent_elem, error_message + + def handle_tag_attributes( self, config ): + """ + Populate or unpopulate the toolshed and changeset_revision attributes of a + <repository> tag. Populating will occur when a dependency definition file + is being uploaded to the repository, while unpopulating will occur when the + repository is being exported. + """ + # Make sure we're looking at a valid repository_dependencies.xml file. + tree, error_message = xml_util.parse_xml( config ) + if tree is None: + return False, None, error_message + root = tree.getroot() + root_altered = False + new_root = copy.deepcopy( root ) + for index, elem in enumerate( root ): + if elem.tag == 'repository': + # <repository name="molecule_datatypes" owner="test" changeset_revision="1a070566e9c6" /> + altered, new_elem, error_message = self.handle_elem( elem ) + if error_message: + error_message = 'The %s file contains an invalid <repository> tag. %s' % ( self.file_name, error_message ) + return False, None, error_message + if altered: + if not root_altered: + root_altered = True + new_root[ index ] = new_elem + return root_altered, new_root, error_message + + +class ToolDependencyAttributeHandler( object ): + + def __init__( self, app, unpopulate ): + self.app = app + self.file_name = TOOL_DEPENDENCY_DEFINITION_FILENAME + self.unpopulate = unpopulate + + def handle_tag_attributes( self, tool_dependencies_config ): + """ + Populate or unpopulate the tooshed and changeset_revision attributes of each <repository> + tag defined within a tool_dependencies.xml file. + """ + rdah = RepositoryDependencyAttributeHandler( self.app, self.unpopulate ) + tah = tag_attribute_handler.TagAttributeHandler( self.app, rdah, self.unpopulate ) + altered = False + error_message = '' + # Make sure we're looking at a valid tool_dependencies.xml file. + tree, error_message = xml_util.parse_xml( tool_dependencies_config ) + if tree is None: + return False, None, error_message + root = tree.getroot() + altered, new_root, error_message = tah.process_config( root ) + return altered, new_root, error_message diff -r 6a548e4e607f57942bc874ab10eb738522d5e9d9 -r 10c58feed93f26d8247be96baa111b3408970784 lib/tool_shed/dependencies/dependency_manager.py --- a/lib/tool_shed/dependencies/dependency_manager.py +++ /dev/null @@ -1,203 +0,0 @@ -import copy -import logging - -from galaxy.util import asbool -from galaxy.util.odict import odict -from galaxy.web import url_for - -from tool_shed.dependencies.tool import tag_attribute_handler -from tool_shed.repository_types.util import REPOSITORY_DEPENDENCY_DEFINITION_FILENAME -from tool_shed.repository_types.util import TOOL_DEPENDENCY_DEFINITION_FILENAME -from tool_shed.util import hg_util -from tool_shed.util import shed_util_common as suc -from tool_shed.util import xml_util -log = logging.getLogger( __name__ ) - - -class RepositoryDependencyAttributeHandler( object ): - - def __init__( self, app, unpopulate ): - self.app = app - self.file_name = REPOSITORY_DEPENDENCY_DEFINITION_FILENAME - self.unpopulate = unpopulate - - def check_tag_attributes( self, elem ): - # <repository name="molecule_datatypes" owner="test" /> - error_message = '' - name = elem.get( 'name' ) - if not name: - error_message += 'The tag is missing the required name attribute. ' - owner = elem.get( 'owner' ) - if not owner: - error_message += 'The tag is missing the required owner attribute. ' - log.debug( error_message ) - return error_message - - def handle_complex_dependency_elem( self, parent_elem, elem_index, elem ): - """ - Populate or unpopulate the toolshed and changeset_revision attributes of a - <repository> tag that defines a complex repository dependency. - """ - # <repository name="package_eigen_2_0" owner="test" prior_installation_required="True" /> - altered, new_elem, error_message = self.handle_elem( elem ) - if error_message: - error_message += ' The %s file contains an invalid <repository> tag.' % TOOL_DEPENDENCY_DEFINITION_FILENAME - return altered, new_elem, error_message - - def handle_elem( self, elem ): - """Populate or unpopulate the changeset_revision and toolshed attributes of repository tags.""" - # <repository name="molecule_datatypes" owner="test" changeset_revision="1a070566e9c6" /> - # <repository changeset_revision="xxx" name="package_xorg_macros_1_17_1" owner="test" toolshed="yyy"> - # <package name="xorg_macros" version="1.17.1" /> - # </repository> - error_message = '' - name = elem.get( 'name' ) - owner = elem.get( 'owner' ) - # The name and owner attributes are always required, so if either are missing, return the error message. - if not name or not owner: - error_message = self.check_tag_attributes( elem ) - return False, elem, error_message - altered = False - toolshed = elem.get( 'toolshed' ) - changeset_revision = elem.get( 'changeset_revision' ) - # Over a short period of time a bug existed which caused the prior_installation_required attribute - # to be set to False and included in the <repository> tag when a repository was exported along with - # its dependencies. The following will eliminate this problematic attribute upon import. - prior_installation_required = elem.get( 'prior_installation_required' ) - if prior_installation_required is not None and not asbool( prior_installation_required ): - del elem.attrib[ 'prior_installation_required' ] - sub_elems = [ child_elem for child_elem in list( elem ) ] - if len( sub_elems ) > 0: - # At this point, a <repository> tag will point only to a package. - # <package name="xorg_macros" version="1.17.1" /> - # Coerce the list to an odict(). - sub_elements = odict() - packages = [] - for sub_elem in sub_elems: - sub_elem_type = sub_elem.tag - sub_elem_name = sub_elem.get( 'name' ) - sub_elem_version = sub_elem.get( 'version' ) - if sub_elem_type and sub_elem_name and sub_elem_version: - packages.append( ( sub_elem_name, sub_elem_version ) ) - sub_elements[ 'packages' ] = packages - else: - # Set to None. - sub_elements = None - if self.unpopulate: - # We're exporting the repository, so eliminate all toolshed and changeset_revision attributes - # from the <repository> tag. - if toolshed or changeset_revision: - attributes = odict() - attributes[ 'name' ] = name - attributes[ 'owner' ] = owner - prior_installation_required = elem.get( 'prior_installation_required' ) - if asbool( prior_installation_required ): - attributes[ 'prior_installation_required' ] = 'True' - new_elem = xml_util.create_element( 'repository', attributes=attributes, sub_elements=sub_elements ) - altered = True - return altered, new_elem, error_message - # From here on we're populating the toolshed and changeset_revision attributes if necessary. - if not toolshed: - # Default the setting to the current tool shed. - toolshed = str( url_for( '/', qualified=True ) ).rstrip( '/' ) - elem.attrib[ 'toolshed' ] = toolshed - altered = True - if not changeset_revision: - # Populate the changeset_revision attribute with the latest installable metadata revision for - # the defined repository. We use the latest installable revision instead of the latest metadata - # revision to ensure that the contents of the revision are valid. - repository = suc.get_repository_by_name_and_owner( self.app, name, owner ) - if repository: - repo = hg_util.get_repo_for_repository( self.app, - repository=repository, - repo_path=None, - create=False ) - lastest_installable_changeset_revision = \ - suc.get_latest_downloadable_changeset_revision( self.app, repository, repo ) - if lastest_installable_changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: - elem.attrib[ 'changeset_revision' ] = lastest_installable_changeset_revision - altered = True - else: - error_message = 'Invalid latest installable changeset_revision %s ' % \ - str( lastest_installable_changeset_revision ) - error_message += 'retrieved for repository %s owned by %s. ' % ( str( name ), str( owner ) ) - else: - error_message = 'Unable to locate repository with name %s and owner %s. ' % ( str( name ), str( owner ) ) - return altered, elem, error_message - - def handle_sub_elem( self, parent_elem, elem_index, elem ): - """ - Populate or unpopulate the toolshed and changeset_revision attributes for each of - the following tag sets. - <action type="set_environment_for_install"> - <action type="setup_r_environment"> - <action type="setup_ruby_environment"> - """ - sub_elem_altered = False - error_message = '' - for sub_index, sub_elem in enumerate( elem ): - # Make sure to skip comments and tags that are not <repository>. - if sub_elem.tag == 'repository': - altered, new_sub_elem, message = self.handle_elem( sub_elem ) - if message: - error_message += 'The %s file contains an invalid <repository> tag. %s' % \ - ( TOOL_DEPENDENCY_DEFINITION_FILENAME, message ) - if altered: - if not sub_elem_altered: - sub_elem_altered = True - elem[ sub_index ] = new_sub_elem - if sub_elem_altered: - parent_elem[ elem_index ] = elem - return sub_elem_altered, parent_elem, error_message - - def handle_tag_attributes( self, config ): - """ - Populate or unpopulate the toolshed and changeset_revision attributes of a - <repository> tag. Populating will occur when a dependency definition file - is being uploaded to the repository, while unpopulating will occur when the - repository is being exported. - """ - # Make sure we're looking at a valid repository_dependencies.xml file. - tree, error_message = xml_util.parse_xml( config ) - if tree is None: - return False, None, error_message - root = tree.getroot() - root_altered = False - new_root = copy.deepcopy( root ) - for index, elem in enumerate( root ): - if elem.tag == 'repository': - # <repository name="molecule_datatypes" owner="test" changeset_revision="1a070566e9c6" /> - altered, new_elem, error_message = self.handle_elem( elem ) - if error_message: - error_message = 'The %s file contains an invalid <repository> tag. %s' % ( self.file_name, error_message ) - return False, None, error_message - if altered: - if not root_altered: - root_altered = True - new_root[ index ] = new_elem - return root_altered, new_root, error_message - - -class ToolDependencyAttributeHandler( object ): - - def __init__( self, app, unpopulate ): - self.app = app - self.file_name = TOOL_DEPENDENCY_DEFINITION_FILENAME - self.unpopulate = unpopulate - - def handle_tag_attributes( self, tool_dependencies_config ): - """ - Populate or unpopulate the tooshed and changeset_revision attributes of each <repository> - tag defined within a tool_dependencies.xml file. - """ - rdah = RepositoryDependencyAttributeHandler( self.app, self.unpopulate ) - tah = tag_attribute_handler.TagAttributeHandler( self.app, rdah, self.unpopulate ) - altered = False - error_message = '' - # Make sure we're looking at a valid tool_dependencies.xml file. - tree, error_message = xml_util.parse_xml( tool_dependencies_config ) - if tree is None: - return False, None, error_message - root = tree.getroot() - altered, new_root, error_message = tah.process_config( root ) - return altered, new_root, error_message Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Visualization framework: fix bug in searching interval features.
by commits-noreply＠bitbucket.org 15 Jul '14

15 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6a548e4e607f/ Changeset: 6a548e4e607f User: jgoecks Date: 2014-07-15 23:04:33 Summary: Visualization framework: fix bug in searching interval features. Affected #: 1 file diff -r 70272fb2005161f34e13173df323b8e5363a9319 -r 6a548e4e607f57942bc874ab10eb738522d5e9d9 lib/galaxy/visualization/data_providers/genome.py --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -93,7 +93,10 @@ else: high = mid + # Need to move back one line because last line read may be included in + # results. position = low * line_len + textloc_file.seek( position ) # At right point in file, generate hits. result = [] Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Eliminate the use of trans in all remaining functions in the metadata_util.py tool shed module.
by commits-noreply＠bitbucket.org 15 Jul '14

15 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/70272fb20051/ Changeset: 70272fb20051 User: greg Date: 2014-07-15 22:44:02 Summary: Eliminate the use of trans in all remaining functions in the metadata_util.py tool shed module. Affected #: 5 files diff -r 0874eb058a0cb79508672cc5ebf25a4ce5063d75 -r 70272fb2005161f34e13173df323b8e5363a9319 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -779,7 +779,7 @@ trans.install_model.context.add( repository ) trans.install_model.context.flush() message = "The repository information has been updated." - containers_dict = metadata_util.populate_containers_dict_from_repository_metadata( trans=trans, + containers_dict = metadata_util.populate_containers_dict_from_repository_metadata( app=trans.app, tool_shed_url=tool_shed_url, tool_path=tool_path, repository=repository, @@ -1697,7 +1697,7 @@ @web.require_admin def reset_metadata_on_selected_installed_repositories( self, trans, **kwd ): if 'reset_metadata_on_selected_repositories_button' in kwd: - message, status = metadata_util.reset_metadata_on_selected_repositories( trans, **kwd ) + message, status = metadata_util.reset_metadata_on_selected_repositories( trans.app, trans.user, **kwd ) else: message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) @@ -1796,7 +1796,7 @@ status = 'error' shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) ) - containers_dict = metadata_util.populate_containers_dict_from_repository_metadata( trans=trans, + containers_dict = metadata_util.populate_containers_dict_from_repository_metadata( app=trans.app, tool_shed_url=tool_shed_url, tool_path=tool_path, repository=repository, diff -r 0874eb058a0cb79508672cc5ebf25a4ce5063d75 -r 70272fb2005161f34e13173df323b8e5363a9319 lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -429,7 +429,7 @@ repository_id = trans.security.encode_id( repository.id ) try: invalid_file_tups, metadata_dict = \ - metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans, repository_id ) + metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans.app, trans.user, repository_id ) if invalid_file_tups: message = tool_util.generate_message_for_invalid_tools( trans.app, invalid_file_tups, @@ -509,7 +509,8 @@ repository_status=[] ) try: invalid_file_tups, metadata_dict = \ - metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans, + metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans.app, + trans.user, trans.security.encode_id( repository.id ) ) if invalid_file_tups: message = tool_util.generate_message_for_invalid_tools( trans.app, diff -r 0874eb058a0cb79508672cc5ebf25a4ce5063d75 -r 70272fb2005161f34e13173df323b8e5363a9319 lib/galaxy/webapps/tool_shed/controllers/admin.py --- a/lib/galaxy/webapps/tool_shed/controllers/admin.py +++ b/lib/galaxy/webapps/tool_shed/controllers/admin.py @@ -344,7 +344,7 @@ @web.require_admin def reset_metadata_on_selected_repositories_in_tool_shed( self, trans, **kwd ): if 'reset_metadata_on_selected_repositories_button' in kwd: - message, status = metadata_util.reset_metadata_on_selected_repositories( trans, **kwd ) + message, status = metadata_util.reset_metadata_on_selected_repositories( trans.app, trans.user, **kwd ) else: message = util.restore_text( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) diff -r 0874eb058a0cb79508672cc5ebf25a4ce5063d75 -r 70272fb2005161f34e13173df323b8e5363a9319 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -2733,7 +2733,8 @@ def reset_all_metadata( self, trans, id, **kwd ): """Reset all metadata on the complete changelog for a single repository in the tool shed.""" # This method is called only from the ~/templates/webapps/tool_shed/repository/manage_repository.mako template. - invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans, id, **kwd ) + invalid_file_tups, metadata_dict = \ + metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans.app, trans.user, id, **kwd ) if invalid_file_tups: repository = suc.get_repository_in_tool_shed( trans.app, id ) message = tool_util.generate_message_for_invalid_tools( trans.app, invalid_file_tups, repository, metadata_dict ) @@ -2750,7 +2751,7 @@ @web.expose def reset_metadata_on_my_writable_repositories_in_tool_shed( self, trans, **kwd ): if 'reset_metadata_on_selected_repositories_button' in kwd: - message, status = metadata_util.reset_metadata_on_selected_repositories( trans, **kwd ) + message, status = metadata_util.reset_metadata_on_selected_repositories( trans.app, trans.user, **kwd ) else: message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) diff -r 0874eb058a0cb79508672cc5ebf25a4ce5063d75 -r 70272fb2005161f34e13173df323b8e5363a9319 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -58,19 +58,22 @@ sa_session.add( repository_metadata ) sa_session.flush() -def clean_repository_metadata( trans, id, changeset_revisions ): - # Delete all repository_metadata records associated with the repository that have a changeset_revision that is not in changeset_revisions. - # We sometimes see multiple records with the same changeset revision value - no idea how this happens. We'll assume we can delete the older - # records, so we'll order by update_time descending and delete records that have the same changeset_revision we come across later.. +def clean_repository_metadata( app, id, changeset_revisions ): + # Delete all repository_metadata records associated with the repository that have + # a changeset_revision that is not in changeset_revisions. We sometimes see multiple + # records with the same changeset revision value - no idea how this happens. We'll + # assume we can delete the older records, so we'll order by update_time descending and + # delete records that have the same changeset_revision we come across later. + sa_session = app.model.context.current changeset_revisions_checked = [] - for repository_metadata in trans.sa_session.query( trans.model.RepositoryMetadata ) \ - .filter( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ) ) \ - .order_by( trans.model.RepositoryMetadata.table.c.changeset_revision, - trans.model.RepositoryMetadata.table.c.update_time.desc() ): + for repository_metadata in sa_session.query( app.model.RepositoryMetadata ) \ + .filter( app.model.RepositoryMetadata.table.c.repository_id == app.security.decode_id( id ) ) \ + .order_by( app.model.RepositoryMetadata.table.c.changeset_revision, + app.model.RepositoryMetadata.table.c.update_time.desc() ): changeset_revision = repository_metadata.changeset_revision if changeset_revision in changeset_revisions_checked or changeset_revision not in changeset_revisions: - trans.sa_session.delete( repository_metadata ) - trans.sa_session.flush() + sa_session.delete( repository_metadata ) + sa_session.flush() def compare_changeset_revisions( app, ancestor_changeset_revision, ancestor_metadata_dict, current_changeset_revision, current_metadata_dict ): @@ -1547,13 +1550,15 @@ return True return False -def new_readme_files_metadata_required( trans, repository_metadata, metadata_dict ): +def new_readme_files_metadata_required( repository_metadata, metadata_dict ): """ - Compare the last saved metadata for each readme file in the repository with the new metadata in metadata_dict to determine if a new - repository_metadata table record is required or if the last saved metadata record can be updated for readme files instead. + Compare the last saved metadata for each readme file in the repository with the new metadata + in metadata_dict to determine if a new repository_metadata table record is required or if the + last saved metadata record can be updated for readme files instead. """ - # Repository README files are kind of a special case because they have no effect on reproducibility. We'll simply inspect the file names to - # determine if any that exist in the saved metadata are eliminated from the new metadata in the received metadata_dict. + # Repository README files are kind of a special case because they have no effect on reproducibility. + # We'll simply inspect the file names to determine if any that exist in the saved metadata are + # eliminated from the new metadata in the received metadata_dict. if 'readme_files' in metadata_dict: current_readme_files = metadata_dict[ 'readme_files' ] if repository_metadata: @@ -1568,15 +1573,19 @@ else: return False else: - # The new metadata includes readme_files, but the stored metadata does not, so we can update the stored metadata. + # The new metadata includes readme_files, but the stored metadata does not, so + # we can update the stored metadata. return False else: - # There is no stored metadata, so we can update the metadata column in the repository_metadata table. + # There is no stored metadata, so we can update the metadata column in the repository_metadata + # table. return False else: - # There is no stored repository metadata, so we need to create a new repository_metadata table record. + # There is no stored repository metadata, so we need to create a new repository_metadata + # table record. return True - # The received metadata_dict includes no metadata for readme_files, so a new repository_metadata table record is not needed. + # The received metadata_dict includes no metadata for readme_files, so a new repository_metadata + # table record is not needed. return False def new_repository_dependency_metadata_required( app, repository_metadata, metadata_dict ): @@ -1728,7 +1737,7 @@ # The received metadata_dict includes no metadata for workflows, so a new repository_metadata table record is not needed. return False -def populate_containers_dict_from_repository_metadata( trans, tool_shed_url, tool_path, repository, +def populate_containers_dict_from_repository_metadata( app, tool_shed_url, tool_path, repository, reinstalling=False, required_repo_info_dicts=None ): """ Retrieve necessary information from the received repository's metadata to populate the @@ -1744,19 +1753,19 @@ invalid_tools = metadata.get( 'invalid_tools', None ) # Handle README files. if repository.has_readme_files: - if reinstalling or repository.status not in [ trans.install_model.ToolShedRepository.installation_status.DEACTIVATED, - trans.install_model.ToolShedRepository.installation_status.INSTALLED ]: + if reinstalling or repository.status not in [ app.install_model.ToolShedRepository.installation_status.DEACTIVATED, + app.install_model.ToolShedRepository.installation_status.INSTALLED ]: # Since we're reinstalling, we need to send a request to the tool shed to get the README files. - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed_url ) + tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) params = '?name=%s&owner=%s&changeset_revision=%s' % ( str( repository.name ), str( repository.owner ), str( repository.installed_changeset_revision ) ) url = common_util.url_join( tool_shed_url, 'repository/get_readme_files%s' % params ) - raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) + raw_text = common_util.tool_shed_get( app, tool_shed_url, url ) readme_files_dict = json.from_json_string( raw_text ) else: - readme_files_dict = readme_util.build_readme_files_dict( trans.app, + readme_files_dict = readme_util.build_readme_files_dict( app, repository, repository.changeset_revision, repository.metadata, tool_path ) @@ -1764,7 +1773,7 @@ readme_files_dict = None # Handle repository dependencies. installed_repository_dependencies, missing_repository_dependencies = \ - trans.app.installed_repository_manager.get_installed_and_missing_repository_dependencies( repository ) + app.installed_repository_manager.get_installed_and_missing_repository_dependencies( repository ) # Handle the current repository's tool dependencies. repository_tool_dependencies = metadata.get( 'tool_dependencies', None ) # Make sure to display missing tool dependencies as well. @@ -1774,17 +1783,17 @@ repository_tool_dependencies = {} repository_tool_dependencies.update( repository_invalid_tool_dependencies ) repository_installed_tool_dependencies, repository_missing_tool_dependencies = \ - tool_dependency_util.get_installed_and_missing_tool_dependencies_for_installed_repository( trans, + tool_dependency_util.get_installed_and_missing_tool_dependencies_for_installed_repository( app, repository, repository_tool_dependencies ) if reinstalling: installed_tool_dependencies, missing_tool_dependencies = \ - tool_dependency_util.populate_tool_dependencies_dicts( app=trans.app, - tool_shed_url=tool_shed_url, - tool_path=tool_path, - repository_installed_tool_dependencies=repository_installed_tool_dependencies, - repository_missing_tool_dependencies=repository_missing_tool_dependencies, - required_repo_info_dicts=required_repo_info_dicts ) + tool_dependency_util.populate_tool_dependencies_dicts( app, + tool_shed_url, + tool_path, + repository_installed_tool_dependencies, + repository_missing_tool_dependencies, + required_repo_info_dicts ) else: installed_tool_dependencies = repository_installed_tool_dependencies missing_tool_dependencies = repository_missing_tool_dependencies @@ -1800,7 +1809,7 @@ valid_data_managers = metadata['data_manager'].get( 'data_managers', None ) invalid_data_managers = metadata['data_manager'].get( 'invalid_data_managers', None ) data_managers_errors = metadata['data_manager'].get( 'messages', None ) - containers_dict = container_util.build_repository_containers_for_galaxy( app=trans.app, + containers_dict = container_util.build_repository_containers_for_galaxy( app=app, repository=repository, datatypes=datatypes, invalid_tools=invalid_tools, @@ -1858,35 +1867,38 @@ log.debug( 'Error locating installation directory for repository %s.' % repository.name ) return invalid_file_tups, metadata_dict -def reset_all_metadata_on_repository_in_tool_shed( trans, id ): +def reset_all_metadata_on_repository_in_tool_shed( app, user, id ): """Reset all metadata on a single repository in a tool shed.""" - def reset_all_tool_versions( trans, id, repo ): + def reset_all_tool_versions( app, id, repo ): """Reset tool version lineage for those changeset revisions that include valid tools.""" + sa_session = app.model.context.current changeset_revisions_that_contain_tools = [] for changeset in repo.changelog: changeset_revision = str( repo.changectx( changeset ) ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, id, changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata if metadata: if metadata.get( 'tools', None ): changeset_revisions_that_contain_tools.append( changeset_revision ) - # The list of changeset_revisions_that_contain_tools is now filtered to contain only those that are downloadable and contain tools. - # If a repository includes tools, build a dictionary of { 'tool id' : 'parent tool id' } pairs for each tool in each changeset revision. + # The list of changeset_revisions_that_contain_tools is now filtered to contain only those that + # are downloadable and contain tools. If a repository includes tools, build a dictionary of + # { 'tool id' : 'parent tool id' } pairs for each tool in each changeset revision. for index, changeset_revision in enumerate( changeset_revisions_that_contain_tools ): tool_versions_dict = {} - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, id, changeset_revision ) metadata = repository_metadata.metadata tool_dicts = metadata[ 'tools' ] if index == 0: - # The first changeset_revision is a special case because it will have no ancestor changeset_revisions in which to match tools. - # The parent tool id for tools in the first changeset_revision will be the "old_id" in the tool config. + # The first changeset_revision is a special case because it will have no ancestor + # changeset_revisions in which to match tools. The parent tool id for tools in the + # first changeset_revision will be the "old_id" in the tool config. for tool_dict in tool_dicts: tool_versions_dict[ tool_dict[ 'guid' ] ] = tool_dict[ 'id' ] else: for tool_dict in tool_dicts: - parent_id = get_parent_id( trans.app, + parent_id = get_parent_id( app, id, tool_dict[ 'id' ], tool_dict[ 'version' ], @@ -1895,24 +1907,26 @@ tool_versions_dict[ tool_dict[ 'guid' ] ] = parent_id if tool_versions_dict: repository_metadata.tool_versions = tool_versions_dict - trans.sa_session.add( repository_metadata ) - trans.sa_session.flush() + sa_session.add( repository_metadata ) + sa_session.flush() - repository = suc.get_repository_in_tool_shed( trans.app, id ) + repository = suc.get_repository_in_tool_shed( app, id ) log.debug( "Resetting all metadata on repository: %s" % repository.name ) - repo_dir = repository.repo_path( trans.app ) - repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) - repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans.user, repository ) - # The list of changeset_revisions refers to repository_metadata records that have been created or updated. When the following loop - # completes, we'll delete all repository_metadata records for this repository that do not have a changeset_revision value in this list. + repo_dir = repository.repo_path( app ) + repo = hg_util.get_repo_for_repository( app, repository=None, repo_path=repo_dir, create=False ) + repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( user, repository ) + # The list of changeset_revisions refers to repository_metadata records that have been created + # or updated. When the following loop completes, we'll delete all repository_metadata records + # for this repository that do not have a changeset_revision value in this list. changeset_revisions = [] - # When a new repository_metadata record is created, it always uses the values of metadata_changeset_revision and metadata_dict. + # When a new repository_metadata record is created, it always uses the values of + # metadata_changeset_revision and metadata_dict. metadata_changeset_revision = None metadata_dict = None ancestor_changeset_revision = None ancestor_metadata_dict = None invalid_file_tups = [] - for changeset in repository.get_changesets_for_setting_metadata( trans.app ): + for changeset in repository.get_changesets_for_setting_metadata( app ): work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-ramorits" ) current_changeset_revision = str( repo.changectx( changeset ) ) ctx = repo.changectx( changeset ) @@ -1920,18 +1934,19 @@ cloned_ok, error_message = hg_util.clone_repository( repository_clone_url, work_dir, str( ctx.rev() ) ) if cloned_ok: log.debug( "Generating metadata for changset revision: %s", str( ctx.rev() ) ) - current_metadata_dict, invalid_tups = generate_metadata_for_changeset_revision( app=trans.app, - repository=repository, - changeset_revision=current_changeset_revision, - repository_clone_url=repository_clone_url, - relative_install_dir=repo_dir, - repository_files_dir=work_dir, - resetting_all_metadata_on_repository=True, - updating_installed_repository=False, - persist=False ) + current_metadata_dict, invalid_tups = \ + generate_metadata_for_changeset_revision( app=app, + repository=repository, + changeset_revision=current_changeset_revision, + repository_clone_url=repository_clone_url, + relative_install_dir=repo_dir, + repository_files_dir=work_dir, + resetting_all_metadata_on_repository=True, + updating_installed_repository=False, + persist=False ) # We'll only display error messages for the repository tip (it may be better to display error # messages for each installable changeset revision). - if current_changeset_revision == repository.tip( trans.app ): + if current_changeset_revision == repository.tip( app ): invalid_file_tups.extend( invalid_tups ) if current_metadata_dict: if metadata_changeset_revision is None and metadata_dict is None: @@ -1945,7 +1960,7 @@ # SUBSET - ancestor metadata is a subset of current metadata, so continue from current # NOT_EQUAL_AND_NOT_SUBSET - ancestor metadata is neither equal to nor a subset of current # metadata, so persist ancestor metadata. - comparison = compare_changeset_revisions( trans.app, + comparison = compare_changeset_revisions( app, ancestor_changeset_revision, ancestor_metadata_dict, current_changeset_revision, @@ -1956,7 +1971,7 @@ elif comparison == NOT_EQUAL_AND_NOT_SUBSET: metadata_changeset_revision = ancestor_changeset_revision metadata_dict = ancestor_metadata_dict - repository_metadata = create_or_update_repository_metadata( trans.app, + repository_metadata = create_or_update_repository_metadata( app, id, repository, metadata_changeset_revision, @@ -1972,7 +1987,7 @@ metadata_changeset_revision = current_changeset_revision metadata_dict = current_metadata_dict # We're at the end of the change log. - repository_metadata = create_or_update_repository_metadata( trans.app, + repository_metadata = create_or_update_repository_metadata( app, id, repository, metadata_changeset_revision, @@ -1984,7 +1999,7 @@ # We reach here only if current_metadata_dict is empty and ancestor_metadata_dict is not. if not ctx.children(): # We're at the end of the change log. - repository_metadata = create_or_update_repository_metadata( trans.app, + repository_metadata = create_or_update_repository_metadata( app, id, repository, metadata_changeset_revision, @@ -1995,15 +2010,15 @@ basic_util.remove_dir( work_dir ) # Delete all repository_metadata records for this repository that do not have a changeset_revision # value in changeset_revisions. - clean_repository_metadata( trans, id, changeset_revisions ) + clean_repository_metadata( app, id, changeset_revisions ) # Set tool version information for all downloadable changeset revisions. Get the list of changeset # revisions from the changelog. - reset_all_tool_versions( trans, id, repo ) + reset_all_tool_versions( app, id, repo ) # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. - tool_util.reset_tool_data_tables( trans.app ) + tool_util.reset_tool_data_tables( app ) return invalid_file_tups, metadata_dict -def reset_metadata_on_selected_repositories( trans, **kwd ): +def reset_metadata_on_selected_repositories( app, user, **kwd ): """ Inspect the repository changelog to reset metadata for all appropriate changeset revisions. This method is called from both Galaxy and the Tool Shed. @@ -2016,18 +2031,23 @@ unsuccessful_count = 0 for repository_id in repository_ids: try: - if trans.webapp.name == 'tool_shed': + if app.name == 'tool_shed': # We're in the tool shed. - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = suc.get_repository_in_tool_shed( app, repository_id ) owner = str( repository.user.username ) - invalid_file_tups, metadata_dict = reset_all_metadata_on_repository_in_tool_shed( trans, repository_id ) + invalid_file_tups, metadata_dict = \ + reset_all_metadata_on_repository_in_tool_shed( app, user, repository_id ) else: # We're in Galaxy. - repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = suc.get_installed_tool_shed_repository( app, repository_id ) owner = str( repository.owner ) - invalid_file_tups, metadata_dict = reset_all_metadata_on_installed_repository( trans.app, repository_id ) + invalid_file_tups, metadata_dict = reset_all_metadata_on_installed_repository( app, repository_id ) if invalid_file_tups: - message = tool_util.generate_message_for_invalid_tools( trans.app, invalid_file_tups, repository, None, as_html=False ) + message = tool_util.generate_message_for_invalid_tools( app, + invalid_file_tups, + repository, + None, + as_html=False ) log.debug( message ) unsuccessful_count += 1 else: @@ -2036,10 +2056,11 @@ except: log.exception( "Error attempting to reset metadata on repository %s", str( repository.name ) ) unsuccessful_count += 1 - message = "Successfully reset metadata on %d %s. " % ( successful_count, inflector.cond_plural( successful_count, "repository" ) ) + message = "Successfully reset metadata on %d %s. " % \ + ( successful_count, inflector.cond_plural( successful_count, "repository" ) ) if unsuccessful_count: - message += "Error setting metadata on %d %s - see the paster log for details. " % ( unsuccessful_count, - inflector.cond_plural( unsuccessful_count, "repository" ) ) + message += "Error setting metadata on %d %s - see the paster log for details. " % \ + ( unsuccessful_count, inflector.cond_plural( unsuccessful_count, "repository" ) ) else: message = 'Select at least one repository to on which to reset all metadata.' status = 'error' Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 15 Jul '14

15 Jul '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/47d7c03dc46f/ Changeset: 47d7c03dc46f User: davebgx Date: 2014-07-15 22:13:28 Summary: Fix rare scenario where installing a set of repositories that depend on each other would result in one of them being installed twice. Affected #: 1 file diff -r 1b1748573103faca069962324a96ded3c801bc7a -r 47d7c03dc46f5fad021af94c44d377192dc534f7 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -1091,9 +1091,10 @@ tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) - ordered_tsr_ids.append( tsr_id ) - ordered_repo_info_dicts.append( repo_info_dict ) - ordered_tool_panel_section_keys.append( tool_panel_section_key ) + if tsr_id not in ordered_tsr_ids: + ordered_tsr_ids.append( tsr_id ) + ordered_repo_info_dicts.append( repo_info_dict ) + ordered_tool_panel_section_keys.append( tool_panel_section_key ) return ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys def populate_containers_dict_for_new_install( self, tool_shed_url, tool_path, readme_files_dict, installed_repository_dependencies, https://bitbucket.org/galaxy/galaxy-central/commits/0874eb058a0c/ Changeset: 0874eb058a0c User: davebgx Date: 2014-07-15 22:30:35 Summary: Fix display of job stdio when using the twill interactor. Affected #: 2 files diff -r 47d7c03dc46f5fad021af94c44d377192dc534f7 -r 0874eb058a0cb79508672cc5ebf25a4ce5063d75 test/base/interactor.py --- a/test/base/interactor.py +++ b/test/base/interactor.py @@ -386,7 +386,8 @@ self.twill_test_case.verify_dataset_correctness( outfile, hid=hid, attributes=attributes, shed_tool_id=shed_tool_id, maxseconds=maxseconds ) def get_job_stream( self, history_id, output_data, stream ): - return self.twill_test_case._get_job_stream_output( output_data.get( 'id' ), stream=stream, format=False ) + data_id = self.twill_test_case.security.encode_id( output_data.get( 'id' ) ) + return self.twill_test_case._get_job_stream_output( data_id, stream=stream, format=False ) def stage_data_async( self, test_data, history, shed_tool_id, async=True ): name = test_data.get( 'name', None ) diff -r 47d7c03dc46f5fad021af94c44d377192dc534f7 -r 0874eb058a0cb79508672cc5ebf25a4ce5063d75 test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -2496,7 +2496,7 @@ return msg def _get_job_stream_output( self, hda_id, stream, format ): - self.visit_url( "/datasets/%s/%s" % ( self.security.encode_id( hda_id ), stream ) ) + self.visit_url( "/datasets/%s/%s" % ( hda_id, stream ) ) output = self.last_page() return self._format_stream( output, stream, format ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Visualization framework bug fixes + remove debugging statement.
by commits-noreply＠bitbucket.org 15 Jul '14

15 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1b1748573103/ Changeset: 1b1748573103 User: jgoecks Date: 2014-07-15 21:41:38 Summary: Visualization framework bug fixes + remove debugging statement. Affected #: 3 files diff -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 -r 1b1748573103faca069962324a96ded3c801bc7a lib/galaxy/visualization/data_providers/cigar.py --- a/lib/galaxy/visualization/data_providers/cigar.py +++ b/lib/galaxy/visualization/data_providers/cigar.py @@ -18,7 +18,7 @@ return read_seq, cigar # Set up position for reference, read. - ref_seq_pos = read_start + ref_seq_pos = read_start - ref_seq_start read_pos = 0 # Create new read sequence, cigar. diff -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 -r 1b1748573103faca069962324a96ded3c801bc7a lib/galaxy/webapps/galaxy/api/datasets.py --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -203,7 +203,8 @@ # get longer, this will need to be increased and/or a handle to the genomic data may be need # to be given to the data provider. region = self.app.genomes.reference( trans, dbkey=dataset.dbkey, chrom=chrom, - low=( int( low ) - 500 ), high=( int( high ) + 500 ) ) + low=( max( 0, int( low ) - 500 ) ), + high=( int( high ) + 500 ) ) # Get mean depth. if not indexer: @@ -211,7 +212,6 @@ stats = indexer.get_data( chrom, low, high, stats=True ) mean_depth = stats[ 'data' ][ 'mean' ] - # Get and return data from data_provider. result = data_provider.get_data( chrom, int( low ), int( high ), int( start_val ), int( max_vals ), ref_seq=region, mean_depth=mean_depth, **kwargs ) diff -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 -r 1b1748573103faca069962324a96ded3c801bc7a static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -1385,7 +1385,6 @@ // captures most (all?) state change that needs to be saved. var self = this; drawable.config.on('change', function() { - console.log(drawable.config.get_value('name') + " changed"); self.changed(); }); }, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add a new RelationBuilder class to handle building repository relationships, eliminating the use of the repository_dependency_util.py module, and rename the RepositoryDependencyManager class to be RepositoryDependencyInstallManager.
by commits-noreply＠bitbucket.org 15 Jul '14

15 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7628e1d9d8e2/ Changeset: 7628e1d9d8e2 User: greg Date: 2014-07-15 19:56:12 Summary: Add a new RelationBuilder class to handle building repository relationships, eliminating the use of the repository_dependency_util.py module, and rename the RepositoryDependencyManager class to be RepositoryDependencyInstallManager. Affected #: 13 files diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -22,7 +22,6 @@ from tool_shed.util import hg_util from tool_shed.util import metadata_util from tool_shed.util import readme_util -from tool_shed.util import repository_dependency_util from tool_shed.util import repository_maintenance_util from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util @@ -31,7 +30,7 @@ from tool_shed.galaxy_install import install_manager from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager import tool_shed.galaxy_install.grids.admin_toolshed_grids as admin_toolshed_grids -from tool_shed.galaxy_install.repository_dependencies.repository_dependency_manager import RepositoryDependencyManager +from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager log = logging.getLogger( __name__ ) @@ -1301,7 +1300,7 @@ Reinstall a tool shed repository that has been previously uninstalled, making sure to handle all repository and tool dependencies of the repository. """ - rdm = RepositoryDependencyManager( trans.app ) + rdim = repository_dependency_manager.RepositoryDependencyInstallManager( trans.app ) message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) repository_id = kwd[ 'id' ] @@ -1365,35 +1364,33 @@ else: # Entering this else block occurs only if the tool_shed_repository does not include any valid tools. if install_repository_dependencies: - repository_dependencies = rdm.get_repository_dependencies_for_installed_tool_shed_repository( trans.app, - tool_shed_repository ) + repository_dependencies = \ + rdim.get_repository_dependencies_for_installed_tool_shed_repository( trans.app, + tool_shed_repository ) else: repository_dependencies = None if metadata: tool_dependencies = metadata.get( 'tool_dependencies', None ) else: tool_dependencies = None - repo_info_dict = \ - repository_maintenance_util.create_repo_info_dict( app=trans.app, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.changeset_revision, - ctx_rev=ctx_rev, - repository_owner=tool_shed_repository.owner, - repository_name=tool_shed_repository.name, - repository=None, - repository_metadata=None, - tool_dependencies=tool_dependencies, - repository_dependencies=repository_dependencies ) + repo_info_dict = repository_maintenance_util.create_repo_info_dict( trans.app, + repository_clone_url=repository_clone_url, + changeset_revision=tool_shed_repository.changeset_revision, + ctx_rev=ctx_rev, + repository_owner=tool_shed_repository.owner, + repository_name=tool_shed_repository.name, + tool_dependencies=tool_dependencies, + repository_dependencies=repository_dependencies ) if repo_info_dict not in repo_info_dicts: repo_info_dicts.append( repo_info_dict ) # Make sure all tool_shed_repository records exist. created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts = \ - rdm.create_repository_dependency_objects( tool_path=tool_path, - tool_shed_url=tool_shed_url, - repo_info_dicts=repo_info_dicts, - install_repository_dependencies=install_repository_dependencies, - no_changes_checked=no_changes_checked, - tool_panel_section_id=tool_panel_section_id ) + rdim.create_repository_dependency_objects( tool_path=tool_path, + tool_shed_url=tool_shed_url, + repo_info_dicts=repo_info_dicts, + install_repository_dependencies=install_repository_dependencies, + no_changes_checked=no_changes_checked, + tool_panel_section_id=tool_panel_section_id ) # Default the selected tool panel location for loading tools included in each newly installed required # tool shed repository to the location selected for the repository selected for re-installation. for index, tps_key in enumerate( tool_panel_section_keys ): @@ -1518,10 +1515,11 @@ @web.require_admin def reselect_tool_panel_section( self, trans, **kwd ): """ - Select or change the tool panel section to contain the tools included in the tool shed repository being reinstalled. If there are updates - available for the repository in the tool shed, the tool_dependencies and repository_dependencies associated with the updated changeset revision - will have been retrieved from the tool shed and passed in the received kwd. In this case, the stored tool shed repository metadata from the - Galaxy database will not be used since it is outdated. + Select or change the tool panel section to contain the tools included in the tool shed repository + being reinstalled. If there are updates available for the repository in the tool shed, the + tool_dependencies and repository_dependencies associated with the updated changeset revision will + have been retrieved from the tool shed and passed in the received kwd. In this case, the stored + tool shed repository metadata from the Galaxy database will not be used since it is outdated. """ message = '' status = 'done' @@ -1534,8 +1532,8 @@ tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( tool_shed_repository.tool_shed ) ) tool_path, relative_install_dir = tool_shed_repository.get_tool_relative_path( trans.app ) if latest_changeset_revision and latest_ctx_rev: - # There are updates available in the tool shed for the repository, so use the receieved dependency information which was retrieved from - # the tool shed. + # There are updates available in the tool shed for the repository, so use the received + # dependency information which was retrieved from the tool shed. encoded_updated_repo_info_dict = kwd.get( 'updated_repo_info_dict', None ) updated_repo_info_dict = encoding_util.tool_shed_decode( encoded_updated_repo_info_dict ) readme_files_dict = updated_repo_info_dict.get( 'readme_files_dict', None ) @@ -1584,20 +1582,18 @@ raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) readme_files_dict = json.from_json_string( raw_text ) tool_dependencies = metadata.get( 'tool_dependencies', None ) - rdm = RepositoryDependencyManager( trans.app ) - repository_dependencies = rdm.get_repository_dependencies_for_installed_tool_shed_repository( trans.app, - tool_shed_repository ) - repo_info_dict = \ - repository_maintenance_util.create_repo_info_dict( app=trans.app, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.installed_changeset_revision, - ctx_rev=tool_shed_repository.ctx_rev, - repository_owner=tool_shed_repository.owner, - repository_name=tool_shed_repository.name, - repository=None, - repository_metadata=None, - tool_dependencies=tool_dependencies, - repository_dependencies=repository_dependencies ) + rdim = repository_dependency_manager.RepositoryDependencyInstallManager( trans.app ) + repository_dependencies = \ + rdim.get_repository_dependencies_for_installed_tool_shed_repository( trans.app, + tool_shed_repository ) + repo_info_dict = repository_maintenance_util.create_repo_info_dict( trans.app, + repository_clone_url=repository_clone_url, + changeset_revision=tool_shed_repository.installed_changeset_revision, + ctx_rev=tool_shed_repository.ctx_rev, + repository_owner=tool_shed_repository.owner, + repository_name=tool_shed_repository.name, + tool_dependencies=tool_dependencies, + repository_dependencies=repository_dependencies ) irm = trans.app.installed_repository_manager dependencies_for_repository_dict = irm.get_dependencies_for_repository( tool_shed_url, repo_info_dict, diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -15,6 +15,7 @@ from galaxy.util import json from galaxy.model.orm import and_ from tool_shed.capsule import capsule_manager +from tool_shed.dependencies.repository import relation_builder from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import container_util @@ -22,7 +23,6 @@ from tool_shed.util import hg_util from tool_shed.util import metadata_util from tool_shed.util import readme_util -from tool_shed.util import repository_dependency_util from tool_shed.util import repository_maintenance_util from tool_shed.util import review_util from tool_shed.util import search_util @@ -1251,15 +1251,10 @@ return opened_archive repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) metadata = repository_metadata.metadata + toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) + rb = relation_builder.RelationBuilder( trans.app, repository, repository_metadata, toolshed_base_url ) # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. - repository_dependencies = \ - repository_dependency_util.get_repository_dependencies_for_changeset_revision( app=trans.app, - repository=repository, - repository_metadata=repository_metadata, - toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ), - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None ) + repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() if repository_dependencies: # Only display repository dependencies if they exist. exclude = [ 'datatypes', 'invalid_repository_dependencies', 'invalid_tool_dependencies', 'invalid_tools', @@ -1777,15 +1772,9 @@ if repository_metadata: metadata = repository_metadata.metadata if metadata: - repository_dependencies = \ - repository_dependency_util.get_repository_dependencies_for_changeset_revision( app=trans.app, - repository=repository, - repository_metadata=repository_metadata, - toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ), - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None, - circular_repository_dependencies=None ) + toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) + rb = relation_builder.RelationBuilder( trans.app, repository, repository_metadata, toolshed_base_url ) + repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() if repository_dependencies: return encoding_util.tool_shed_encode( repository_dependencies ) return '' @@ -2413,14 +2402,9 @@ skip_tool_tests_checked = True metadata = repository_metadata.metadata # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. - repository_dependencies = \ - repository_dependency_util.get_repository_dependencies_for_changeset_revision( app=trans.app, - repository=repository, - repository_metadata=repository_metadata, - toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ), - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None ) + toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) + rb = relation_builder.RelationBuilder( trans.app, repository, repository_metadata, toolshed_base_url ) + repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() if str( repository.type ) != rt_util.REPOSITORY_SUITE_DEFINITION: # Handle messaging for resetting repository type to the optimal value. change_repository_type_message = rt_util.generate_message_for_repository_type_change( trans.app, @@ -2461,9 +2445,9 @@ repository_metadata ) heads = hg_util.get_repository_heads( repo ) deprecated_repository_dependency_tups = \ - repository_dependency_util.get_repository_dependency_tups_from_repository_metadata( trans.app, - repository_metadata, - deprecated_only=True ) + metadata_util.get_repository_dependency_tups_from_repository_metadata( trans.app, + repository_metadata, + deprecated_only=True ) return trans.fill_template( '/webapps/tool_shed/repository/manage_repository.mako', repo_name=repo_name, description=description, @@ -2614,14 +2598,8 @@ metadata = repository_metadata.metadata # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) - repository_dependencies = \ - repository_dependency_util.get_repository_dependencies_for_changeset_revision( app=trans.app, - repository=repository, - repository_metadata=repository_metadata, - toolshed_base_url=toolshed_base_url, - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None ) + rb = relation_builder.RelationBuilder( trans.app, repository, repository_metadata, toolshed_base_url ) + repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() if metadata: if 'repository_dependencies' in metadata and not repository_dependencies: # See if we have an invalid repository dependency definition or if the repository dependency is required @@ -2636,8 +2614,8 @@ invalid = True break if invalid: - message = repository_dependency_util.generate_message_for_invalid_repository_dependencies( metadata, - error_from_tuple=False ) + message = metadata_util.generate_message_for_invalid_repository_dependencies( metadata, + error_from_tuple=False ) status = 'error' else: repository_metadata_id = None @@ -3329,14 +3307,9 @@ if repository_metadata: metadata = repository_metadata.metadata # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. - repository_dependencies = \ - repository_dependency_util.get_repository_dependencies_for_changeset_revision( app=trans.app, - repository=repository, - repository_metadata=repository_metadata, - toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ), - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None ) + toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) + rb = relation_builder.RelationBuilder( trans.app, repository, repository_metadata, toolshed_base_url ) + repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() if str( repository.type ) != rt_util.TOOL_DEPENDENCY_DEFINITION: # Handle messaging for orphan tool dependency definitions. orphan_message = tool_dependency_util.generate_message_for_orphan_tool_dependencies( trans, repository, metadata ) diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -14,7 +14,6 @@ from tool_shed.util import commit_util from tool_shed.util import hg_util from tool_shed.util import metadata_util -from tool_shed.util import repository_dependency_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util @@ -303,8 +302,8 @@ status = 'error' # Handle messaging for invalid repository dependencies. invalid_repository_dependencies_message = \ - repository_dependency_util.generate_message_for_invalid_repository_dependencies( metadata_dict, - error_from_tuple=True ) + metadata_util.generate_message_for_invalid_repository_dependencies( metadata_dict, + error_from_tuple=True ) if invalid_repository_dependencies_message: message += invalid_repository_dependencies_message status = 'error' diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/tool_shed/capsule/capsule_manager.py --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -12,18 +12,19 @@ from galaxy.util import asbool from galaxy.util import CHUNK_SIZE from galaxy.util.odict import odict -from tool_shed.dependencies import dependency_manager +from tool_shed.dependencies.repository.relation_builder import RelationBuilder +from tool_shed.dependencies.dependency_manager import RepositoryDependencyAttributeHandler +from tool_shed.dependencies.dependency_manager import ToolDependencyAttributeHandler from tool_shed.util import basic_util from tool_shed.util import commit_util from tool_shed.util import common_util from tool_shed.util import encoding_util from tool_shed.util import hg_util from tool_shed.util import metadata_util -from tool_shed.util import repository_dependency_util from tool_shed.util import repository_maintenance_util from tool_shed.util import shed_util_common as suc from tool_shed.util import xml_util -from tool_shed.galaxy_install.repository_dependencies.repository_dependency_manager import RepositoryDependencyManager +from tool_shed.galaxy_install.repository_dependencies.repository_dependency_manager import RepositoryDependencyInstallManager log = logging.getLogger( __name__ ) @@ -129,8 +130,8 @@ return sub_elements def generate_repository_archive( self, repository, changeset_revision, work_dir ): - rdah = dependency_manager.RepositoryDependencyAttributeHandler( self.app, unpopulate=True ) - tdah = dependency_manager.ToolDependencyAttributeHandler( self.app, unpopulate=True ) + rdah = RepositoryDependencyAttributeHandler( self.app, unpopulate=True ) + tdah = ToolDependencyAttributeHandler( self.app, unpopulate=True ) file_type_str = basic_util.get_file_type_str( changeset_revision, self.file_type ) file_name = '%s-%s' % ( repository.name, file_type_str ) return_code, error_message = hg_util.archive_repository_revision( self.app, @@ -235,21 +236,15 @@ Return a list of dictionaries defining repositories that are required by the repository associated with self.repository_id. """ - rdm = RepositoryDependencyManager( self.app ) + rdim = RepositoryDependencyInstallManager( self.app ) repository = suc.get_repository_in_tool_shed( self.app, self.repository_id ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, self.repository_id, self.changeset_revision ) # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) - repository_dependencies = \ - repository_dependency_util.get_repository_dependencies_for_changeset_revision( app=self.app, - repository=self.repository, - repository_metadata=repository_metadata, - toolshed_base_url=toolshed_base_url, - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None ) + rb = RelationBuilder( self.app, repository, repository_metadata, toolshed_base_url ) + repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() repo = hg_util.get_repo_for_repository( self.app, repository=self.repository, repo_path=None, @@ -265,7 +260,7 @@ str( self.repository.user.username ), repository_dependencies, None ) - all_required_repo_info_dict = rdm.get_required_repo_info_dicts( self.tool_shed_url, [ repo_info_dict ] ) + all_required_repo_info_dict = rdim.get_required_repo_info_dicts( self.tool_shed_url, [ repo_info_dict ] ) all_repo_info_dicts = all_required_repo_info_dict.get( 'all_repo_info_dicts', [] ) return all_repo_info_dicts @@ -651,8 +646,8 @@ def import_repository_archive( self, repository, repository_archive_dict ): """Import a repository archive contained within a repository capsule.""" - rdah = dependency_manager.RepositoryDependencyAttributeHandler( self.app, unpopulate=False ) - tdah = dependency_manager.ToolDependencyAttributeHandler( self.app, unpopulate=False ) + rdah = RepositoryDependencyAttributeHandler( self.app, unpopulate=False ) + tdah = ToolDependencyAttributeHandler( self.app, unpopulate=False ) archive_file_name = repository_archive_dict.get( 'archive_file_name', None ) capsule_file_name = repository_archive_dict[ 'capsule_file_name' ] encoded_file_path = repository_archive_dict[ 'encoded_file_path' ] diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/tool_shed/dependencies/repository/relation_builder.py --- /dev/null +++ b/lib/tool_shed/dependencies/repository/relation_builder.py @@ -0,0 +1,497 @@ +import logging +import os + +from galaxy.util import asbool +from galaxy.util import listify +from tool_shed.util import common_util +from tool_shed.util import container_util +from tool_shed.util import hg_util +from tool_shed.util import metadata_util +from tool_shed.util import shed_util_common as suc + +log = logging.getLogger( __name__ ) + + +class RelationBuilder( object ): + + def __init__( self, app, repository, repository_metadata, tool_shed_url ): + self.all_repository_dependencies = {} + self.app = app + self.circular_repository_dependencies = [] + self.repository = repository + self.repository_metadata = repository_metadata + self.handled_key_rd_dicts = [] + self.key_rd_dicts_to_be_processed = [] + self.tool_shed_url = tool_shed_url + + def can_add_to_key_rd_dicts( self, key_rd_dict, key_rd_dicts ): + """Handle the case where an update to the changeset revision was done.""" + k = key_rd_dict.keys()[ 0 ] + rd = key_rd_dict[ k ] + partial_rd = rd[ 0:3 ] + for kr_dict in key_rd_dicts: + key = kr_dict.keys()[ 0 ] + if key == k: + repository_dependency = kr_dict[ key ] + if repository_dependency[ 0:3 ] == partial_rd: + return False + return True + + def filter_only_if_compiling_contained_td( self, key_rd_dict ): + """ + Return a copy of the received key_rd_dict with repository dependencies that are needed + only_if_compiling_contained_td filtered out of the list of repository dependencies for + each rd_key. + """ + filtered_key_rd_dict = {} + for rd_key, required_rd_tup in key_rd_dict.items(): + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( required_rd_tup ) + if not asbool( only_if_compiling_contained_td ): + filtered_key_rd_dict[ rd_key ] = required_rd_tup + return filtered_key_rd_dict + + def get_prior_installation_required_and_only_if_compiling_contained_td( self ): + """ + This method is called from the tool shed and never Galaxy. If self.all_repository_dependencies + contains a repository dependency tuple that is associated with self.repository, return the + value of the tuple's prior_installation_required component. + """ + cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url( self.tool_shed_url ) + if self.all_repository_dependencies: + for rd_key, rd_tups in self.all_repository_dependencies.items(): + if rd_key in [ 'root_key', 'description' ]: + continue + for rd_tup in rd_tups: + rd_toolshed, \ + rd_name, \ + rd_owner, \ + rd_changeset_revision, \ + rd_prior_installation_required, \ + rd_only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( rd_tup ) + cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed ) + if cleaned_rd_toolshed == cleaned_toolshed_base_url and \ + rd_name == self.repository.name and \ + rd_owner == self.repository.user.username and \ + rd_changeset_revision == self.repository_metadata.changeset_revision: + return rd_prior_installation_required, rd_only_if_compiling_contained_td + elif self.repository_metadata: + # Get the list of changeset revisions from the tool shed to which self.repository may be updated. + metadata = self.repository_metadata.metadata + current_changeset_revision = str( self.repository_metadata.changeset_revision ) + # Get the changeset revision to which the current value of required_repository_changeset_revision + # should be updated if it's not current. + text = suc.get_updated_changeset_revisions( self.app, + name=str( self.repository.name ), + owner=str( self.repository.user.username ), + changeset_revision=current_changeset_revision ) + if text: + valid_changeset_revisions = listify( text ) + if current_changeset_revision not in valid_changeset_revisions: + valid_changeset_revisions.append( current_changeset_revision ) + else: + valid_changeset_revisions = [ current_changeset_revision ] + repository_dependencies_dict = metadata[ 'repository_dependencies' ] + rd_tups = repository_dependencies_dict.get( 'repository_dependencies', [] ) + for rd_tup in rd_tups: + rd_toolshed, \ + rd_name, \ + rd_owner, \ + rd_changeset_revision, \ + rd_prior_installation_required, \ + rd_only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( rd_tup ) + cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed ) + if cleaned_rd_toolshed == cleaned_toolshed_base_url and \ + rd_name == self.repository.name and \ + rd_owner == self.repository.user.username and \ + rd_changeset_revision in valid_changeset_revisions: + return rd_prior_installation_required, rd_only_if_compiling_contained_td + # Default both prior_installation_required and only_if_compiling_contained_td to False. + return 'False', 'False' + + def get_key_for_repository_changeset_revision( self ): + # The received toolshed_base_url must include the port, but doesn't have to include the protocol. + prior_installation_required, only_if_compiling_contained_td = \ + self.get_prior_installation_required_and_only_if_compiling_contained_td() + # Create a key with the value of prior_installation_required defaulted to False. + key = container_util.generate_repository_dependencies_key_for_repository( self.tool_shed_url, + self.repository.name, + self.repository.user.username, + self.repository_metadata.changeset_revision, + prior_installation_required, + only_if_compiling_contained_td ) + return key + + def get_repository_dependencies_for_changeset_revision( self ): + """ + Return a dictionary of all repositories upon which the contents of self.repository_metadata + record depend. The dictionary keys are name-spaced values consisting of: + self.tool_shed_url/repository_name/repository_owner/changeset_revision + and the values are lists of repository_dependency tuples consisting of: + ( self.tool_shed_url, repository_name, repository_owner, changeset_revision ). + This method ensures that all required repositories to the nth degree are returned. + """ + # Assume the current repository does not have repository dependencies defined for it. + current_repository_key = None + metadata = self.repository_metadata.metadata + if metadata: + # The value of self.tool_shed_url must include the port, but doesn't have to include + # the protocol. + if 'repository_dependencies' in metadata: + current_repository_key = self.get_key_for_repository_changeset_revision() + repository_dependencies_dict = metadata[ 'repository_dependencies' ] + if not self.all_repository_dependencies: + self.initialize_all_repository_dependencies( current_repository_key, repository_dependencies_dict ) + # Handle the repository dependencies defined in the current repository, if any, and populate + # the various repository dependency objects for this round of processing. + current_repository_key_rd_dicts = \ + self.populate_repository_dependency_objects_for_processing( current_repository_key, + repository_dependencies_dict ) + if current_repository_key: + if current_repository_key_rd_dicts: + # There should be only a single current_repository_key_rd_dict in this list. + current_repository_key_rd_dict = current_repository_key_rd_dicts[ 0 ] + # Handle circular repository dependencies. + if not self.in_circular_repository_dependencies( current_repository_key_rd_dict ): + if current_repository_key in self.all_repository_dependencies: + self.handle_current_repository_dependency( current_repository_key ) + elif self.key_rd_dicts_to_be_processed: + self.handle_next_repository_dependency() + elif self.key_rd_dicts_to_be_processed: + self.handle_next_repository_dependency() + elif self.key_rd_dicts_to_be_processed: + self.handle_next_repository_dependency() + self.all_repository_dependencies = self.prune_invalid_repository_dependencies( self.all_repository_dependencies ) + return self.all_repository_dependencies + + def get_repository_dependency_as_key( self, repository_dependency ): + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( repository_dependency ) + return container_util.generate_repository_dependencies_key_for_repository( tool_shed, + name, + owner, + changeset_revision, + prior_installation_required, + only_if_compiling_contained_td ) + + def get_updated_changeset_revisions_for_repository_dependencies( self, key_rd_dicts ): + updated_key_rd_dicts = [] + for key_rd_dict in key_rd_dicts: + key = key_rd_dict.keys()[ 0 ] + repository_dependency = key_rd_dict[ key ] + rd_toolshed, \ + rd_name, \ + rd_owner, \ + rd_changeset_revision, \ + rd_prior_installation_required, \ + rd_only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( repository_dependency ) + if suc.tool_shed_is_this_tool_shed( rd_toolshed ): + repository = suc.get_repository_by_name_and_owner( self.app, rd_name, rd_owner ) + if repository: + repository_id = self.app.security.encode_id( repository.id ) + repository_metadata = \ + metadata_util.get_repository_metadata_by_repository_id_changeset_revision( self.app, + repository_id, + rd_changeset_revision ) + if repository_metadata: + # The repository changeset_revision is installable, so no updates are available. + new_key_rd_dict = {} + new_key_rd_dict[ key ] = repository_dependency + updated_key_rd_dicts.append( key_rd_dict ) + else: + # The repository changeset_revision is no longer installable, so see if there's been an update. + repo = hg_util.get_repo_for_repository( self.app, repository=repository, repo_path=None, create=False ) + changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, rd_changeset_revision ) + repository_metadata = \ + metadata_util.get_repository_metadata_by_repository_id_changeset_revision( self.app, + repository_id, + changeset_revision ) + if repository_metadata: + new_key_rd_dict = {} + new_key_rd_dict[ key ] = \ + [ rd_toolshed, \ + rd_name, \ + rd_owner, \ + repository_metadata.changeset_revision, \ + rd_prior_installation_required, \ + rd_only_if_compiling_contained_td ] + # We have the updated changset revision. + updated_key_rd_dicts.append( new_key_rd_dict ) + else: + repository_components_tuple = container_util.get_components_from_key( key ) + components_list = suc.extract_components_from_tuple( repository_components_tuple ) + toolshed, repository_name, repository_owner, repository_changeset_revision = components_list[ 0:4 ] + # For backward compatibility to the 12/20/12 Galaxy release. + if len( components_list ) == 4: + prior_installation_required = 'False' + rd_only_if_compiling_contained_td = 'False' + elif len( components_list ) == 5: + rd_only_if_compiling_contained_td = 'False' + message = "The revision %s defined for repository %s owned by %s is invalid, so repository " % \ + ( str( rd_changeset_revision ), str( rd_name ), str( rd_owner ) ) + message += "dependencies defined for repository %s will be ignored." % str( repository_name ) + log.debug( message ) + else: + repository_components_tuple = container_util.get_components_from_key( key ) + components_list = suc.extract_components_from_tuple( repository_components_tuple ) + toolshed, repository_name, repository_owner, repository_changeset_revision = components_list[ 0:4 ] + message = "The revision %s defined for repository %s owned by %s is invalid, so repository " % \ + ( str( rd_changeset_revision ), str( rd_name ), str( rd_owner ) ) + message += "dependencies defined for repository %s will be ignored." % str( repository_name ) + log.debug( message ) + return updated_key_rd_dicts + + def handle_circular_repository_dependency( self, repository_key, repository_dependency ): + all_repository_dependencies_root_key = self.all_repository_dependencies[ 'root_key' ] + repository_dependency_as_key = self.get_repository_dependency_as_key( repository_dependency ) + repository_key_as_repository_dependency = repository_key.split( container_util.STRSEP ) + self.update_circular_repository_dependencies( repository_key, + repository_dependency, + self.all_repository_dependencies[ repository_dependency_as_key ] ) + if all_repository_dependencies_root_key != repository_dependency_as_key: + self.all_repository_dependencies[ repository_key ] = [ repository_dependency ] + + def handle_current_repository_dependency( self, current_repository_key ): + current_repository_key_rd_dicts = [] + for rd in self.all_repository_dependencies[ current_repository_key ]: + rd_copy = [ str( item ) for item in rd ] + new_key_rd_dict = {} + new_key_rd_dict[ current_repository_key ] = rd_copy + current_repository_key_rd_dicts.append( new_key_rd_dict ) + if current_repository_key_rd_dicts: + self.handle_key_rd_dicts_for_repository( current_repository_key, current_repository_key_rd_dicts ) + return self.get_repository_dependencies_for_changeset_revision() + + def handle_key_rd_dicts_for_repository( self, current_repository_key, repository_key_rd_dicts ): + key_rd_dict = repository_key_rd_dicts.pop( 0 ) + repository_dependency = key_rd_dict[ current_repository_key ] + toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( repository_dependency ) + if suc.tool_shed_is_this_tool_shed( toolshed ): + required_repository = suc.get_repository_by_name_and_owner( self.app, name, owner ) + self.repository = required_repository + repository_id = self.app.security.encode_id( required_repository.id ) + required_repository_metadata = \ + metadata_util.get_repository_metadata_by_repository_id_changeset_revision( self.app, + repository_id, + changeset_revision ) + self.repository_metadata = required_repository_metadata + if required_repository_metadata: + # The required_repository_metadata changeset_revision is installable. + required_metadata = required_repository_metadata.metadata + if required_metadata: + for current_repository_key_rd_dict in repository_key_rd_dicts: + if not self.in_key_rd_dicts( current_repository_key_rd_dict, self.key_rd_dicts_to_be_processed ): + # Add the current repository_dependency into self.key_rd_dicts_to_be_processed. + self.key_rd_dicts_to_be_processed.append( current_repository_key_rd_dict ) + if not self.in_key_rd_dicts( key_rd_dict, self.handled_key_rd_dicts ): + # Add the current repository_dependency into self.handled_key_rd_dicts. + self.handled_key_rd_dicts.append( key_rd_dict ) + if self.in_key_rd_dicts( key_rd_dict, self.key_rd_dicts_to_be_processed ): + # Remove the current repository from self.key_rd_dicts_to_be_processed. + self.key_rd_dicts_to_be_processed = self.remove_from_key_rd_dicts( key_rd_dict, self.key_rd_dicts_to_be_processed ) + else: + # The repository is in a different tool shed, so build an url and send a request. + error_message = "Repository dependencies are currently supported only within the same Tool Shed. " + error_message += "Ignoring repository dependency definition for tool shed " + error_message += "%s, name %s, owner %s, changeset revision %s" % ( toolshed, name, owner, changeset_revision ) + log.debug( error_message ) + + def handle_next_repository_dependency( self ): + next_repository_key_rd_dict = self.key_rd_dicts_to_be_processed.pop( 0 ) + next_repository_key_rd_dicts = [ next_repository_key_rd_dict ] + next_repository_key = next_repository_key_rd_dict.keys()[ 0 ] + self.handle_key_rd_dicts_for_repository( next_repository_key, next_repository_key_rd_dicts ) + return self.get_repository_dependencies_for_changeset_revision() + + def in_all_repository_dependencies( self, repository_key, repository_dependency ): + """ + Return True if { repository_key : repository_dependency } is in self.all_repository_dependencies. + """ + for key, val in self.all_repository_dependencies.items(): + if key != repository_key: + continue + if repository_dependency in val: + return True + return False + + def in_circular_repository_dependencies( self, repository_key_rd_dict ): + """ + Return True if any combination of a circular dependency tuple is the key : value pair defined + in the received repository_key_rd_dict. This means that each circular dependency tuple is converted + into the key : value pair for comparison. + """ + for tup in self.circular_repository_dependencies: + rd_0, rd_1 = tup + rd_0_as_key = self.get_repository_dependency_as_key( rd_0 ) + rd_1_as_key = self.get_repository_dependency_as_key( rd_1 ) + if rd_0_as_key in repository_key_rd_dict and repository_key_rd_dict[ rd_0_as_key ] == rd_1: + return True + if rd_1_as_key in repository_key_rd_dict and repository_key_rd_dict[ rd_1_as_key ] == rd_0: + return True + return False + + def in_key_rd_dicts( self, key_rd_dict, key_rd_dicts ): + """Return True if key_rd_dict is contained in the list of key_rd_dicts.""" + k = key_rd_dict.keys()[ 0 ] + v = key_rd_dict[ k ] + for key_rd_dict in key_rd_dicts: + for key, val in key_rd_dict.items(): + if key == k and val == v: + return True + return False + + def initialize_all_repository_dependencies( self, current_repository_key, repository_dependencies_dict ): + """Initialize the self.all_repository_dependencies dictionary.""" + # It's safe to assume that current_repository_key in this case will have a value. + self.all_repository_dependencies[ 'root_key' ] = current_repository_key + self.all_repository_dependencies[ current_repository_key ] = [] + # Store the value of the 'description' key only once, the first time through this recursive method. + description = repository_dependencies_dict.get( 'description', None ) + self.all_repository_dependencies[ 'description' ] = description + + def is_circular_repository_dependency( self, repository_key, repository_dependency ): + """ + Return True if the received repository_dependency is a key in self.all_repository_dependencies + whose list of repository dependencies includes the received repository_key. + """ + repository_dependency_as_key = self.get_repository_dependency_as_key( repository_dependency ) + repository_key_as_repository_dependency = repository_key.split( container_util.STRSEP ) + for key, val in self.all_repository_dependencies.items(): + if key != repository_dependency_as_key: + continue + if repository_key_as_repository_dependency in val: + return True + return False + + def populate_repository_dependency_objects_for_processing( self, current_repository_key, repository_dependencies_dict ): + """ + The process that discovers all repository dependencies for a specified repository's changeset + revision uses this method to populate the following items for the current processing loop: + filtered_current_repository_key_rd_dicts, self.key_rd_dicts_to_be_processed, + self.handled_key_rd_dicts, self.all_repository_dependencies. Each processing loop may discover + more repository dependencies, so this method is repeatedly called until all repository + dependencies have been discovered. + """ + current_repository_key_rd_dicts = [] + filtered_current_repository_key_rd_dicts = [] + for rd_tup in repository_dependencies_dict[ 'repository_dependencies' ]: + new_key_rd_dict = {} + new_key_rd_dict[ current_repository_key ] = rd_tup + current_repository_key_rd_dicts.append( new_key_rd_dict ) + if current_repository_key_rd_dicts and current_repository_key: + # Remove all repository dependencies that point to a revision within its own repository. + current_repository_key_rd_dicts = \ + self.remove_ropository_dependency_reference_to_self( current_repository_key_rd_dicts ) + current_repository_key_rd_dicts = \ + self.get_updated_changeset_revisions_for_repository_dependencies( current_repository_key_rd_dicts ) + for key_rd_dict in current_repository_key_rd_dicts: + # Filter out repository dependencies that are required only if compiling the dependent + # repository's tool dependency. + key_rd_dict = self.filter_only_if_compiling_contained_td( key_rd_dict ) + if key_rd_dict: + is_circular = False + in_handled_key_rd_dicts = self.in_key_rd_dicts( key_rd_dict, self.handled_key_rd_dicts ) + in_key_rd_dicts_to_be_processed = self.in_key_rd_dicts( key_rd_dict, self.key_rd_dicts_to_be_processed ) + if not in_handled_key_rd_dicts and not in_key_rd_dicts_to_be_processed: + filtered_current_repository_key_rd_dicts.append( key_rd_dict ) + repository_dependency = key_rd_dict[ current_repository_key ] + if current_repository_key in self.all_repository_dependencies: + # Add all repository dependencies for the current repository into its entry + # in self.all_repository_dependencies. + all_repository_dependencies_val = self.all_repository_dependencies[ current_repository_key ] + if repository_dependency not in all_repository_dependencies_val: + all_repository_dependencies_val.append( repository_dependency ) + self.all_repository_dependencies[ current_repository_key ] = all_repository_dependencies_val + elif not self.in_all_repository_dependencies( current_repository_key, repository_dependency ): + # Handle circular repository dependencies. + if self.is_circular_repository_dependency( current_repository_key, repository_dependency ): + is_circular = True + self.handle_circular_repository_dependency( current_repository_key, repository_dependency ) + else: + self.all_repository_dependencies[ current_repository_key ] = [ repository_dependency ] + if not is_circular and self.can_add_to_key_rd_dicts( key_rd_dict, self.key_rd_dicts_to_be_processed ): + new_key_rd_dict = {} + new_key_rd_dict[ current_repository_key ] = repository_dependency + self.key_rd_dicts_to_be_processed.append( new_key_rd_dict ) + return filtered_current_repository_key_rd_dicts + + def prune_invalid_repository_dependencies( self, repository_dependencies ): + """ + Eliminate all invalid entries in the received repository_dependencies dictionary. An entry + is invalid if the value_list of the key/value pair is empty. This occurs when an invalid + combination of tool shed, name , owner, changeset_revision is used and a repository_metadata + record is not found. + """ + valid_repository_dependencies = {} + description = repository_dependencies.get( 'description', None ) + root_key = repository_dependencies.get( 'root_key', None ) + if root_key is None: + return valid_repository_dependencies + for key, value in repository_dependencies.items(): + if key in [ 'description', 'root_key' ]: + continue + if value: + valid_repository_dependencies[ key ] = value + if valid_repository_dependencies: + valid_repository_dependencies[ 'description' ] = description + valid_repository_dependencies[ 'root_key' ] = root_key + return valid_repository_dependencies + + def remove_from_key_rd_dicts( self, key_rd_dict, key_rd_dicts ): + """Eliminate the key_rd_dict from the list of key_rd_dicts if it is contained in the list.""" + k = key_rd_dict.keys()[ 0 ] + v = key_rd_dict[ k ] + clean_key_rd_dicts = [] + for krd_dict in key_rd_dicts: + key = krd_dict.keys()[ 0 ] + val = krd_dict[ key ] + if key == k and val == v: + continue + clean_key_rd_dicts.append( krd_dict ) + return clean_key_rd_dicts + + def remove_ropository_dependency_reference_to_self( self, key_rd_dicts ): + """Remove all repository dependencies that point to a revision within its own repository.""" + clean_key_rd_dicts = [] + key = key_rd_dicts[ 0 ].keys()[ 0 ] + repository_tup = key.split( container_util.STRSEP ) + rd_toolshed, \ + rd_name, \ + rd_owner, \ + rd_changeset_revision, \ + rd_prior_installation_required, \ + rd_only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( repository_tup ) + cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed ) + for key_rd_dict in key_rd_dicts: + k = key_rd_dict.keys()[ 0 ] + repository_dependency = key_rd_dict[ k ] + toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( repository_dependency ) + cleaned_toolshed = common_util.remove_protocol_from_tool_shed_url( toolshed ) + if cleaned_rd_toolshed == cleaned_toolshed and rd_name == name and rd_owner == owner: + debug_msg = "Removing repository dependency for repository %s owned by %s " % ( name, owner ) + debug_msg += 'since it refers to a revision within itself.' + log.debug( debug_msg ) + else: + new_key_rd_dict = {} + new_key_rd_dict[ key ] = repository_dependency + clean_key_rd_dicts.append( new_key_rd_dict ) + return clean_key_rd_dicts + + def update_circular_repository_dependencies( self, repository_key, repository_dependency, repository_dependencies ): + repository_dependency_as_key = self.get_repository_dependency_as_key( repository_dependency ) + repository_key_as_repository_dependency = repository_key.split( container_util.STRSEP ) + if repository_key_as_repository_dependency in repository_dependencies: + found = False + for tup in self.circular_repository_dependencies: + if repository_dependency in tup and repository_key_as_repository_dependency in tup: + # The circular dependency has already been included. + found = True + if not found: + new_circular_tup = [ repository_dependency, repository_key_as_repository_dependency ] + self.circular_repository_dependencies.append( new_circular_tup ) diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -25,7 +25,6 @@ from tool_shed.util import encoding_util from tool_shed.util import hg_util from tool_shed.util import metadata_util -from tool_shed.util import repository_dependency_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util @@ -35,7 +34,7 @@ from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import StepManager from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import TagManager -from tool_shed.galaxy_install.repository_dependencies.repository_dependency_manager import RepositoryDependencyManager +from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager log = logging.getLogger( __name__ ) @@ -612,15 +611,15 @@ tool_panel_section_id = installation_dict[ 'tool_panel_section_id' ] tool_path = installation_dict[ 'tool_path' ] tool_shed_url = installation_dict[ 'tool_shed_url' ] - rdm = RepositoryDependencyManager( self.app ) + rdim = repository_dependency_manager.RepositoryDependencyInstallManager( self.app ) created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts = \ - rdm.create_repository_dependency_objects( tool_path=tool_path, - tool_shed_url=tool_shed_url, - repo_info_dicts=repo_info_dicts, - install_repository_dependencies=install_repository_dependencies, - no_changes_checked=no_changes_checked, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) + rdim.create_repository_dependency_objects( tool_path=tool_path, + tool_shed_url=tool_shed_url, + repo_info_dicts=repo_info_dicts, + install_repository_dependencies=install_repository_dependencies, + no_changes_checked=no_changes_checked, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) return created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts def initiate_repository_installation( self, installation_dict ): diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/tool_shed/galaxy_install/installed_repository_manager.py --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -9,14 +9,13 @@ from tool_shed.util import container_util from tool_shed.util import data_manager_util from tool_shed.util import datatype_util -from tool_shed.util import repository_dependency_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util from tool_shed.util import xml_util from galaxy.model.orm import and_ -from tool_shed.galaxy_install.repository_dependencies.repository_dependency_manager import RepositoryDependencyManager +from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager log = logging.getLogger( __name__ ) @@ -229,7 +228,7 @@ Return dictionaries containing the sets of installed and missing tool dependencies and repository dependencies associated with the repository defined by the received repo_info_dict. """ - rdm = RepositoryDependencyManager( self.app ) + rdim = repository_dependency_manager.RepositoryDependencyInstallManager( self.app ) repository = None installed_rd = {} installed_td = {} @@ -261,7 +260,7 @@ installed_rd, missing_rd = \ self.get_installed_and_missing_repository_dependencies_for_new_or_updated_install( repo_info_tuple ) # Discover all repository dependencies and retrieve information for installing them. - all_repo_info_dict = rdm.get_required_repo_info_dicts( tool_shed_url, util.listify( repo_info_dict ) ) + all_repo_info_dict = rdim.get_required_repo_info_dicts( tool_shed_url, util.listify( repo_info_dict ) ) has_repository_dependencies = all_repo_info_dict.get( 'has_repository_dependencies', False ) has_repository_dependencies_only_if_compiling_contained_td = \ all_repo_info_dict.get( 'has_repository_dependencies_only_if_compiling_contained_td', False ) @@ -299,7 +298,7 @@ missing_td[ td_key ] = td_dict else: # We have a single repository with (possibly) no defined repository dependencies. - all_repo_info_dict = rdm.get_required_repo_info_dicts( tool_shed_url, util.listify( repo_info_dict ) ) + all_repo_info_dict = rdim.get_required_repo_info_dicts( tool_shed_url, util.listify( repo_info_dict ) ) has_repository_dependencies = all_repo_info_dict.get( 'has_repository_dependencies', False ) has_repository_dependencies_only_if_compiling_contained_td = \ all_repo_info_dict.get( 'has_repository_dependencies_only_if_compiling_contained_td', False ) diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/tool_shed/galaxy_install/repair_repository_manager.py --- a/lib/tool_shed/galaxy_install/repair_repository_manager.py +++ b/lib/tool_shed/galaxy_install/repair_repository_manager.py @@ -4,12 +4,11 @@ log = logging.getLogger( __name__ ) from tool_shed.galaxy_install import install_manager -from tool_shed.galaxy_install.repository_dependencies.repository_dependency_manager import RepositoryDependencyManager +from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager from tool_shed.util import common_util from tool_shed.util import container_util from tool_shed.util import shed_util_common as suc -from tool_shed.util import repository_dependency_util from tool_shed.util import repository_maintenance_util from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util @@ -56,7 +55,7 @@ issues with an installed repository that has installation problems somewhere in its dependency hierarchy. """ - rdm = RepositoryDependencyManager( self.app ) + rdim = repository_dependency_manager.RepositoryDependencyInstallManager( self.app ) tsr_ids = [] repo_info_dicts = [] tool_panel_section_keys = [] @@ -64,8 +63,8 @@ irm = install_manager.InstallRepositoryManager( self.app ) # Get a dictionary of all repositories upon which the contents of the current repository_metadata #record depend. - repository_dependencies_dict = rdm.get_repository_dependencies_for_installed_tool_shed_repository( self.app, - repository ) + repository_dependencies_dict = rdim.get_repository_dependencies_for_installed_tool_shed_repository( self.app, + repository ) if repository_dependencies_dict: # Generate the list of installed repositories from the information contained in the # repository_dependencies dictionary. @@ -75,14 +74,14 @@ # repaired in the required order. for installed_repository in installed_repositories: tsr_ids.append( self.app.security.encode_id( installed_repository.id ) ) - repo_info_dict, tool_panel_section_key = self.get_repo_info_dict_for_repair( rdm, + repo_info_dict, tool_panel_section_key = self.get_repo_info_dict_for_repair( rdim, installed_repository ) tool_panel_section_keys.append( tool_panel_section_key ) repo_info_dicts.append( repo_info_dict ) else: # The received repository has no repository dependencies. tsr_ids.append( self.app.security.encode_id( repository.id ) ) - repo_info_dict, tool_panel_section_key = self.get_repo_info_dict_for_repair( rdm, + repo_info_dict, tool_panel_section_key = self.get_repo_info_dict_for_repair( rdim, repository ) tool_panel_section_keys.append( tool_panel_section_key ) repo_info_dicts.append( repo_info_dict ) @@ -95,11 +94,11 @@ repair_dict[ 'ordered_tool_panel_section_keys' ] = ordered_tool_panel_section_keys return repair_dict - def get_repo_info_dict_for_repair( self, rdm, repository ): + def get_repo_info_dict_for_repair( self, rdim, repository ): tool_panel_section_key = None repository_clone_url = common_util.generate_clone_url_for_installed_repository( self.app, repository ) - repository_dependencies = rdm.get_repository_dependencies_for_installed_tool_shed_repository( self.app, - repository ) + repository_dependencies = rdim.get_repository_dependencies_for_installed_tool_shed_repository( self.app, + repository ) metadata = repository.metadata if metadata: tool_dependencies = metadata.get( 'tool_dependencies', None ) diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py --- a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py +++ b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py @@ -19,7 +19,7 @@ log = logging.getLogger( __name__ ) -class RepositoryDependencyManager( object ): +class RepositoryDependencyInstallManager( object ): def __init__( self, app ): self.app = app diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -607,6 +607,48 @@ tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) return '%s/%s/%s/%s' % ( tmp_url, guid_type, obj_id, version ) +def generate_message_for_invalid_repository_dependencies( metadata_dict, error_from_tuple=False ): + """Get or generate and return an error message associated with an invalid repository dependency.""" + message = '' + if metadata_dict: + if error_from_tuple: + # Return the error messages associated with a set of one or more invalid repository dependency tuples. + invalid_repository_dependencies_dict = metadata_dict.get( 'invalid_repository_dependencies', None ) + if invalid_repository_dependencies_dict is not None: + invalid_repository_dependencies = invalid_repository_dependencies_dict.get( 'invalid_repository_dependencies', [] ) + for repository_dependency_tup in invalid_repository_dependencies: + toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error = \ + common_util.parse_repository_dependency_tuple( repository_dependency_tup, contains_error=True ) + if error: + message += '%s ' % str( error ) + else: + # The complete dependency hierarchy could not be determined for a repository being installed into + # Galaxy. This is likely due to invalid repository dependency definitions, so we'll get them from + # the metadata and parse them for display in an error message. This will hopefully communicate the + # problem to the user in such a way that a resolution can be determined. + message += 'The complete dependency hierarchy could not be determined for this repository, so no required ' + message += 'repositories will not be installed. This is likely due to invalid repository dependency definitions. ' + repository_dependencies_dict = metadata_dict.get( 'repository_dependencies', None ) + if repository_dependencies_dict is not None: + rd_tups = repository_dependencies_dict.get( 'repository_dependencies', None ) + if rd_tups is not None: + message += 'Here are the attributes of the dependencies defined for this repository to help determine the ' + message += 'cause of this problem. ' + message += '<table cellpadding="2" cellspacing="2">' + message += '<tr><th>Tool shed</th><th>Repository name</th><th>Owner</th><th>Changeset revision</th>' + message += '<th>Prior install required</th></tr>' + for rd_tup in rd_tups: + tool_shed, name, owner, changeset_revision, pir, oicct = \ + common_util.parse_repository_dependency_tuple( rd_tup ) + if util.asbool( pir ): + pir_str = 'True' + else: + pir_str = '' + message += '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % \ + ( tool_shed, name, owner, changeset_revision, pir_str ) + message += '</table>' + return message + def generate_metadata_for_changeset_revision( app, repository, changeset_revision, repository_clone_url, shed_config_dict=None, relative_install_dir=None, repository_files_dir=None, resetting_all_metadata_on_repository=False, updating_installed_repository=False, @@ -1156,6 +1198,36 @@ relative_path_to_file = relative_path_to_file[ len( shed_config_dict.get( 'tool_path' ) ) + 1: ] return relative_path_to_file +def get_repository_dependency_tups_from_repository_metadata( app, repository_metadata, deprecated_only=False ): + """ + Return a list of of tuples defining repository objects required by the received repository. The returned + list defines the entire repository dependency tree. This method is called only from the Tool Shed. + """ + dependency_tups = [] + if repository_metadata is not None: + metadata = repository_metadata.metadata + if metadata: + repository_dependencies_dict = metadata.get( 'repository_dependencies', None ) + if repository_dependencies_dict is not None: + repository_dependency_tups = repository_dependencies_dict.get( 'repository_dependencies', None ) + if repository_dependency_tups is not None: + # The value of repository_dependency_tups is a list of repository dependency tuples like this: + # ['http://localhost:9009', 'package_samtools_0_1_18', 'devteam', 'ef37fc635cb9', 'False', 'False'] + for repository_dependency_tup in repository_dependency_tups: + toolshed, name, owner, changeset_revision, pir, oicct = \ + common_util.parse_repository_dependency_tuple( repository_dependency_tup ) + repository = suc.get_repository_by_name_and_owner( app, name, owner ) + if repository: + if deprecated_only: + if repository.deprecated: + dependency_tups.append( repository_dependency_tup ) + else: + dependency_tups.append( repository_dependency_tup ) + else: + log.debug( "Cannot locate repository %s owned by %s for inclusion in repository dependency tups." % \ + ( name, owner ) ) + return dependency_tups + def get_repository_metadata_by_id( app, id ): """Get repository metadata from the database""" sa_session = app.model.context.current diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/tool_shed/util/repository_dependency_util.py --- a/lib/tool_shed/util/repository_dependency_util.py +++ /dev/null @@ -1,609 +0,0 @@ -import json -import logging -import os - -from galaxy.util import asbool -from galaxy.util import listify - -import tool_shed.util.shed_util_common as suc -from tool_shed.util import common_util -from tool_shed.util import container_util -from tool_shed.util import encoding_util -from tool_shed.util import hg_util -from tool_shed.util import metadata_util -from tool_shed.util import tool_util - -log = logging.getLogger( __name__ ) - -def can_add_to_key_rd_dicts( key_rd_dict, key_rd_dicts ): - """Handle the case where an update to the changeset revision was done.""" - k = key_rd_dict.keys()[ 0 ] - rd = key_rd_dict[ k ] - partial_rd = rd[ 0:3 ] - for kr_dict in key_rd_dicts: - key = kr_dict.keys()[ 0 ] - if key == k: - repository_dependency = kr_dict[ key ] - if repository_dependency[ 0:3 ] == partial_rd: - return False - return True - -def generate_message_for_invalid_repository_dependencies( metadata_dict, error_from_tuple=False ): - """Get or generate and return an error message associated with an invalid repository dependency.""" - message = '' - if metadata_dict: - if error_from_tuple: - # Return the error messages associated with a set of one or more invalid repository dependency tuples. - invalid_repository_dependencies_dict = metadata_dict.get( 'invalid_repository_dependencies', None ) - if invalid_repository_dependencies_dict is not None: - invalid_repository_dependencies = invalid_repository_dependencies_dict.get( 'invalid_repository_dependencies', [] ) - for repository_dependency_tup in invalid_repository_dependencies: - toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error = \ - common_util.parse_repository_dependency_tuple( repository_dependency_tup, contains_error=True ) - if error: - message += '%s ' % str( error ) - else: - # The complete dependency hierarchy could not be determined for a repository being installed into - # Galaxy. This is likely due to invalid repository dependency definitions, so we'll get them from - # the metadata and parse them for display in an error message. This will hopefully communicate the - # problem to the user in such a way that a resolution can be determined. - message += 'The complete dependency hierarchy could not be determined for this repository, so no required ' - message += 'repositories will not be installed. This is likely due to invalid repository dependency definitions. ' - repository_dependencies_dict = metadata_dict.get( 'repository_dependencies', None ) - if repository_dependencies_dict is not None: - rd_tups = repository_dependencies_dict.get( 'repository_dependencies', None ) - if rd_tups is not None: - message += 'Here are the attributes of the dependencies defined for this repository to help determine the ' - message += 'cause of this problem. ' - message += '<table cellpadding="2" cellspacing="2">' - message += '<tr><th>Tool shed</th><th>Repository name</th><th>Owner</th><th>Changeset revision</th>' - message += '<th>Prior install required</th></tr>' - for rd_tup in rd_tups: - tool_shed, name, owner, changeset_revision, pir, oicct = \ - common_util.parse_repository_dependency_tuple( rd_tup ) - if asbool( pir ): - pir_str = 'True' - else: - pir_str = '' - message += '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % \ - ( tool_shed, name, owner, changeset_revision, pir_str ) - message += '</table>' - return message - -def get_key_for_repository_changeset_revision( app, toolshed_base_url, repository, repository_metadata, all_repository_dependencies ): - # The received toolshed_base_url must include the port, but doesn't have to include the protocol. - prior_installation_required, only_if_compiling_contained_td = \ - get_prior_installation_required_and_only_if_compiling_contained_td( app, - toolshed_base_url, - repository, - repository_metadata, - all_repository_dependencies ) - # Create a key with the value of prior_installation_required defaulted to False. - key = container_util.generate_repository_dependencies_key_for_repository( toolshed_base_url=toolshed_base_url, - repository_name=repository.name, - repository_owner=repository.user.username, - changeset_revision=repository_metadata.changeset_revision, - prior_installation_required=prior_installation_required, - only_if_compiling_contained_td=only_if_compiling_contained_td ) - return key - -def get_prior_installation_required_and_only_if_compiling_contained_td( app, toolshed_base_url, repository, repository_metadata, - all_repository_dependencies ): - """ - This method is called from the tool shed and never Galaxy. If all_repository_dependencies contains - a repository dependency tuple that is associated with the received repository, return the value of - the tuple's prior_installation_required component. - """ - cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url( toolshed_base_url ) - if all_repository_dependencies: - for rd_key, rd_tups in all_repository_dependencies.items(): - if rd_key in [ 'root_key', 'description' ]: - continue - for rd_tup in rd_tups: - rd_toolshed, rd_name, rd_owner, rd_changeset_revision, rd_prior_installation_required, rd_only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( rd_tup ) - cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed ) - if cleaned_rd_toolshed == cleaned_toolshed_base_url and \ - rd_name == repository.name and \ - rd_owner == repository.user.username and \ - rd_changeset_revision == repository_metadata.changeset_revision: - return rd_prior_installation_required, rd_only_if_compiling_contained_td - elif repository_metadata: - # Get the list of changeset revisions from the tool shed to which the repository may be updated. - metadata = repository_metadata.metadata - current_changeset_revision = str( repository_metadata.changeset_revision ) - # Get the changeset revision to which the current value of required_repository_changeset_revision should be updated if it's not current. - text = suc.get_updated_changeset_revisions( app, - name=str( repository.name ), - owner=str( repository.user.username ), - changeset_revision=current_changeset_revision ) - if text: - valid_changeset_revisions = listify( text ) - if current_changeset_revision not in valid_changeset_revisions: - valid_changeset_revisions.append( current_changeset_revision ) - else: - valid_changeset_revisions = [ current_changeset_revision ] - repository_dependencies_dict = metadata[ 'repository_dependencies' ] - rd_tups = repository_dependencies_dict.get( 'repository_dependencies', [] ) - for rd_tup in rd_tups: - rd_toolshed, rd_name, rd_owner, rd_changeset_revision, rd_prior_installation_required, rd_only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( rd_tup ) - cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed ) - if cleaned_rd_toolshed == cleaned_toolshed_base_url and \ - rd_name == repository.name and \ - rd_owner == repository.user.username and \ - rd_changeset_revision in valid_changeset_revisions: - return rd_prior_installation_required, rd_only_if_compiling_contained_td - # Default both prior_installation_required and only_if_compiling_contained_td to False. - return 'False', 'False' - -def get_repository_dependency_as_key( repository_dependency ): - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( repository_dependency ) - return container_util.generate_repository_dependencies_key_for_repository( tool_shed, - name, - owner, - changeset_revision, - prior_installation_required, - only_if_compiling_contained_td ) - -def get_repository_dependencies_for_changeset_revision( app, repository, repository_metadata, toolshed_base_url, - key_rd_dicts_to_be_processed=None, all_repository_dependencies=None, - handled_key_rd_dicts=None, circular_repository_dependencies=None ): - """ - Return a dictionary of all repositories upon which the contents of the received - repository_metadata record depend. The dictionary keys are name-spaced values - consisting of: - toolshed_base_url/repository_name/repository_owner/changeset_revision - and the values are lists of repository_dependency tuples consisting of: - ( toolshed_base_url, repository_name, repository_owner, changeset_revision ). - This method ensures that all required repositories to the nth degree are returned. - """ - if handled_key_rd_dicts is None: - handled_key_rd_dicts = [] - if all_repository_dependencies is None: - all_repository_dependencies = {} - if key_rd_dicts_to_be_processed is None: - key_rd_dicts_to_be_processed = [] - if circular_repository_dependencies is None: - circular_repository_dependencies = [] - # Assume the current repository does not have repository dependencies defined for it. - current_repository_key = None - metadata = repository_metadata.metadata - if metadata: - # The value of the received toolshed_base_url must include the port, but doesn't have - # to include the protocol. - if 'repository_dependencies' in metadata: - current_repository_key = get_key_for_repository_changeset_revision( app, - toolshed_base_url, - repository, - repository_metadata, - all_repository_dependencies ) - repository_dependencies_dict = metadata[ 'repository_dependencies' ] - if not all_repository_dependencies: - all_repository_dependencies = initialize_all_repository_dependencies( current_repository_key, - repository_dependencies_dict, - all_repository_dependencies ) - # Handle the repository dependencies defined in the current repository, if any, and populate - # the various repository dependency objects for this round of processing. - current_repository_key_rd_dicts, key_rd_dicts_to_be_processed, handled_key_rd_dicts, all_repository_dependencies = \ - populate_repository_dependency_objects_for_processing( app, - current_repository_key, - repository_dependencies_dict, - key_rd_dicts_to_be_processed, - handled_key_rd_dicts, - circular_repository_dependencies, - all_repository_dependencies ) - if current_repository_key: - if current_repository_key_rd_dicts: - # There should be only a single current_repository_key_rd_dict in this list. - current_repository_key_rd_dict = current_repository_key_rd_dicts[ 0 ] - # Handle circular repository dependencies. - if not in_circular_repository_dependencies( current_repository_key_rd_dict, - circular_repository_dependencies ): - if current_repository_key in all_repository_dependencies: - handle_current_repository_dependency( app, - current_repository_key, - key_rd_dicts_to_be_processed, - all_repository_dependencies, - handled_key_rd_dicts, - circular_repository_dependencies ) - elif key_rd_dicts_to_be_processed: - handle_next_repository_dependency( app, - key_rd_dicts_to_be_processed, - all_repository_dependencies, - handled_key_rd_dicts, - circular_repository_dependencies ) - elif key_rd_dicts_to_be_processed: - handle_next_repository_dependency( app, - key_rd_dicts_to_be_processed, - all_repository_dependencies, - handled_key_rd_dicts, - circular_repository_dependencies ) - elif key_rd_dicts_to_be_processed: - handle_next_repository_dependency( app, - key_rd_dicts_to_be_processed, - all_repository_dependencies, - handled_key_rd_dicts, - circular_repository_dependencies ) - all_repository_dependencies = prune_invalid_repository_dependencies( all_repository_dependencies ) - return all_repository_dependencies - -def get_repository_dependency_tups_from_repository_metadata( app, repository_metadata, deprecated_only=False ): - """ - Return a list of of tuples defining repository objects required by the received repository. The returned - list defines the entire repository dependency tree. This method is called only from the Tool Shed. - """ - dependency_tups = [] - if repository_metadata is not None: - metadata = repository_metadata.metadata - if metadata: - repository_dependencies_dict = metadata.get( 'repository_dependencies', None ) - if repository_dependencies_dict is not None: - repository_dependency_tups = repository_dependencies_dict.get( 'repository_dependencies', None ) - if repository_dependency_tups is not None: - # The value of repository_dependency_tups is a list of repository dependency tuples like this: - # ['http://localhost:9009', 'package_samtools_0_1_18', 'devteam', 'ef37fc635cb9', 'False', 'False'] - for repository_dependency_tup in repository_dependency_tups: - toolshed, name, owner, changeset_revision, pir, oicct = \ - common_util.parse_repository_dependency_tuple( repository_dependency_tup ) - repository = suc.get_repository_by_name_and_owner( app, name, owner ) - if repository: - if deprecated_only: - if repository.deprecated: - dependency_tups.append( repository_dependency_tup ) - else: - dependency_tups.append( repository_dependency_tup ) - else: - log.debug( "Cannot locate repository %s owned by %s for inclusion in repository dependency tups." % \ - ( name, owner ) ) - return dependency_tups - -def get_updated_changeset_revisions_for_repository_dependencies( app, key_rd_dicts ): - updated_key_rd_dicts = [] - for key_rd_dict in key_rd_dicts: - key = key_rd_dict.keys()[ 0 ] - repository_dependency = key_rd_dict[ key ] - rd_toolshed, rd_name, rd_owner, rd_changeset_revision, rd_prior_installation_required, rd_only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( repository_dependency ) - if suc.tool_shed_is_this_tool_shed( rd_toolshed ): - repository = suc.get_repository_by_name_and_owner( app, rd_name, rd_owner ) - if repository: - repository_metadata = \ - metadata_util.get_repository_metadata_by_repository_id_changeset_revision( app, - app.security.encode_id( repository.id ), - rd_changeset_revision ) - if repository_metadata: - # The repository changeset_revision is installable, so no updates are available. - new_key_rd_dict = {} - new_key_rd_dict[ key ] = repository_dependency - updated_key_rd_dicts.append( key_rd_dict ) - else: - # The repository changeset_revision is no longer installable, so see if there's been an update. - repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) - changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, rd_changeset_revision ) - repository_metadata = \ - metadata_util.get_repository_metadata_by_repository_id_changeset_revision( app, - app.security.encode_id( repository.id ), - changeset_revision ) - if repository_metadata: - new_key_rd_dict = {} - new_key_rd_dict[ key ] = \ - [ rd_toolshed, rd_name, rd_owner, repository_metadata.changeset_revision, rd_prior_installation_required, rd_only_if_compiling_contained_td ] - # We have the updated changset revision. - updated_key_rd_dicts.append( new_key_rd_dict ) - else: - repository_components_tuple = container_util.get_components_from_key( key ) - components_list = suc.extract_components_from_tuple( repository_components_tuple ) - toolshed, repository_name, repository_owner, repository_changeset_revision = components_list[ 0:4 ] - # For backward compatibility to the 12/20/12 Galaxy release. - if len( components_list ) == 4: - prior_installation_required = 'False' - rd_only_if_compiling_contained_td = 'False' - elif len( components_list ) == 5: - rd_only_if_compiling_contained_td = 'False' - message = "The revision %s defined for repository %s owned by %s is invalid, so repository dependencies defined for repository %s will be ignored." % \ - ( str( rd_changeset_revision ), str( rd_name ), str( rd_owner ), str( repository_name ) ) - log.debug( message ) - else: - repository_components_tuple = container_util.get_components_from_key( key ) - components_list = suc.extract_components_from_tuple( repository_components_tuple ) - toolshed, repository_name, repository_owner, repository_changeset_revision = components_list[ 0:4 ] - message = "The revision %s defined for repository %s owned by %s is invalid, so repository dependencies defined for repository %s will be ignored." % \ - ( str( rd_changeset_revision ), str( rd_name ), str( rd_owner ), str( repository_name ) ) - log.debug( message ) - return updated_key_rd_dicts - -def handle_circular_repository_dependency( repository_key, repository_dependency, circular_repository_dependencies, - handled_key_rd_dicts, all_repository_dependencies ): - all_repository_dependencies_root_key = all_repository_dependencies[ 'root_key' ] - repository_dependency_as_key = get_repository_dependency_as_key( repository_dependency ) - repository_key_as_repository_dependency = repository_key.split( container_util.STRSEP ) - update_circular_repository_dependencies( repository_key, - repository_dependency, - all_repository_dependencies[ repository_dependency_as_key ], - circular_repository_dependencies ) - if all_repository_dependencies_root_key != repository_dependency_as_key: - all_repository_dependencies[ repository_key ] = [ repository_dependency ] - return circular_repository_dependencies, handled_key_rd_dicts, all_repository_dependencies - -def handle_current_repository_dependency( app, current_repository_key, key_rd_dicts_to_be_processed, all_repository_dependencies, handled_key_rd_dicts, - circular_repository_dependencies ): - current_repository_key_rd_dicts = [] - for rd in all_repository_dependencies[ current_repository_key ]: - rd_copy = [ str( item ) for item in rd ] - new_key_rd_dict = {} - new_key_rd_dict[ current_repository_key ] = rd_copy - current_repository_key_rd_dicts.append( new_key_rd_dict ) - if current_repository_key_rd_dicts: - toolshed, required_repository, required_repository_metadata, repository_key_rd_dicts, key_rd_dicts_to_be_processed, handled_key_rd_dicts = \ - handle_key_rd_dicts_for_repository( app, - current_repository_key, - current_repository_key_rd_dicts, - key_rd_dicts_to_be_processed, - handled_key_rd_dicts, - circular_repository_dependencies ) - return get_repository_dependencies_for_changeset_revision( app=app, - repository=required_repository, - repository_metadata=required_repository_metadata, - toolshed_base_url=toolshed, - key_rd_dicts_to_be_processed=key_rd_dicts_to_be_processed, - all_repository_dependencies=all_repository_dependencies, - handled_key_rd_dicts=handled_key_rd_dicts, - circular_repository_dependencies=circular_repository_dependencies ) - -def handle_key_rd_dicts_for_repository( app, current_repository_key, repository_key_rd_dicts, key_rd_dicts_to_be_processed, handled_key_rd_dicts, circular_repository_dependencies ): - key_rd_dict = repository_key_rd_dicts.pop( 0 ) - repository_dependency = key_rd_dict[ current_repository_key ] - toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( repository_dependency ) - if suc.tool_shed_is_this_tool_shed( toolshed ): - required_repository = suc.get_repository_by_name_and_owner( app, name, owner ) - required_repository_metadata = \ - metadata_util.get_repository_metadata_by_repository_id_changeset_revision( app, - app.security.encode_id( required_repository.id ), - changeset_revision ) - if required_repository_metadata: - # The required_repository_metadata changeset_revision is installable. - required_metadata = required_repository_metadata.metadata - if required_metadata: - for current_repository_key_rd_dict in repository_key_rd_dicts: - if not in_key_rd_dicts( current_repository_key_rd_dict, key_rd_dicts_to_be_processed ): - key_rd_dicts_to_be_processed.append( current_repository_key_rd_dict ) - # Mark the current repository_dependency as handled_key_rd_dicts. - if not in_key_rd_dicts( key_rd_dict, handled_key_rd_dicts ): - handled_key_rd_dicts.append( key_rd_dict ) - # Remove the current repository from the list of repository_dependencies to be processed. - if in_key_rd_dicts( key_rd_dict, key_rd_dicts_to_be_processed ): - key_rd_dicts_to_be_processed = remove_from_key_rd_dicts( key_rd_dict, key_rd_dicts_to_be_processed ) - else: - # The repository is in a different tool shed, so build an url and send a request. - error_message = "Repository dependencies are currently supported only within the same Tool Shed. Ignoring repository dependency definition " - error_message += "for tool shed %s, name %s, owner %s, changeset revision %s" % ( toolshed, name, owner, changeset_revision ) - log.debug( error_message ) - return toolshed, required_repository, required_repository_metadata, repository_key_rd_dicts, key_rd_dicts_to_be_processed, handled_key_rd_dicts - -def handle_next_repository_dependency( app, key_rd_dicts_to_be_processed, all_repository_dependencies, handled_key_rd_dicts, - circular_repository_dependencies ): - next_repository_key_rd_dict = key_rd_dicts_to_be_processed.pop( 0 ) - next_repository_key_rd_dicts = [ next_repository_key_rd_dict ] - next_repository_key = next_repository_key_rd_dict.keys()[ 0 ] - toolshed, required_repository, required_repository_metadata, repository_key_rd_dicts, key_rd_dicts_to_be_processed, handled_key_rd_dicts = \ - handle_key_rd_dicts_for_repository( app, - next_repository_key, - next_repository_key_rd_dicts, - key_rd_dicts_to_be_processed, - handled_key_rd_dicts, - circular_repository_dependencies ) - return get_repository_dependencies_for_changeset_revision( app=app, - repository=required_repository, - repository_metadata=required_repository_metadata, - toolshed_base_url=toolshed, - key_rd_dicts_to_be_processed=key_rd_dicts_to_be_processed, - all_repository_dependencies=all_repository_dependencies, - handled_key_rd_dicts=handled_key_rd_dicts, - circular_repository_dependencies=circular_repository_dependencies ) - -def in_all_repository_dependencies( repository_key, repository_dependency, all_repository_dependencies ): - """Return True if { repository_key : repository_dependency } is in all_repository_dependencies.""" - for key, val in all_repository_dependencies.items(): - if key != repository_key: - continue - if repository_dependency in val: - return True - return False - -def in_circular_repository_dependencies( repository_key_rd_dict, circular_repository_dependencies ): - """ - Return True if any combination of a circular dependency tuple is the key : value pair defined - in the received repository_key_rd_dict. This means that each circular dependency tuple is converted - into the key : value pair for comparison. - """ - for tup in circular_repository_dependencies: - rd_0, rd_1 = tup - rd_0_as_key = get_repository_dependency_as_key( rd_0 ) - rd_1_as_key = get_repository_dependency_as_key( rd_1 ) - if rd_0_as_key in repository_key_rd_dict and repository_key_rd_dict[ rd_0_as_key ] == rd_1: - return True - if rd_1_as_key in repository_key_rd_dict and repository_key_rd_dict[ rd_1_as_key ] == rd_0: - return True - return False - -def initialize_all_repository_dependencies( current_repository_key, repository_dependencies_dict, all_repository_dependencies ): - """Initialize the all_repository_dependencies dictionary.""" - # It's safe to assume that current_repository_key in this case will have a value. - all_repository_dependencies[ 'root_key' ] = current_repository_key - all_repository_dependencies[ current_repository_key ] = [] - # Store the value of the 'description' key only once, the first time through this recursive method. - description = repository_dependencies_dict.get( 'description', None ) - all_repository_dependencies[ 'description' ] = description - return all_repository_dependencies - -def in_key_rd_dicts( key_rd_dict, key_rd_dicts ): - """Return True if key_rd_dict is contained in the list of key_rd_dicts.""" - k = key_rd_dict.keys()[ 0 ] - v = key_rd_dict[ k ] - for key_rd_dict in key_rd_dicts: - for key, val in key_rd_dict.items(): - if key == k and val == v: - return True - return False - -def is_circular_repository_dependency( repository_key, repository_dependency, all_repository_dependencies ): - """ - Return True if the received repository_dependency is a key in all_repository_dependencies whose list of repository dependencies - includes the received repository_key. - """ - repository_dependency_as_key = get_repository_dependency_as_key( repository_dependency ) - repository_key_as_repository_dependency = repository_key.split( container_util.STRSEP ) - for key, val in all_repository_dependencies.items(): - if key != repository_dependency_as_key: - continue - if repository_key_as_repository_dependency in val: - return True - return False - -def filter_only_if_compiling_contained_td( key_rd_dict ): - """ - Return a copy of the received key_rd_dict with repository dependencies that are needed - only_if_compiling_contained_td filtered out of the list of repository dependencies for - each rd_key. - """ - filtered_key_rd_dict = {} - for rd_key, required_rd_tup in key_rd_dict.items(): - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( required_rd_tup ) - if not asbool( only_if_compiling_contained_td ): - filtered_key_rd_dict[ rd_key ] = required_rd_tup - return filtered_key_rd_dict - -def populate_repository_dependency_objects_for_processing( app, current_repository_key, repository_dependencies_dict, - key_rd_dicts_to_be_processed, handled_key_rd_dicts, - circular_repository_dependencies, all_repository_dependencies ): - """ - The process that discovers all repository dependencies for a specified repository's - changeset revision uses this method to populate the following items for the current - processing loop: filtered_current_repository_key_rd_dicts, key_rd_dicts_to_be_processed, - handled_key_rd_dicts, all_repository_dependencies. Each processing loop may discover - more repository dependencies, so this method is repeatedly called until all repository - dependencies have been discovered. - """ - current_repository_key_rd_dicts = [] - filtered_current_repository_key_rd_dicts = [] - for rd_tup in repository_dependencies_dict[ 'repository_dependencies' ]: - new_key_rd_dict = {} - new_key_rd_dict[ current_repository_key ] = rd_tup - current_repository_key_rd_dicts.append( new_key_rd_dict ) - if current_repository_key_rd_dicts and current_repository_key: - # Remove all repository dependencies that point to a revision within its own repository. - current_repository_key_rd_dicts = remove_ropository_dependency_reference_to_self( current_repository_key_rd_dicts ) - current_repository_key_rd_dicts = \ - get_updated_changeset_revisions_for_repository_dependencies( app, current_repository_key_rd_dicts ) - for key_rd_dict in current_repository_key_rd_dicts: - # Filter out repository dependencies that are required only if compiling the dependent repository's tool dependency. - key_rd_dict = filter_only_if_compiling_contained_td( key_rd_dict ) - if key_rd_dict: - is_circular = False - if not in_key_rd_dicts( key_rd_dict, handled_key_rd_dicts ) and not in_key_rd_dicts( key_rd_dict, - key_rd_dicts_to_be_processed ): - filtered_current_repository_key_rd_dicts.append( key_rd_dict ) - repository_dependency = key_rd_dict[ current_repository_key ] - if current_repository_key in all_repository_dependencies: - # Add all repository dependencies for the current repository into its entry in all_repository_dependencies. - all_repository_dependencies_val = all_repository_dependencies[ current_repository_key ] - if repository_dependency not in all_repository_dependencies_val: - all_repository_dependencies_val.append( repository_dependency ) - all_repository_dependencies[ current_repository_key ] = all_repository_dependencies_val - elif not in_all_repository_dependencies( current_repository_key, repository_dependency, all_repository_dependencies ): - # Handle circular repository dependencies. - if is_circular_repository_dependency( current_repository_key, - repository_dependency, - all_repository_dependencies ): - is_circular = True - circular_repository_dependencies, handled_key_rd_dicts, all_repository_dependencies = \ - handle_circular_repository_dependency( current_repository_key, - repository_dependency, - circular_repository_dependencies, - handled_key_rd_dicts, - all_repository_dependencies ) - else: - all_repository_dependencies[ current_repository_key ] = [ repository_dependency ] - if not is_circular and can_add_to_key_rd_dicts( key_rd_dict, key_rd_dicts_to_be_processed ): - new_key_rd_dict = {} - new_key_rd_dict[ current_repository_key ] = repository_dependency - key_rd_dicts_to_be_processed.append( new_key_rd_dict ) - return filtered_current_repository_key_rd_dicts, key_rd_dicts_to_be_processed, handled_key_rd_dicts, all_repository_dependencies - -def prune_invalid_repository_dependencies( repository_dependencies ): - """ - Eliminate all invalid entries in the received repository_dependencies dictionary. An entry - is invalid if the value_list of the key/value pair is empty. This occurs when an invalid - combination of tool shed, name , owner, changeset_revision is used and a repository_metadata - record is not found. - """ - valid_repository_dependencies = {} - description = repository_dependencies.get( 'description', None ) - root_key = repository_dependencies.get( 'root_key', None ) - if root_key is None: - return valid_repository_dependencies - for key, value in repository_dependencies.items(): - if key in [ 'description', 'root_key' ]: - continue - if value: - valid_repository_dependencies[ key ] = value - if valid_repository_dependencies: - valid_repository_dependencies[ 'description' ] = description - valid_repository_dependencies[ 'root_key' ] = root_key - return valid_repository_dependencies - -def remove_from_key_rd_dicts( key_rd_dict, key_rd_dicts ): - """Eliminate the key_rd_dict from the list of key_rd_dicts if it is contained in the list.""" - k = key_rd_dict.keys()[ 0 ] - v = key_rd_dict[ k ] - clean_key_rd_dicts = [] - for krd_dict in key_rd_dicts: - key = krd_dict.keys()[ 0 ] - val = krd_dict[ key ] - if key == k and val == v: - continue - clean_key_rd_dicts.append( krd_dict ) - return clean_key_rd_dicts - -def remove_ropository_dependency_reference_to_self( key_rd_dicts ): - """Remove all repository dependencies that point to a revision within its own repository.""" - clean_key_rd_dicts = [] - key = key_rd_dicts[ 0 ].keys()[ 0 ] - repository_tup = key.split( container_util.STRSEP ) - rd_toolshed, rd_name, rd_owner, rd_changeset_revision, rd_prior_installation_required, rd_only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( repository_tup ) - cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed ) - for key_rd_dict in key_rd_dicts: - k = key_rd_dict.keys()[ 0 ] - repository_dependency = key_rd_dict[ k ] - toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( repository_dependency ) - cleaned_toolshed = common_util.remove_protocol_from_tool_shed_url( toolshed ) - if cleaned_rd_toolshed == cleaned_toolshed and rd_name == name and rd_owner == owner: - debug_msg = "Removing repository dependency for repository %s owned by %s " % ( name, owner ) - debug_msg += 'since it refers to a revision within itself.' - log.debug( debug_msg ) - else: - new_key_rd_dict = {} - new_key_rd_dict[ key ] = repository_dependency - clean_key_rd_dicts.append( new_key_rd_dict ) - return clean_key_rd_dicts - -def update_circular_repository_dependencies( repository_key, repository_dependency, repository_dependencies, - circular_repository_dependencies ): - repository_dependency_as_key = get_repository_dependency_as_key( repository_dependency ) - repository_key_as_repository_dependency = repository_key.split( container_util.STRSEP ) - if repository_key_as_repository_dependency in repository_dependencies: - found = False - for tup in circular_repository_dependencies: - if repository_dependency in tup and repository_key_as_repository_dependency in tup: - # The circular dependency has already been included. - found = True - if not found: - new_circular_tup = [ repository_dependency, repository_key_as_repository_dependency ] - circular_repository_dependencies.append( new_circular_tup ) - return circular_repository_dependencies diff -r 893a727a7b7cc83355bc977d034622da50af16e5 -r 7628e1d9d8e2900cda5fb90ff239a32c0e091df9 lib/tool_shed/util/repository_maintenance_util.py --- a/lib/tool_shed/util/repository_maintenance_util.py +++ b/lib/tool_shed/util/repository_maintenance_util.py @@ -2,10 +2,10 @@ import logging import os import re -import tool_shed.util.shed_util_common as suc +from tool_shed.dependencies.repository import relation_builder from tool_shed.util import common_util from tool_shed.util import hg_util -from tool_shed.util import repository_dependency_util +from tool_shed.util import shed_util_common as suc from galaxy import util from galaxy import web from galaxy.web.form_builder import build_select_field @@ -90,16 +90,9 @@ metadata = repository_metadata.metadata if metadata: tool_shed_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) + rb = relation_builder.RelationBuilder( app, repository, repository_metadata, tool_shed_url ) # Get a dictionary of all repositories upon which the contents of the received repository depends. - repository_dependencies = \ - repository_dependency_util.get_repository_dependencies_for_changeset_revision( app=app, - repository=repository, - repository_metadata=repository_metadata, - toolshed_base_url=tool_shed_url, - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None, - circular_repository_dependencies=None ) + repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() tool_dependencies = metadata.get( 'tool_dependencies', {} ) if tool_dependencies: new_tool_dependencies = {} Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0