1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b2cdaba8715e/ Changeset: b2cdaba8715e User: greg Date: 2014-07-22 19:46:40 Summary: Add a DataManagerHandler for Galaxy installs from the Tool Shed, eliminating the use of the data_manager_util.py module. Affected #: 5 files diff -r 09e86deab7932a72af7717bef9ec67ea70d694c9 -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -12,7 +12,6 @@ import tool_shed.repository_types.util as rt_util from tool_shed.util import common_util -from tool_shed.util import data_manager_util from tool_shed.util import datatype_util from tool_shed.util import encoding_util from tool_shed.util import hg_util @@ -30,6 +29,7 @@ from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager +from tool_shed.galaxy_install.tools import data_manager from tool_shed.galaxy_install.tools import tool_panel_manager from tool_shed.tools import tool_version_manager @@ -252,7 +252,8 @@ shed_tool_conf, uninstall=remove_from_disk_checked ) if tool_shed_repository.includes_data_managers: - data_manager_util.remove_from_data_manager( trans.app, tool_shed_repository ) + dmh = data_manager.DataManagerHandler( trans.app ) + dmh.remove_from_data_manager( tool_shed_repository ) if tool_shed_repository.includes_datatypes: # Deactivate proprietary datatypes. installed_repository_dict = datatype_util.load_installed_datatypes( trans.app, @@ -1933,7 +1934,8 @@ hg_util.update_repository( repo, latest_ctx_rev ) # Remove old Data Manager entries if repository.includes_data_managers: - data_manager_util.remove_from_data_manager( trans.app, repository ) + dmh = data_manager.DataManagerHandler( trans.app ) + dmh.remove_from_data_manager( repository ) # Update the repository metadata. tpm = tool_panel_manager.ToolPanelManager( trans.app ) irmm = InstalledRepositoryMetadataManager( trans.app, tpm ) @@ -1962,13 +1964,13 @@ new_install=False ) # Add new Data Manager entries if 'data_manager' in metadata_dict: - new_data_managers = data_manager_util.install_data_managers( trans.app, - trans.app.config.shed_data_manager_config_file, - metadata_dict, - repository.get_shed_config_dict( trans.app ), - os.path.join( relative_install_dir, name ), - repository, - repository_tools_tups ) + dmh = data_manager.DataManagerHandler( trans.app ) + new_data_managers = dmh.install_data_managers( trans.app.config.shed_data_manager_config_file, + metadata_dict, + repository.get_shed_config_dict( trans.app ), + os.path.join( relative_install_dir, name ), + repository, + repository_tools_tups ) if 'repository_dependencies' in metadata_dict or 'tool_dependencies' in metadata_dict: new_repository_dependencies_dict = metadata_dict.get( 'repository_dependencies', {} ) new_repository_dependencies = new_repository_dependencies_dict.get( 'repository_dependencies', [] ) diff -r 09e86deab7932a72af7717bef9ec67ea70d694c9 -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -18,7 +18,6 @@ from tool_shed.util import basic_util from tool_shed.util import common_util -from tool_shed.util import data_manager_util from tool_shed.util import datatype_util from tool_shed.util import encoding_util from tool_shed.util import hg_util @@ -33,6 +32,7 @@ from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import StepManager from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import TagManager +from tool_shed.galaxy_install.tools import data_manager from tool_shed.galaxy_install.tools import tool_panel_manager from tool_shed.tools import tool_version_manager @@ -575,13 +575,13 @@ tool_panel_dict=tool_panel_dict, new_install=True ) if 'data_manager' in metadata_dict: - new_data_managers = data_manager_util.install_data_managers( self.app, - self.app.config.shed_data_manager_config_file, - metadata_dict, - shed_config_dict, - relative_install_dir, - tool_shed_repository, - repository_tools_tups ) + dmh = data_manager.DataManagerHandler( self.app ) + new_data_managers = dmh.install_data_managers( self.app.config.shed_data_manager_config_file, + metadata_dict, + shed_config_dict, + relative_install_dir, + tool_shed_repository, + repository_tools_tups ) if 'datatypes' in metadata_dict: tool_shed_repository.status = self.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES if not tool_shed_repository.includes_datatypes: diff -r 09e86deab7932a72af7717bef9ec67ea70d694c9 -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 lib/tool_shed/galaxy_install/installed_repository_manager.py --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -7,7 +7,6 @@ from galaxy import util from tool_shed.util import common_util from tool_shed.util import container_util -from tool_shed.util import data_manager_util from tool_shed.util import datatype_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util @@ -16,6 +15,7 @@ from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager +from tool_shed.galaxy_install.tools import data_manager from tool_shed.galaxy_install.tools import tool_panel_manager log = logging.getLogger( __name__ ) @@ -99,13 +99,13 @@ tp, data_manager_relative_install_dir = repository.get_tool_relative_path( self.app ) # Hack to add repository.name here, which is actually the root of the installed repository data_manager_relative_install_dir = os.path.join( data_manager_relative_install_dir, repository.name ) - new_data_managers = data_manager_util.install_data_managers( self.app, - self.app.config.shed_data_manager_config_file, - metadata, - repository.get_shed_config_dict( self.app ), - data_manager_relative_install_dir, - repository, - repository_tools_tups ) + dmh = data_manager.DataManagerHandler( self.app ) + new_data_managers = dmh.install_data_managers( self.app.config.shed_data_manager_config_file, + metadata, + repository.get_shed_config_dict( self.app ), + data_manager_relative_install_dir, + repository, + repository_tools_tups ) self.install_model.context.add( repository ) self.install_model.context.flush() if repository.includes_datatypes: diff -r 09e86deab7932a72af7717bef9ec67ea70d694c9 -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 lib/tool_shed/galaxy_install/tools/data_manager.py --- /dev/null +++ b/lib/tool_shed/galaxy_install/tools/data_manager.py @@ -0,0 +1,150 @@ +import logging +import os + +from tool_shed.galaxy_install.tools import tool_panel_manager + +from tool_shed.util import xml_util + +log = logging.getLogger( __name__ ) + + +class DataManagerHandler( object ): + + def __init__( self, app ): + self.app = app + + def data_manager_config_elems_to_xml_file( self, config_elems, config_filename ): + """ + Persist the current in-memory list of config_elems to a file named by the value + of config_filename. + """ + fh = open( config_filename, 'wb' ) + fh.write( '<?xml version="1.0"?>\n<data_managers>\n' ) + for elem in config_elems: + fh.write( xml_util.xml_to_string( elem ) ) + fh.write( '</data_managers>\n' ) + fh.close() + + def install_data_managers( self, shed_data_manager_conf_filename, metadata_dict, shed_config_dict, + relative_install_dir, repository, repository_tools_tups ): + rval = [] + if 'data_manager' in metadata_dict: + tpm = tool_panel_manager.ToolPanelManager( self.app ) + repository_tools_by_guid = {} + for tool_tup in repository_tools_tups: + repository_tools_by_guid[ tool_tup[ 1 ] ] = dict( tool_config_filename=tool_tup[ 0 ], tool=tool_tup[ 2 ] ) + # Load existing data managers. + tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename ) + if tree is None: + return rval + config_elems = [ elem for elem in tree.getroot() ] + repo_data_manager_conf_filename = metadata_dict['data_manager'].get( 'config_filename', None ) + if repo_data_manager_conf_filename is None: + log.debug( "No data_manager_conf.xml file has been defined." ) + return rval + data_manager_config_has_changes = False + relative_repo_data_manager_dir = os.path.join( shed_config_dict.get( 'tool_path', '' ), relative_install_dir ) + repo_data_manager_conf_filename = os.path.join( relative_repo_data_manager_dir, repo_data_manager_conf_filename ) + tree, error_message = xml_util.parse_xml( repo_data_manager_conf_filename ) + if tree is None: + return rval + root = tree.getroot() + for elem in root: + if elem.tag == 'data_manager': + data_manager_id = elem.get( 'id', None ) + if data_manager_id is None: + log.error( "A data manager was defined that does not have an id and will not be installed:\n%s" % \ + xml_util.xml_to_string( elem ) ) + continue + data_manager_dict = metadata_dict['data_manager'].get( 'data_managers', {} ).get( data_manager_id, None ) + if data_manager_dict is None: + log.error( "Data manager metadata is not defined properly for '%s'." % ( data_manager_id ) ) + continue + guid = data_manager_dict.get( 'guid', None ) + if guid is None: + log.error( "Data manager guid '%s' is not set in metadata for '%s'." % ( guid, data_manager_id ) ) + continue + elem.set( 'guid', guid ) + tool_guid = data_manager_dict.get( 'tool_guid', None ) + if tool_guid is None: + log.error( "Data manager tool guid '%s' is not set in metadata for '%s'." % ( tool_guid, data_manager_id ) ) + continue + tool_dict = repository_tools_by_guid.get( tool_guid, None ) + if tool_dict is None: + log.error( "Data manager tool guid '%s' could not be found for '%s'. Perhaps the tool is invalid?" % \ + ( tool_guid, data_manager_id ) ) + continue + tool = tool_dict.get( 'tool', None ) + if tool is None: + log.error( "Data manager tool with guid '%s' could not be found for '%s'. Perhaps the tool is invalid?" % \ + ( tool_guid, data_manager_id ) ) + continue + tool_config_filename = tool_dict.get( 'tool_config_filename', None ) + if tool_config_filename is None: + log.error( "Data manager metadata is missing 'tool_config_file' for '%s'." % ( data_manager_id ) ) + continue + elem.set( 'shed_conf_file', shed_config_dict['config_filename'] ) + if elem.get( 'tool_file', None ) is not None: + del elem.attrib[ 'tool_file' ] #remove old tool_file info + tool_elem = tpm.generate_tool_elem( repository.tool_shed, + repository.name, + repository.installed_changeset_revision, + repository.owner, + tool_config_filename, + tool, + None ) + elem.insert( 0, tool_elem ) + data_manager = \ + self.app.data_managers.load_manager_from_elem( elem, + tool_path=shed_config_dict.get( 'tool_path', '' ), + replace_existing=True ) + if data_manager: + rval.append( data_manager ) + else: + log.warning( "Encountered unexpected element '%s':\n%s" % ( elem.tag, xml_util.xml_to_string( elem ) ) ) + config_elems.append( elem ) + data_manager_config_has_changes = True + # Persist the altered shed_data_manager_config file. + if data_manager_config_has_changes: + self.data_manager_config_elems_to_xml_file( config_elems, shed_data_manager_conf_filename ) + return rval + + def remove_from_data_manager( self, repository ): + metadata_dict = repository.metadata + if metadata_dict and 'data_manager' in metadata_dict: + shed_data_manager_conf_filename = self.app.config.shed_data_manager_config_file + tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename ) + if tree: + root = tree.getroot() + assert root.tag == 'data_managers', 'The file provided (%s) for removing data managers from is not a valid data manager xml file.' % ( shed_data_manager_conf_filename ) + guids = [ data_manager_dict.get( 'guid' ) for data_manager_dict in metadata_dict.get( 'data_manager', {} ).get( 'data_managers', {} ).itervalues() if 'guid' in data_manager_dict ] + load_old_data_managers_by_guid = {} + data_manager_config_has_changes = False + config_elems = [] + for elem in root: + # Match Data Manager elements by guid and installed_changeset_revision + elem_matches_removed_data_manager = False + if elem.tag == 'data_manager': + guid = elem.get( 'guid', None ) + if guid in guids: + tool_elem = elem.find( 'tool' ) + if tool_elem is not None: + installed_changeset_revision_elem = tool_elem.find( 'installed_changeset_revision' ) + if installed_changeset_revision_elem is not None: + if installed_changeset_revision_elem.text == repository.installed_changeset_revision: + elem_matches_removed_data_manager = True + else: + # This is a different version, which had been previously overridden + load_old_data_managers_by_guid[ guid ] = elem + if elem_matches_removed_data_manager: + data_manager_config_has_changes = True + else: + config_elems.append( elem ) + # Remove data managers from in memory + self.app.data_managers.remove_manager( guids ) + # Load other versions of any now uninstalled data managers, if any + for elem in load_old_data_managers_by_guid.itervalues(): + self.app.data_managers.load_manager_from_elem( elem ) + # Persist the altered shed_data_manager_config file. + if data_manager_config_has_changes: + self.data_manager_config_elems_to_xml_file( config_elems, shed_data_manager_conf_filename ) diff -r 09e86deab7932a72af7717bef9ec67ea70d694c9 -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 lib/tool_shed/util/data_manager_util.py --- a/lib/tool_shed/util/data_manager_util.py +++ /dev/null @@ -1,135 +0,0 @@ -import logging -import os - -from tool_shed.galaxy_install.tools import tool_panel_manager - -from tool_shed.util import shed_util_common as suc -from tool_shed.util import xml_util - -log = logging.getLogger( __name__ ) - -def data_manager_config_elems_to_xml_file( app, config_elems, config_filename ):#, shed_tool_conf_filename ): - # Persist the current in-memory list of config_elems to a file named by the value of config_filename. - fh = open( config_filename, 'wb' ) - fh.write( '<?xml version="1.0"?>\n<data_managers>\n' )#% ( shed_tool_conf_filename )) - for elem in config_elems: - fh.write( xml_util.xml_to_string( elem ) ) - fh.write( '</data_managers>\n' ) - fh.close() - -def install_data_managers( app, shed_data_manager_conf_filename, metadata_dict, shed_config_dict, relative_install_dir, repository, repository_tools_tups ): - rval = [] - if 'data_manager' in metadata_dict: - tpm = tool_panel_manager.ToolPanelManager( app ) - repository_tools_by_guid = {} - for tool_tup in repository_tools_tups: - repository_tools_by_guid[ tool_tup[ 1 ] ] = dict( tool_config_filename=tool_tup[ 0 ], tool=tool_tup[ 2 ] ) - # Load existing data managers. - tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename ) - if tree is None: - return rval - config_elems = [ elem for elem in tree.getroot() ] - repo_data_manager_conf_filename = metadata_dict['data_manager'].get( 'config_filename', None ) - if repo_data_manager_conf_filename is None: - log.debug( "No data_manager_conf.xml file has been defined." ) - return rval - data_manager_config_has_changes = False - relative_repo_data_manager_dir = os.path.join( shed_config_dict.get( 'tool_path', '' ), relative_install_dir ) - repo_data_manager_conf_filename = os.path.join( relative_repo_data_manager_dir, repo_data_manager_conf_filename ) - tree, error_message = xml_util.parse_xml( repo_data_manager_conf_filename ) - if tree is None: - return rval - root = tree.getroot() - for elem in root: - if elem.tag == 'data_manager': - data_manager_id = elem.get( 'id', None ) - if data_manager_id is None: - log.error( "A data manager was defined that does not have an id and will not be installed:\n%s" % ( xml_util.xml_to_string( elem ) ) ) - continue - data_manager_dict = metadata_dict['data_manager'].get( 'data_managers', {} ).get( data_manager_id, None ) - if data_manager_dict is None: - log.error( "Data manager metadata is not defined properly for '%s'." % ( data_manager_id ) ) - continue - guid = data_manager_dict.get( 'guid', None ) - if guid is None: - log.error( "Data manager guid '%s' is not set in metadata for '%s'." % ( guid, data_manager_id ) ) - continue - elem.set( 'guid', guid ) - tool_guid = data_manager_dict.get( 'tool_guid', None ) - if tool_guid is None: - log.error( "Data manager tool guid '%s' is not set in metadata for '%s'." % ( tool_guid, data_manager_id ) ) - continue - tool_dict = repository_tools_by_guid.get( tool_guid, None ) - if tool_dict is None: - log.error( "Data manager tool guid '%s' could not be found for '%s'. Perhaps the tool is invalid?" % ( tool_guid, data_manager_id ) ) - continue - tool = tool_dict.get( 'tool', None ) - if tool is None: - log.error( "Data manager tool with guid '%s' could not be found for '%s'. Perhaps the tool is invalid?" % ( tool_guid, data_manager_id ) ) - continue - tool_config_filename = tool_dict.get( 'tool_config_filename', None ) - if tool_config_filename is None: - log.error( "Data manager metadata is missing 'tool_config_file' for '%s'." % ( data_manager_id ) ) - continue - elem.set( 'shed_conf_file', shed_config_dict['config_filename'] ) - if elem.get( 'tool_file', None ) is not None: - del elem.attrib[ 'tool_file' ] #remove old tool_file info - tool_elem = tpm.generate_tool_elem( repository.tool_shed, - repository.name, - repository.installed_changeset_revision, - repository.owner, - tool_config_filename, - tool, - None ) - elem.insert( 0, tool_elem ) - data_manager = app.data_managers.load_manager_from_elem( elem, tool_path=shed_config_dict.get( 'tool_path', '' ), replace_existing=True ) - if data_manager: - rval.append( data_manager ) - else: - log.warning( "Encountered unexpected element '%s':\n%s" % ( elem.tag, xml_util.xml_to_string( elem ) ) ) - config_elems.append( elem ) - data_manager_config_has_changes = True - # Persist the altered shed_data_manager_config file. - if data_manager_config_has_changes: - data_manager_config_elems_to_xml_file( app, config_elems, shed_data_manager_conf_filename ) - return rval - -def remove_from_data_manager( app, repository ): - metadata_dict = repository.metadata - if metadata_dict and 'data_manager' in metadata_dict: - shed_data_manager_conf_filename = app.config.shed_data_manager_config_file - tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename ) - if tree: - root = tree.getroot() - assert root.tag == 'data_managers', 'The file provided (%s) for removing data managers from is not a valid data manager xml file.' % ( shed_data_manager_conf_filename ) - guids = [ data_manager_dict.get( 'guid' ) for data_manager_dict in metadata_dict.get( 'data_manager', {} ).get( 'data_managers', {} ).itervalues() if 'guid' in data_manager_dict ] - load_old_data_managers_by_guid = {} - data_manager_config_has_changes = False - config_elems = [] - for elem in root: - # Match Data Manager elements by guid and installed_changeset_revision - elem_matches_removed_data_manager = False - if elem.tag == 'data_manager': - guid = elem.get( 'guid', None ) - if guid in guids: - tool_elem = elem.find( 'tool' ) - if tool_elem is not None: - installed_changeset_revision_elem = tool_elem.find( 'installed_changeset_revision' ) - if installed_changeset_revision_elem is not None: - if installed_changeset_revision_elem.text == repository.installed_changeset_revision: - elem_matches_removed_data_manager = True - else: - # This is a different version, which had been previously overridden - load_old_data_managers_by_guid[ guid ] = elem - if elem_matches_removed_data_manager: - data_manager_config_has_changes = True - else: - config_elems.append( elem ) - # Remove data managers from in memory - app.data_managers.remove_manager( guids ) - # Load other versions of any now uninstalled data managers, if any - for elem in load_old_data_managers_by_guid.itervalues(): - app.data_managers.load_manager_from_elem( elem ) - # Persist the altered shed_data_manager_config file. - if data_manager_config_has_changes: - data_manager_config_elems_to_xml_file( app, config_elems, shed_data_manager_conf_filename ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.