1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/4df405e7a2af/ Changeset: 4df405e7a2af Branch: next-stable User: greg Date: 2013-05-28 18:24:52 Summary: Eliminate the use of Galaxy's elementree egg within the tool shed. Affected #: 18 files diff -r c666aa204369e3c36498e1d4211e1ec591c3aeeb -r 4df405e7a2aff50c4ef83806c55d01448227cc5d lib/tool_shed/galaxy_install/__init__.py --- a/lib/tool_shed/galaxy_install/__init__.py +++ b/lib/tool_shed/galaxy_install/__init__.py @@ -5,12 +5,7 @@ import tool_shed.util.shed_util_common import tool_shed.util.datatype_util from galaxy.model.orm import and_ - -from galaxy import eggs -import pkg_resources - -pkg_resources.require( 'elementtree' ) -from elementtree import ElementTree, ElementInclude +from tool_shed.util import xml_util log = logging.getLogger( __name__ ) @@ -25,13 +20,10 @@ self.installed_repository_dicts = [] def get_repository_install_dir( self, tool_shed_repository ): for tool_config in self.tool_configs: - try: - tree = ElementTree.parse( tool_config ) - except Exception, e: - log.debug( "Exception attempting to parse %s: %s" % ( str( tool_config ), str( e ) ) ) + tree, error_message = xml_util.parse_xml( tool_config ) + if tree is None: return None root = tree.getroot() - ElementInclude.include( root ) tool_path = root.get( 'tool_path', None ) if tool_path: ts = tool_shed.util.shed_util_common.clean_tool_shed_url( tool_shed_repository.tool_shed ) diff -r c666aa204369e3c36498e1d4211e1ec591c3aeeb -r 4df405e7a2aff50c4ef83806c55d01448227cc5d lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -38,50 +38,56 @@ self.proprietary_tool_confs = self.non_shed_tool_panel_configs self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems( latest_migration_script_number ) # Set the location where the repositories will be installed by retrieving the tool_path setting from migrated_tools_config. - tree = xml_util.parse_xml( migrated_tools_config ) - root = tree.getroot() - self.tool_path = root.get( 'tool_path' ) - print "Repositories will be installed into configured tool_path location ", str( self.tool_path ) - # Parse tool_shed_install_config to check each of the tools. - self.tool_shed_install_config = tool_shed_install_config - tree = xml_util.parse_xml( tool_shed_install_config ) - root = tree.getroot() - self.tool_shed = suc.clean_tool_shed_url( root.get( 'name' ) ) - self.repository_owner = common_util.REPOSITORY_OWNER - index, self.shed_config_dict = suc.get_shed_tool_conf_dict( app, self.migrated_tools_config ) - # Since tool migration scripts can be executed any number of times, we need to make sure the appropriate tools are defined in - # tool_conf.xml. If no tools associated with the migration stage are defined, no repositories will be installed on disk. - # The default behavior is that the tool shed is down. - tool_shed_accessible = False - tool_panel_configs = common_util.get_non_shed_tool_panel_configs( app ) - if tool_panel_configs: - # The missing_tool_configs_dict contents are something like: - # {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]} - tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, tool_panel_configs, latest_migration_script_number ) + tree, error_message = xml_util.parse_xml( migrated_tools_config ) + if tree is None: + print error_message else: - # It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but - # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in - # the database. - tool_shed_accessible = True - missing_tool_configs_dict = odict() - if tool_shed_accessible: - if len( self.proprietary_tool_confs ) == 1: - plural = '' - file_names = self.proprietary_tool_confs[ 0 ] + root = tree.getroot() + self.tool_path = root.get( 'tool_path' ) + print "Repositories will be installed into configured tool_path location ", str( self.tool_path ) + # Parse tool_shed_install_config to check each of the tools. + self.tool_shed_install_config = tool_shed_install_config + tree, error_message = xml_util.parse_xml( tool_shed_install_config ) + if tree is None: + print error_message else: - plural = 's' - file_names = ', '.join( self.proprietary_tool_confs ) - if missing_tool_configs_dict: - for repository_elem in root: - self.install_repository( repository_elem, install_dependencies ) - else: - message = "\nNo tools associated with migration stage %s are defined in your " % str( latest_migration_script_number ) - message += "file%s named %s,\nso no repositories will be installed on disk.\n" % ( plural, file_names ) - print message - else: - message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % str( latest_migration_script_number ) - message += "Try again later.\n" - print message + root = tree.getroot() + self.tool_shed = suc.clean_tool_shed_url( root.get( 'name' ) ) + self.repository_owner = common_util.REPOSITORY_OWNER + index, self.shed_config_dict = suc.get_shed_tool_conf_dict( app, self.migrated_tools_config ) + # Since tool migration scripts can be executed any number of times, we need to make sure the appropriate tools are defined in + # tool_conf.xml. If no tools associated with the migration stage are defined, no repositories will be installed on disk. + # The default behavior is that the tool shed is down. + tool_shed_accessible = False + tool_panel_configs = common_util.get_non_shed_tool_panel_configs( app ) + if tool_panel_configs: + # The missing_tool_configs_dict contents are something like: + # {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]} + tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, tool_panel_configs, latest_migration_script_number ) + else: + # It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but + # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in + # the database. + tool_shed_accessible = True + missing_tool_configs_dict = odict() + if tool_shed_accessible: + if len( self.proprietary_tool_confs ) == 1: + plural = '' + file_names = self.proprietary_tool_confs[ 0 ] + else: + plural = 's' + file_names = ', '.join( self.proprietary_tool_confs ) + if missing_tool_configs_dict: + for repository_elem in root: + self.install_repository( repository_elem, install_dependencies ) + else: + message = "\nNo tools associated with migration stage %s are defined in your " % str( latest_migration_script_number ) + message += "file%s named %s,\nso no repositories will be installed on disk.\n" % ( plural, file_names ) + print message + else: + message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % str( latest_migration_script_number ) + message += "Try again later.\n" + print message def get_guid( self, repository_clone_url, relative_install_dir, tool_config ): if self.shed_config_dict.get( 'tool_path' ): @@ -108,7 +114,9 @@ tools_xml_file_path = os.path.abspath( os.path.join( 'scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number ) ) # Parse the XML and load the file attributes for later checking against the integrated elements from self.proprietary_tool_confs. migrated_tool_configs = [] - tree = xml_util.parse_xml( tools_xml_file_path ) + tree, error_message = xml_util.parse_xml( tools_xml_file_path ) + if tree is None: + return [] root = tree.getroot() for elem in root: if elem.tag == 'repository': @@ -117,7 +125,9 @@ # Parse each file in self.proprietary_tool_confs and generate the integrated list of tool panel Elements that contain them. tool_panel_elems = [] for proprietary_tool_conf in self.proprietary_tool_confs: - tree = xml_util.parse_xml( proprietary_tool_conf ) + tree, error_message = xml_util.parse_xml( proprietary_tool_conf ) + if tree is None: + return [] root = tree.getroot() for elem in root: if elem.tag == 'tool': diff -r c666aa204369e3c36498e1d4211e1ec591c3aeeb -r 4df405e7a2aff50c4ef83806c55d01448227cc5d lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -19,6 +19,7 @@ from tool_shed.util import metadata_util from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util +from xml.etree import ElementTree as XmlET from galaxy import eggs import pkg_resources @@ -28,10 +29,6 @@ from mercurial import hg from mercurial import ui -pkg_resources.require( 'elementtree' ) -from elementtree import ElementTree -from elementtree.ElementTree import Element - log = logging.getLogger( __name__ ) def create_repo_info_dict( trans, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_name=None, repository=None, @@ -401,7 +398,7 @@ else: # Appending a new section to trans.app.toolbox.tool_panel log.debug( "Loading new tool panel section: %s" % new_tool_panel_section ) - elem = Element( 'section' ) + elem = XmlET.Element( 'section' ) elem.attrib[ 'name' ] = new_tool_panel_section elem.attrib[ 'id' ] = section_id elem.attrib[ 'version' ] = '' diff -r c666aa204369e3c36498e1d4211e1ec591c3aeeb -r 4df405e7a2aff50c4ef83806c55d01448227cc5d lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -14,15 +14,6 @@ from galaxy.model.orm import and_ from galaxy.web import url_for -from galaxy import eggs -import pkg_resources - -pkg_resources.require( 'elementtree' ) -from elementtree import ElementTree -from elementtree import ElementInclude -from elementtree.ElementTree import Element -from elementtree.ElementTree import SubElement - log = logging.getLogger( __name__ ) def clean_tool_shed_url( base_url ): @@ -479,7 +470,7 @@ tool_dependency = None action_dict = {} if tool_dependencies_config: - required_td_tree = xml_util.parse_xml( tool_dependencies_config ) + required_td_tree, error_message = xml_util.parse_xml( tool_dependencies_config ) if required_td_tree: required_td_root = required_td_tree.getroot() for required_td_elem in required_td_root: diff -r c666aa204369e3c36498e1d4211e1ec591c3aeeb -r 4df405e7a2aff50c4ef83806c55d01448227cc5d lib/tool_shed/tool_shed_registry.py --- a/lib/tool_shed/tool_shed_registry.py +++ b/lib/tool_shed/tool_shed_registry.py @@ -10,29 +10,32 @@ class Registry( object ): def __init__( self, root_dir=None, config=None ): - self.tool_sheds = odict() + self.tool_sheds = odict() self.tool_sheds_auth = odict() if root_dir and config: # Parse tool_sheds_conf.xml - tree = xml_util.parse_xml( config ) - root = tree.getroot() - log.debug( 'Loading references to tool sheds from %s' % config ) - for elem in root.findall( 'tool_shed' ): - try: - name = elem.get( 'name', None ) - url = elem.get( 'url', None ) - username = elem.get( 'user', None ) - password = elem.get( 'pass', None ) - if name and url: - self.tool_sheds[ name ] = url - self.tool_sheds_auth[ name ] = None - log.debug( 'Loaded reference to tool shed: %s' % name ) - if name and url and username and password: - pass_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() - pass_mgr.add_password(None, url, username, password) - self.tool_sheds_auth[ name ] = pass_mgr - except Exception, e: - log.warning( 'Error loading reference to tool shed "%s", problem: %s' % ( name, str( e ) ) ) + tree, error_message = xml_util.parse_xml( config ) + if tree is None: + log.warning( "Unable to load references to tool sheds defined in file %s" % str( config ) ) + else: + root = tree.getroot() + log.debug( 'Loading references to tool sheds from %s' % config ) + for elem in root.findall( 'tool_shed' ): + try: + name = elem.get( 'name', None ) + url = elem.get( 'url', None ) + username = elem.get( 'user', None ) + password = elem.get( 'pass', None ) + if name and url: + self.tool_sheds[ name ] = url + self.tool_sheds_auth[ name ] = None + log.debug( 'Loaded reference to tool shed: %s' % name ) + if name and url and username and password: + pass_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() + pass_mgr.add_password( None, url, username, password ) + self.tool_sheds_auth[ name ] = pass_mgr + except Exception, e: + log.warning( 'Error loading reference to tool shed "%s", problem: %s' % ( name, str( e ) ) ) def password_manager_for_url( self, url ): """ diff -r c666aa204369e3c36498e1d4211e1ec591c3aeeb -r 4df405e7a2aff50c4ef83806c55d01448227cc5d lib/tool_shed/util/commit_util.py --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -185,14 +185,11 @@ def handle_repository_dependencies_definition( trans, repository_dependencies_config ): altered = False - try: - # Make sure we're looking at a valid repository_dependencies.xml file. - tree = xml_util.parse_xml( repository_dependencies_config ) - root = tree.getroot() - except Exception, e: - error_message = "Error parsing %s in handle_repository_dependencies_definition: " % str( repository_dependencies_config ) - log.exception( error_message ) + # Make sure we're looking at a valid repository_dependencies.xml file. + tree, error_message = xml_util.parse_xml( repository_dependencies_config ) + if tree is None: return False, None + root = tree.getroot() if root.tag == 'repositories': for index, elem in enumerate( root ): # <repository name="molecule_datatypes" owner="test" changeset_revision="1a070566e9c6" /> @@ -232,14 +229,11 @@ def handle_tool_dependencies_definition( trans, tool_dependencies_config ): altered = False - try: - # Make sure we're looking at a valid tool_dependencies.xml file. - tree = xml_util.parse_xml( tool_dependencies_config ) - root = tree.getroot() - except Exception, e: - error_message = "Error parsing %s in handle_tool_dependencies_definition: " % str( tool_dependencies_config ) - log.exception( error_message ) + # Make sure we're looking at a valid tool_dependencies.xml file. + tree, error_message = xml_util.parse_xml( tool_dependencies_config ) + if tree is None: return False, None + root = tree.getroot() if root.tag == 'tool_dependency': for root_index, root_elem in enumerate( root ): # <package name="eigen" version="2.0.17"> diff -r c666aa204369e3c36498e1d4211e1ec591c3aeeb -r 4df405e7a2aff50c4ef83806c55d01448227cc5d lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -13,17 +13,10 @@ from tool_shed.util import data_manager_util from tool_shed.util import datatype_util from tool_shed.util import tool_util +from tool_shed.util import xml_util from tool_shed.galaxy_install.tool_dependencies.install_util import install_package from tool_shed.galaxy_install.tool_dependencies.install_util import set_environment -import pkg_resources - -pkg_resources.require( 'elementtree' ) -from elementtree import ElementTree -from elementtree import ElementInclude -from elementtree.ElementTree import Element -from elementtree.ElementTree import SubElement - log = logging.getLogger( __name__ ) def activate_repository( trans, repository ): @@ -318,13 +311,10 @@ sa_session = app.model.context.current installed_tool_dependencies = [] # Parse the tool_dependencies.xml config. - try: - tree = ElementTree.parse( tool_dependencies_config ) - except Exception, e: - log.debug( "Exception attempting to parse %s: %s" % ( str( tool_dependencies_config ), str( e ) ) ) + tree, error_message = xml_util.parse_xml( tool_dependencies_config ) + if tree is None: return installed_tool_dependencies root = tree.getroot() - ElementInclude.include( root ) fabric_version_checked = False for elem in root: if elem.tag == 'package': diff -r c666aa204369e3c36498e1d4211e1ec591c3aeeb -r 4df405e7a2aff50c4ef83806c55d01448227cc5d lib/tool_shed/util/common_util.py --- a/lib/tool_shed/util/common_util.py +++ b/lib/tool_shed/util/common_util.py @@ -11,7 +11,9 @@ tools_xml_file_path = os.path.abspath( os.path.join( 'scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number ) ) # Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config. migrated_tool_configs_dict = odict() - tree = xml_util.parse_xml( tools_xml_file_path ) + tree, error_message = xml_util.parse_xml( tools_xml_file_path ) + if tree is None: + return False, odict() root = tree.getroot() tool_shed = root.get( 'name' ) tool_shed_url = get_tool_shed_url_from_tools_xml_file_path( app, tool_shed ) @@ -48,15 +50,16 @@ # Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names. missing_tool_configs_dict = odict() for tool_panel_config in tool_panel_configs: - tree = xml_util.parse_xml( tool_panel_config ) - root = tree.getroot() - for elem in root: - if elem.tag == 'tool': - missing_tool_configs_dict = check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict ) - elif elem.tag == 'section': - for section_elem in elem: - if section_elem.tag == 'tool': - missing_tool_configs_dict = check_tool_tag_set( section_elem, migrated_tool_configs_dict, missing_tool_configs_dict ) + tree, error_message = xml_util.parse_xml( tool_panel_config ) + if tree: + root = tree.getroot() + for elem in root: + if elem.tag == 'tool': + missing_tool_configs_dict = check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict ) + elif elem.tag == 'section': + for section_elem in elem: + if section_elem.tag == 'tool': + missing_tool_configs_dict = check_tool_tag_set( section_elem, migrated_tool_configs_dict, missing_tool_configs_dict ) else: exception_msg = '\n\nThe entry for the main Galaxy tool shed at %s is missing from the %s file. ' % ( tool_shed, app.config.tool_sheds_config ) exception_msg += 'The entry for this tool shed must always be available in this file, so re-add it before attempting to start your Galaxy server.\n' @@ -78,7 +81,9 @@ for config_filename in app.config.tool_configs: # Any config file that includes a tool_path attribute in the root tag set like the following is shed-related. # <toolbox tool_path="../shed_tools"> - tree = xml_util.parse_xml( config_filename ) + tree, error_message = xml_util.parse_xml( config_filename ) + if tree is None: + continue root = tree.getroot() tool_path = root.get( 'tool_path', None ) if tool_path is None: diff -r c666aa204369e3c36498e1d4211e1ec591c3aeeb -r 4df405e7a2aff50c4ef83806c55d01448227cc5d lib/tool_shed/util/data_manager_util.py --- a/lib/tool_shed/util/data_manager_util.py +++ b/lib/tool_shed/util/data_manager_util.py @@ -21,7 +21,10 @@ for tool_tup in repository_tools_tups: repository_tools_by_guid[ tool_tup[ 1 ] ] = dict( tool_config_filename=tool_tup[ 0 ], tool=tool_tup[ 2 ] ) # Load existing data managers. - config_elems = [ elem for elem in xml_util.parse_xml( shed_data_manager_conf_filename ).getroot() ] + tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename ) + if tree is None: + return rval + config_elems = [ elem for elem in tree.getroot() ] repo_data_manager_conf_filename = metadata_dict['data_manager'].get( 'config_filename', None ) if repo_data_manager_conf_filename is None: log.debug( "No data_manager_conf.xml file has been defined." ) @@ -29,7 +32,9 @@ data_manager_config_has_changes = False relative_repo_data_manager_dir = os.path.join( shed_config_dict.get( 'tool_path', '' ), relative_install_dir ) repo_data_manager_conf_filename = os.path.join( relative_repo_data_manager_dir, repo_data_manager_conf_filename ) - tree = xml_util.parse_xml( repo_data_manager_conf_filename ) + tree, error_message = xml_util.parse_xml( repo_data_manager_conf_filename ) + if tree is None: + return rval root = tree.getroot() for elem in root: if elem.tag == 'data_manager': @@ -89,37 +94,38 @@ metadata_dict = repository.metadata if metadata_dict and 'data_manager' in metadata_dict: shed_data_manager_conf_filename = app.config.shed_data_manager_config_file - tree = xml_util.parse_xml( shed_data_manager_conf_filename ) - root = tree.getroot() - assert root.tag == 'data_managers', 'The file provided (%s) for removing data managers from is not a valid data manager xml file.' % ( shed_data_manager_conf_filename ) - guids = [ data_manager_dict.get( 'guid' ) for data_manager_dict in metadata_dict.get( 'data_manager', {} ).get( 'data_managers', {} ).itervalues() if 'guid' in data_manager_dict ] - load_old_data_managers_by_guid = {} - data_manager_config_has_changes = False - config_elems = [] - for elem in root: - # Match Data Manager elements by guid and installed_changeset_revision - elem_matches_removed_data_manager = False - if elem.tag == 'data_manager': - guid = elem.get( 'guid', None ) - if guid in guids: - tool_elem = elem.find( 'tool' ) - if tool_elem is not None: - installed_changeset_revision_elem = tool_elem.find( 'installed_changeset_revision' ) - if installed_changeset_revision_elem is not None: - if installed_changeset_revision_elem.text == repository.installed_changeset_revision: - elem_matches_removed_data_manager = True - else: - # This is a different version, which had been previously overridden - load_old_data_managers_by_guid[ guid ] = elem - if elem_matches_removed_data_manager: - data_manager_config_has_changes = True - else: - config_elems.append( elem ) - # Remove data managers from in memory - app.data_managers.remove_manager( guids ) - # Load other versions of any now uninstalled data managers, if any - for elem in load_old_data_managers_by_guid.itervalues(): - app.data_managers.load_manager_from_elem( elem ) - # Persist the altered shed_data_manager_config file. - if data_manager_config_has_changes: - data_manager_config_elems_to_xml_file( app, config_elems, shed_data_manager_conf_filename ) + tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename ) + if tree: + root = tree.getroot() + assert root.tag == 'data_managers', 'The file provided (%s) for removing data managers from is not a valid data manager xml file.' % ( shed_data_manager_conf_filename ) + guids = [ data_manager_dict.get( 'guid' ) for data_manager_dict in metadata_dict.get( 'data_manager', {} ).get( 'data_managers', {} ).itervalues() if 'guid' in data_manager_dict ] + load_old_data_managers_by_guid = {} + data_manager_config_has_changes = False + config_elems = [] + for elem in root: + # Match Data Manager elements by guid and installed_changeset_revision + elem_matches_removed_data_manager = False + if elem.tag == 'data_manager': + guid = elem.get( 'guid', None ) + if guid in guids: + tool_elem = elem.find( 'tool' ) + if tool_elem is not None: + installed_changeset_revision_elem = tool_elem.find( 'installed_changeset_revision' ) + if installed_changeset_revision_elem is not None: + if installed_changeset_revision_elem.text == repository.installed_changeset_revision: + elem_matches_removed_data_manager = True + else: + # This is a different version, which had been previously overridden + load_old_data_managers_by_guid[ guid ] = elem + if elem_matches_removed_data_manager: + data_manager_config_has_changes = True + else: + config_elems.append( elem ) + # Remove data managers from in memory + app.data_managers.remove_manager( guids ) + # Load other versions of any now uninstalled data managers, if any + for elem in load_old_data_managers_by_guid.itervalues(): + app.data_managers.load_manager_from_elem( elem ) + # Persist the altered shed_data_manager_config file. + if data_manager_config_has_changes: + data_manager_config_elems_to_xml_file( app, config_elems, shed_data_manager_conf_filename ) diff -r c666aa204369e3c36498e1d4211e1ec591c3aeeb -r 4df405e7a2aff50c4ef83806c55d01448227cc5d lib/tool_shed/util/datatype_util.py --- a/lib/tool_shed/util/datatype_util.py +++ b/lib/tool_shed/util/datatype_util.py @@ -5,14 +5,6 @@ from tool_shed.util import xml_util import tool_shed.util.shed_util_common as suc -import pkg_resources - -pkg_resources.require( 'elementtree' ) -from elementtree import ElementTree -from elementtree import ElementInclude -from elementtree.ElementTree import Element -from elementtree.ElementTree import SubElement - log = logging.getLogger( __name__ ) def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=False, override=True ): @@ -23,10 +15,8 @@ be False when a tool shed repository is being installed. Since installation is occurring after the datatypes registry has been initialized, the registry's contents cannot be overridden by conflicting data types. """ - try: - tree = xml_util.parse_xml( datatypes_config ) - except Exception, e: - log.debug( "Error parsing %s, exception: %s" % ( datatypes_config, str( e ) ) ) + tree, error_message = xml_util.parse_xml( datatypes_config ) + if tree is None: return None, None datatypes_config_root = tree.getroot() registration = datatypes_config_root.find( 'registration' ) diff -r c666aa204369e3c36498e1d4211e1ec591c3aeeb -r 4df405e7a2aff50c4ef83806c55d01448227cc5d lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -26,10 +26,6 @@ from mercurial import hg from mercurial import ui -pkg_resources.require( 'elementtree' ) -from elementtree import ElementTree -from elementtree import ElementInclude - log = logging.getLogger( __name__ ) REPOSITORY_DATA_MANAGER_CONFIG_FILENAME = "data_manager_conf.xml" @@ -363,12 +359,9 @@ 'invalid_data_managers': invalid_data_managers, 'error_messages': [] } metadata_dict[ 'data_manager' ] = data_manager_metadata - try: - tree = xml_util.parse_xml( data_manager_config_filename ) - except Exception, e: + tree, error_message = xml_util.parse_xml( data_manager_config_filename ) + if tree is None: # We are not able to load any data managers. - error_message = 'There was an error parsing your Data Manager config file "%s": %s' % ( data_manager_config_filename, e ) - log.error( error_message ) data_manager_metadata[ 'error_messages' ].append( error_message ) return metadata_dict tool_path = None @@ -436,13 +429,10 @@ def generate_datatypes_metadata( app, repository, repository_clone_url, repository_files_dir, datatypes_config, metadata_dict ): """Update the received metadata_dict with information from the parsed datatypes_config.""" - try: - tree = ElementTree.parse( datatypes_config ) - except Exception, e: - log.debug( "Exception attempting to parse %s: %s" % ( str( datatypes_config ), str( e ) ) ) + tree, error_message = xml_util.parse_xml( datatypes_config ) + if tree is None: return metadata_dict root = tree.getroot() - ElementInclude.include( root ) repository_datatype_code_files = [] datatype_files = root.find( 'datatype_files' ) if datatype_files: @@ -637,14 +627,13 @@ if os.path.getsize( full_path ) > 0: if not ( checkers.check_binary( full_path ) or checkers.check_image( full_path ) or checkers.check_gzip( full_path )[ 0 ] or checkers.check_bz2( full_path )[ 0 ] or checkers.check_zip( full_path ) ): - try: - # Make sure we're looking at a tool config and not a display application config or something else. - element_tree = xml_util.parse_xml( full_path ) + # Make sure we're looking at a tool config and not a display application config or something else. + element_tree, error_message = xml_util.parse_xml( full_path ) + if element_tree is None: + is_tool = False + else: element_tree_root = element_tree.getroot() is_tool = element_tree_root.tag == 'tool' - except Exception, e: - log.debug( "Error parsing %s, exception: %s" % ( full_path, str( e ) ) ) - is_tool = False if is_tool: tool, valid, error_message = tool_util.load_tool_from_config( app, app.security.encode_id( repository.id ), full_path ) if tool is None: @@ -750,15 +739,13 @@ is called from the tool shed as well as from Galaxy. """ error_message = '' - try: - # Make sure we're looking at a valid repository_dependencies.xml file. - tree = xml_util.parse_xml( repository_dependencies_config ) + # Make sure we're looking at a valid repository_dependencies.xml file. + tree, error_message = xml_util.parse_xml( repository_dependencies_config ) + if tree is None: + xml_is_valid = False + else: root = tree.getroot() xml_is_valid = root.tag == 'repositories' - except Exception, e: - error_message = "Error parsing %s, exception: %s" % ( repository_dependencies_config, str( e ) ) - log.debug( error_message ) - xml_is_valid = False if xml_is_valid: invalid_repository_dependencies_dict = dict( description=root.get( 'description' ) ) invalid_repository_dependency_tups = [] @@ -795,14 +782,10 @@ else: original_valid_tool_dependencies_dict = None original_invalid_tool_dependencies_dict = None - try: - tree = ElementTree.parse( tool_dependencies_config ) - except Exception, e: - error_message = "Exception attempting to parse tool_dependencies.xml: %s" %str( e ) - log.debug( error_message ) + tree, error_message = xml_util.parse_xml( tool_dependencies_config ) + if tree is None: return metadata_dict, error_message root = tree.getroot() - ElementInclude.include( root ) tool_dependency_is_valid = True valid_tool_dependencies_dict = {} invalid_tool_dependencies_dict = {} diff -r c666aa204369e3c36498e1d4211e1ec591c3aeeb -r 4df405e7a2aff50c4ef83806c55d01448227cc5d lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -15,18 +15,13 @@ import sqlalchemy.orm.exc from tool_shed.util import common_util from tool_shed.util import xml_util +from xml.etree import ElementTree as XmlET from galaxy import eggs import pkg_resources pkg_resources.require( 'mercurial' ) from mercurial import hg, ui, commands -pkg_resources.require( 'elementtree' ) -from elementtree import ElementTree -from elementtree import ElementInclude -from elementtree.ElementTree import Element -from elementtree.ElementTree import SubElement - eggs.require( 'markupsafe' ) import markupsafe @@ -290,22 +285,22 @@ def generate_tool_elem( tool_shed, repository_name, changeset_revision, owner, tool_file_path, tool, tool_section ): """Create and return an ElementTree tool Element.""" if tool_section is not None: - tool_elem = SubElement( tool_section, 'tool' ) + tool_elem = XmlET.SubElement( tool_section, 'tool' ) else: - tool_elem = Element( 'tool' ) + tool_elem = XmlET.Element( 'tool' ) tool_elem.attrib[ 'file' ] = tool_file_path tool_elem.attrib[ 'guid' ] = tool.guid - tool_shed_elem = SubElement( tool_elem, 'tool_shed' ) + tool_shed_elem = XmlET.SubElement( tool_elem, 'tool_shed' ) tool_shed_elem.text = tool_shed - repository_name_elem = SubElement( tool_elem, 'repository_name' ) + repository_name_elem = XmlET.SubElement( tool_elem, 'repository_name' ) repository_name_elem.text = repository_name - repository_owner_elem = SubElement( tool_elem, 'repository_owner' ) + repository_owner_elem = XmlET.SubElement( tool_elem, 'repository_owner' ) repository_owner_elem.text = owner - changeset_revision_elem = SubElement( tool_elem, 'installed_changeset_revision' ) + changeset_revision_elem = XmlET.SubElement( tool_elem, 'installed_changeset_revision' ) changeset_revision_elem.text = changeset_revision - id_elem = SubElement( tool_elem, 'id' ) + id_elem = XmlET.SubElement( tool_elem, 'id' ) id_elem.text = tool.id - version_elem = SubElement( tool_elem, 'version' ) + version_elem = XmlET.SubElement( tool_elem, 'version' ) version_elem.text = tool.version return tool_elem @@ -335,7 +330,9 @@ file_name = strip_path( tool_config ) guids_and_configs[ guid ] = file_name # Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict. - tree = xml_util.parse_xml( shed_tool_conf ) + tree, error_message = xml_util.parse_xml( shed_tool_conf ) + if tree is None: + return tool_panel_dict root = tree.getroot() for elem in root: if elem.tag == 'tool': @@ -1253,20 +1250,21 @@ for tool_config_filename, guid, tool in repository_tools_tups: guid_to_tool_elem_dict[ guid ] = generate_tool_elem( tool_shed, repository.name, repository.changeset_revision, repository.owner or '', tool_config_filename, tool, None ) config_elems = [] - tree = xml_util.parse_xml( shed_tool_conf ) - root = tree.getroot() - for elem in root: - if elem.tag == 'section': - for i, tool_elem in enumerate( elem ): - guid = tool_elem.attrib.get( 'guid' ) + tree, error_message = xml_util.parse_xml( shed_tool_conf ) + if tree: + root = tree.getroot() + for elem in root: + if elem.tag == 'section': + for i, tool_elem in enumerate( elem ): + guid = tool_elem.attrib.get( 'guid' ) + if guid in guid_to_tool_elem_dict: + elem[i] = guid_to_tool_elem_dict[ guid ] + elif elem.tag == 'tool': + guid = elem.attrib.get( 'guid' ) if guid in guid_to_tool_elem_dict: - elem[i] = guid_to_tool_elem_dict[ guid ] - elif elem.tag == 'tool': - guid = elem.attrib.get( 'guid' ) - if guid in guid_to_tool_elem_dict: - elem = guid_to_tool_elem_dict[ guid ] - config_elems.append( elem ) - config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path ) + elem = guid_to_tool_elem_dict[ guid ] + config_elems.append( elem ) + config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path ) def update_repository( repo, ctx_rev=None ): """ diff -r c666aa204369e3c36498e1d4211e1ec591c3aeeb -r 4df405e7a2aff50c4ef83806c55d01448227cc5d lib/tool_shed/util/tool_dependency_util.py --- a/lib/tool_shed/util/tool_dependency_util.py +++ b/lib/tool_shed/util/tool_dependency_util.py @@ -5,10 +5,7 @@ from galaxy import util from galaxy.model.orm import and_ import tool_shed.util.shed_util_common as suc - -eggs.require( 'elementtree' ) -from elementtree import ElementTree -from elementtree import ElementInclude +from tool_shed.util import xml_util log = logging.getLogger( __name__ ) @@ -67,13 +64,10 @@ relative_install_dir = os.path.join( shed_config_dict.get( 'tool_path' ), relative_install_dir ) # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', relative_install_dir ) - try: - tree = ElementTree.parse( tool_dependencies_config ) - except Exception, e: - log.debug( "Exception attempting to parse tool_dependencies.xml: %s" %str( e ) ) + tree, error_message = xml_util.parse_xml( tool_dependencies_config ) + if tree is None: return tool_dependency_objects root = tree.getroot() - ElementInclude.include( root ) fabric_version_checked = False for elem in root: tool_dependency_type = elem.tag diff -r c666aa204369e3c36498e1d4211e1ec591c3aeeb -r 4df405e7a2aff50c4ef83806c55d01448227cc5d lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -14,6 +14,7 @@ from galaxy.web.form_builder import SelectField from tool_shed.util import xml_util import tool_shed.util.shed_util_common as suc +from xml.etree import ElementTree as XmlET import pkg_resources @@ -22,12 +23,6 @@ from mercurial import hg from mercurial import ui -pkg_resources.require( 'elementtree' ) -from elementtree import ElementTree -from elementtree import ElementInclude -from elementtree.ElementTree import Element -from elementtree.ElementTree import SubElement - log = logging.getLogger( __name__ ) def add_to_shed_tool_config( app, shed_tool_conf_dict, elem_list ): @@ -37,15 +32,16 @@ shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] tool_path = shed_tool_conf_dict[ 'tool_path' ] config_elems = [] - tree = xml_util.parse_xml( shed_tool_conf ) - root = tree.getroot() - for elem in root: - config_elems.append( elem ) - # Add the elements to the in-memory list of config_elems. - for elem_entry in elem_list: - config_elems.append( elem_entry ) - # Persist the altered shed_tool_config file. - suc.config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path ) + tree, error_message = xml_util.parse_xml( shed_tool_conf ) + if tree: + root = tree.getroot() + for elem in root: + config_elems.append( elem ) + # Add the elements to the in-memory list of config_elems. + for elem_entry in elem_list: + config_elems.append( elem_entry ) + # Persist the altered shed_tool_config file. + suc.config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path ) def add_to_tool_panel( app, repository_name, repository_clone_url, changeset_revision, repository_tools_tups, owner, shed_tool_conf, tool_panel_dict, new_install=True ): @@ -381,7 +377,7 @@ # { id: <ToolSection id>, version : <ToolSection version>, name : <TooSection name>} if tool_section_dict[ 'id' ]: # Create a new tool section. - tool_section = Element( 'section' ) + tool_section = XmlET.Element( 'section' ) tool_section.attrib[ 'id' ] = tool_section_dict[ 'id' ] tool_section.attrib[ 'name' ] = tool_section_dict[ 'name' ] tool_section.attrib[ 'version' ] = tool_section_dict[ 'version' ] @@ -639,7 +635,7 @@ tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ] else: # The section in which the tool was originally loaded used to be in the tool panel, but no longer is. - elem = Element( 'section' ) + elem = XmlET.Element( 'section' ) elem.attrib[ 'name' ] = original_section_name elem.attrib[ 'id' ] = original_section_id elem.attrib[ 'version' ] = '' @@ -657,7 +653,7 @@ else: # Appending a new section to trans.app.toolbox.tool_panel log.debug( "Loading new tool panel section: %s" % new_tool_panel_section ) - elem = Element( 'section' ) + elem = XmlET.Element( 'section' ) elem.attrib[ 'name' ] = new_tool_panel_section elem.attrib[ 'id' ] = section_id elem.attrib[ 'version' ] = '' @@ -804,7 +800,9 @@ message = '' tmp_tool_config = suc.get_named_tmpfile_from_ctx( ctx, ctx_file, work_dir ) if tmp_tool_config: - element_tree = xml_util.parse_xml( tmp_tool_config ) + element_tree, error_message = xml_util.parse_xml( tmp_tool_config ) + if element_tree is None: + return tool, message element_tree_root = element_tree.getroot() # Look for code files required by the tool config. tmp_code_files = [] @@ -846,30 +844,31 @@ shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] tool_path = shed_tool_conf_dict[ 'tool_path' ] config_elems = [] - tree = xml_util.parse_xml( shed_tool_conf ) - root = tree.getroot() - for elem in root: - config_elems.append( elem ) - config_elems_to_remove = [] - for config_elem in config_elems: - if config_elem.tag == 'section': - tool_elems_to_remove = [] - for tool_elem in config_elem: - if tool_elem.get( 'guid' ) in guids_to_remove: - tool_elems_to_remove.append( tool_elem ) - for tool_elem in tool_elems_to_remove: - # Remove all of the appropriate tool sub-elements from the section element. - config_elem.remove( tool_elem ) - if len( config_elem ) < 1: - # Keep a list of all empty section elements so they can be removed. - config_elems_to_remove.append( config_elem ) - elif config_elem.tag == 'tool': - if config_elem.get( 'guid' ) in guids_to_remove: - config_elems_to_remove.append( config_elem ) - for config_elem in config_elems_to_remove: - config_elems.remove( config_elem ) - # Persist the altered in-memory version of the tool config. - suc.config_elems_to_xml_file( trans.app, config_elems, shed_tool_conf, tool_path ) + tree, error_message = xml_util.parse_xml( shed_tool_conf ) + if tree: + root = tree.getroot() + for elem in root: + config_elems.append( elem ) + config_elems_to_remove = [] + for config_elem in config_elems: + if config_elem.tag == 'section': + tool_elems_to_remove = [] + for tool_elem in config_elem: + if tool_elem.get( 'guid' ) in guids_to_remove: + tool_elems_to_remove.append( tool_elem ) + for tool_elem in tool_elems_to_remove: + # Remove all of the appropriate tool sub-elements from the section element. + config_elem.remove( tool_elem ) + if len( config_elem ) < 1: + # Keep a list of all empty section elements so they can be removed. + config_elems_to_remove.append( config_elem ) + elif config_elem.tag == 'tool': + if config_elem.get( 'guid' ) in guids_to_remove: + config_elems_to_remove.append( config_elem ) + for config_elem in config_elems_to_remove: + config_elems.remove( config_elem ) + # Persist the altered in-memory version of the tool config. + suc.config_elems_to_xml_file( trans.app, config_elems, shed_tool_conf, tool_path ) def remove_from_tool_panel( trans, repository, shed_tool_conf, uninstall ): """A tool shed repository is being deactivated or uninstalled so handle tool panel alterations accordingly.""" diff -r c666aa204369e3c36498e1d4211e1ec591c3aeeb -r 4df405e7a2aff50c4ef83806c55d01448227cc5d lib/tool_shed/util/xml_util.py --- a/lib/tool_shed/util/xml_util.py +++ b/lib/tool_shed/util/xml_util.py @@ -1,13 +1,17 @@ import logging import os +import sys import tempfile from xml.etree import ElementTree as XmlET import xml.etree.ElementTree log = logging.getLogger( __name__ ) +using_python_27 = sys.version_info[ :2 ] >= ( 2, 7 ) -class CommentedTreeBuilder ( XmlET.XMLTreeBuilder ): +class Py26CommentedTreeBuilder ( XmlET.XMLTreeBuilder ): + # Python 2.6 uses ElementTree 1.2.x. + def __init__ ( self, html=0, target=None ): XmlET.XMLTreeBuilder.__init__( self, html, target ) self._parser.CommentHandler = self.handle_comment @@ -17,6 +21,15 @@ self._target.data( data ) self._target.end( XmlET.Comment ) + +class Py27CommentedTreeBuilder ( XmlET.TreeBuilder ): + # Python 2.7 uses ElementTree 1.3.x. + + def comment( self, data ): + self.start( XmlET.Comment, {} ) + self.data( data ) + self.end( XmlET.Comment ) + def create_and_write_tmp_file( elem ): tmp_str = xml_to_string( elem ) fh = tempfile.NamedTemporaryFile( 'wb' ) @@ -30,14 +43,30 @@ def parse_xml( file_name ): """Returns a parsed xml tree with comments intact.""" - try: - fobj = open( file_name, 'r' ) - tree = XmlET.parse( fobj, parser=CommentedTreeBuilder() ) - fobj.close() - except Exception, e: - log.exception( "Exception attempting to parse %s: %s" % ( str( file_name ), str( e ) ) ) - return None - return tree + error_message = '' + fobj = open( file_name, 'r' ) + if using_python_27: + try: + tree = XmlET.parse( fobj, parser=XmlET.XMLParser( target=Py27CommentedTreeBuilder() ) ) + except Exception, e: + fobj.close() + error_message = "Exception attempting to parse %s: %s" % ( str( file_name ), str( e ) ) + log.exception( error_message ) + return None, error_message + else: + try: + tree = XmlET.parse( fobj, parser=Py26CommentedTreeBuilder() ) + except Exception, e: + fobj.close() + error_message = "Exception attempting to parse %s: %s" % ( str( file_name ), str( e ) ) + log.exception( error_message ) + return None, error_message + fobj.close() + return tree, error_message def xml_to_string( elem, encoding='utf-8' ): - return '%s\n' % xml.etree.ElementTree.tostring( elem, encoding=encoding ) + if using_python_27: + xml_str = '%s\n' % xml.etree.ElementTree.tostring( elem, encoding=encoding, method="xml" ) + else: + xml_str = '%s\n' % xml.etree.ElementTree.tostring( elem, encoding=encoding ) + return xml_str diff -r c666aa204369e3c36498e1d4211e1ec591c3aeeb -r 4df405e7a2aff50c4ef83806c55d01448227cc5d test/tool_shed/functional/test_0010_repository_with_tool_dependencies.py --- a/test/tool_shed/functional/test_0010_repository_with_tool_dependencies.py +++ b/test/tool_shed/functional/test_0010_repository_with_tool_dependencies.py @@ -118,7 +118,7 @@ uncompress_file=False, remove_repo_files_not_in_tar=False, commit_message='Uploaded malformed tool dependency XML.', - strings_displayed=[ 'Exception attempting to parse tool_dependencies.xml', 'not well-formed' ], + strings_displayed=[ 'Exception attempting to parse', 'not well-formed' ], strings_not_displayed=[] ) def test_0030_upload_invalid_tool_dependency_xml( self ): diff -r c666aa204369e3c36498e1d4211e1ec591c3aeeb -r 4df405e7a2aff50c4ef83806c55d01448227cc5d test/tool_shed/functional/test_1010_install_repository_with_tool_dependencies.py --- a/test/tool_shed/functional/test_1010_install_repository_with_tool_dependencies.py +++ b/test/tool_shed/functional/test_1010_install_repository_with_tool_dependencies.py @@ -71,7 +71,7 @@ uncompress_file=False, remove_repo_files_not_in_tar=False, commit_message='Uploaded malformed tool dependency XML.', - strings_displayed=[ 'Exception attempting to parse tool_dependencies.xml', 'not well-formed' ], + strings_displayed=[ 'Exception attempting to parse', 'not well-formed' ], strings_not_displayed=[] ) self.upload_file( repository, filename=os.path.join( 'freebayes', 'invalid_tool_dependencies', 'tool_dependencies.xml' ), diff -r c666aa204369e3c36498e1d4211e1ec591c3aeeb -r 4df405e7a2aff50c4ef83806c55d01448227cc5d test/tool_shed/functional/test_1210_uninstall_reinstall_repository_with_tool_dependencies.py --- a/test/tool_shed/functional/test_1210_uninstall_reinstall_repository_with_tool_dependencies.py +++ b/test/tool_shed/functional/test_1210_uninstall_reinstall_repository_with_tool_dependencies.py @@ -65,7 +65,7 @@ uncompress_file=False, remove_repo_files_not_in_tar=False, commit_message='Uploaded malformed tool dependency XML.', - strings_displayed=[ 'Exception attempting to parse tool_dependencies.xml', 'not well-formed' ], + strings_displayed=[ 'Exception attempting to parse', 'not well-formed' ], strings_not_displayed=[] ) self.upload_file( repository, filename=os.path.join( 'freebayes', 'invalid_tool_dependencies', 'tool_dependencies.xml' ), Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.