1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/48b3531465ee/ changeset: 48b3531465ee user: greg date: 2011-12-16 22:10:06 summary: Enhance the datatypes registry so that it can be persisted as an xml file, which is then used for all tools instead of the datatypes config file. affected #: 11 files diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -118,9 +118,16 @@ self.job_stop_queue = self.job_manager.job_stop_queue # Initialize the external service types self.external_service_types = external_service_types.ExternalServiceTypesCollection( self.config.external_service_type_config_file, self.config.external_service_type_path, self ) - def shutdown( self ): self.job_manager.shutdown() self.object_store.shutdown() if self.heartbeat: self.heartbeat.shutdown() + try: + # If the datatypes registry was persisted, attempt to + # remove the temporary file in which it was written. + tmp_filename = self.datatypes_registry.xml_filename + if tmp_filename: + os.unlink( tmp_filename ) + except: + pass diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -544,6 +544,7 @@ if config_root is None: config_root = os.path.abspath( os.getcwd() ) if datatypes_config is None: + raise Exception( 'In setup_external_metadata, the received datatypes_config is None.' ) datatypes_config = 'datatypes_conf.xml' metadata_files_list = [] for dataset in datasets: diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -27,9 +27,15 @@ self.sniff_order = [] self.upload_file_formats = [] self.display_applications = odict() #map a display application id to a display application + self.converters_path_attr = None self.datatype_converters_path = None + self.indexers_path_attr = None self.datatype_indexers_path = None + self.display_path_attr = None self.display_applications_path = None + self.datatype_elems = [] + self.sniffer_elems = [] + self.xml_filename = None def load_datatypes( self, root_dir=None, config=None, imported_module=None ): if root_dir and config: inherit_display_application_by_class = [] @@ -45,16 +51,21 @@ # files installed with repositories from tool sheds must use the same paths. However, we # may discover at some future time that allowing for multiple paths is more optimal. if not self.datatype_converters_path: - self.datatype_converters_path = os.path.join( root_dir, registration.get( 'converters_path', 'lib/galaxy/datatypes/converters' ) ) + self.converters_path_attr = registration.get( 'converters_path', 'lib/galaxy/datatypes/converters' ) + self.datatype_converters_path = os.path.join( root_dir, self.converters_path_attr ) if not os.path.isdir( self.datatype_converters_path ): raise ConfigurationError( "Directory does not exist: %s" % self.datatype_converters_path ) if not self.datatype_indexers_path: - self.datatype_indexers_path = os.path.join( root_dir, registration.get( 'indexers_path', 'lib/galaxy/datatypes/indexers' ) ) + self.indexers_path_attr = registration.get( 'indexers_path', 'lib/galaxy/datatypes/indexers' ) + self.datatype_indexers_path = os.path.join( root_dir, self.indexers_path_attr ) if not os.path.isdir( self.datatype_indexers_path ): raise ConfigurationError( "Directory does not exist: %s" % self.datatype_indexers_path ) if not self.display_applications_path: - self.display_applications_path = os.path.join( root_dir, registration.get( 'display_path', 'display_applications' ) ) + self.display_path_attr = registration.get( 'display_path', 'display_applications' ) + self.display_applications_path = os.path.join( root_dir, self.display_path_attr ) for elem in registration.findall( 'datatype' ): + # Keep an in-memory list of datatype elems to enable persistence. + self.datatype_elems.append( elem ) try: extension = elem.get( 'extension', None ) dtype = elem.get( 'type', None ) @@ -147,6 +158,8 @@ sniffers = root.find( 'sniffers' ) if sniffers: for elem in sniffers.findall( 'sniffer' ): + # Keep an in-memory list of sniffer elems to enable persistence. + self.sniffer_elems.append( elem ) dtype = elem.get( 'type', None ) if dtype: try: @@ -418,3 +431,31 @@ if 'auto' not in rval and 'txt' in rval: #need to manually add 'auto' datatype rval[ 'auto' ] = rval[ 'txt' ] return rval + def to_xml_file( self ): + fd, filename = tempfile.mkstemp() + self.xml_filename = filename + if self.converters_path_attr: + converters_path_str = ' converters_path="%s"' % self.converters_path_attr + else: + converters_path_str = '' + if self.indexers_path_attr: + indexers_path_str = ' indexers_path="%s"' % self.indexers_path_attr + else: + indexers_path_str = '' + if self.display_path_attr: + display_path_str = ' display_path="%s"' % self.display_path_attr + else: + display_path_str = '' + os.write( fd, '<?xml version="1.0"?>\n' ) + os.write( fd, '<datatypes>\n' ) + os.write( fd, '<registration%s%s%s>\n' % ( converters_path_str, indexers_path_str, display_path_str ) ) + for elem in self.datatype_elems: + os.write( fd, '%s' % galaxy.util.xml_to_string( elem ) ) + os.write( fd, '</registration>\n' ) + os.write( fd, '<sniffers>\n' ) + for elem in self.sniffer_elems: + os.write( fd, '%s' % galaxy.util.xml_to_string( elem ) ) + os.write( fd, '</sniffers>\n' ) + os.write( fd, '</datatypes>\n' ) + os.close( fd ) + return filename diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -868,7 +868,7 @@ if config_root is None: config_root = self.app.config.root if datatypes_config is None: - datatypes_config = self.app.config.datatypes_config + datatypes_config = self.app.datatypes_registry.to_xml_file() return self.external_output_metadata.setup_external_metadata( [ output_dataset_assoc.dataset for output_dataset_assoc in job.output_datasets ], self.sa_session, exec_dir = exec_dir, diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/tool_shed/__init__.py --- a/lib/galaxy/tool_shed/__init__.py +++ b/lib/galaxy/tool_shed/__init__.py @@ -14,7 +14,8 @@ def load_datatypes( self ): for tool_shed_repository in self.sa_session.query( self.model.ToolShedRepository ) \ .filter( and_( self.model.ToolShedRepository.table.c.includes_datatypes==True, - self.model.ToolShedRepository.table.c.deleted==False ) ): + self.model.ToolShedRepository.table.c.deleted==False ) ) \ + .order_by( self.model.ToolShedRepository.table.c.id ): metadata = tool_shed_repository.metadata datatypes_config = metadata[ 'datatypes_config' ] full_path = os.path.abspath( datatypes_config ) diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1672,7 +1672,8 @@ # For the upload tool, we need to know the root directory and the # datatypes conf path, so we can load the datatypes registry param_dict['__root_dir__'] = param_dict['GALAXY_ROOT_DIR'] = os.path.abspath( self.app.config.root ) - param_dict['__datatypes_config__'] = param_dict['GALAXY_DATATYPES_CONF_FILE'] = os.path.abspath( self.app.config.datatypes_config ) + datatypes_config = self.app.datatypes_registry.to_xml_file() + param_dict['__datatypes_config__'] = param_dict['GALAXY_DATATYPES_CONF_FILE'] = os.path.abspath( datatypes_config ) # Return the dictionary of parameters return param_dict diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -107,10 +107,28 @@ ElementInclude.include(root) return tree -def xml_to_string(elem): - """Returns an string from and xml tree""" - text = ElementTree.tostring(elem) - return text +def xml_to_string( elem, pretty=False ): + """Returns a string from an xml tree""" + if pretty: + return ElementTree.tostring( pretty_print_xml( elem ) ) + return ElementTree.tostring( elem ) + +def pretty_print_xml( elem, level=0 ): + pad = ' ' + i = "\n" + level * pad + if len( elem ): + if not elem.text or not elem.text.strip(): + elem.text = i + pad + pad + if not elem.tail or not elem.tail.strip(): + elem.tail = i + for e in elem: + pretty_print_xml( e, level + 1 ) + if not elem.tail or not elem.tail.strip(): + elem.tail = i + else: + if level and ( not elem.tail or not elem.tail.strip() ): + elem.tail = i + pad + return elem # characters that are valid valid_chars = set(string.letters + string.digits + " -=_.()/+*^,:?!") diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/util/none_like.py --- a/lib/galaxy/util/none_like.py +++ b/lib/galaxy/util/none_like.py @@ -21,6 +21,7 @@ self.ext = self.extension = ext self.dbkey = dbkey if datatypes_registry is None: + # Default Value Required for unit tests datatypes_registry = Registry() datatypes_registry.load_datatypes() self.datatype = datatypes_registry.get_datatype_by_extension( ext ) diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -9,7 +9,7 @@ pkg_resources.require( 'elementtree' ) from elementtree import ElementTree, ElementInclude -from elementtree.ElementTree import Element, SubElement, tostring +from elementtree.ElementTree import Element, SubElement log = logging.getLogger( __name__ ) @@ -39,7 +39,7 @@ if line.startswith( '</toolbox>' ): # We're at the end of the original config file, so add our entry. new_shed_tool_conf.write( ' ' ) - new_shed_tool_conf.write( tostring( pretty_print_xml( tool_panel_entry ) ) ) + new_shed_tool_conf.write( util.xml_to_string( tool_panel_entry, pretty=True ) ) new_shed_tool_conf.write( line ) else: new_shed_tool_conf.write( line ) @@ -554,22 +554,6 @@ log.debug( "Adding new row (or updating an existing row) for repository '%s' in the tool_shed_repository table." % name ) create_or_update_tool_shed_repository( app, name, description, changeset_revision, repository_clone_url, metadata_dict ) return metadata_dict -def pretty_print_xml( elem, level=0 ): - pad = ' ' - i = "\n" + level * pad - if len( elem ): - if not elem.text or not elem.text.strip(): - elem.text = i + pad + pad - if not elem.tail or not elem.tail.strip(): - elem.tail = i - for e in elem: - pretty_print_xml( e, level + 1 ) - if not elem.tail or not elem.tail.strip(): - elem.tail = i - else: - if level and ( not elem.tail or not elem.tail.strip() ): - elem.tail = i + pad - return elem def pull_repository( current_working_dir, repo_files_dir, name ): # Pull the latest possible contents to the repository. log.debug( "Pulling latest updates to the repository named '%s'" % name ) diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/web/base/controller.py --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -22,10 +22,6 @@ from Cheetah.Template import Template -pkg_resources.require( 'elementtree' ) -from elementtree import ElementTree, ElementInclude -from elementtree.ElementTree import Element, SubElement, tostring - log = logging.getLogger( __name__ ) # States for passing messages diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/webapps/community/app.py --- a/lib/galaxy/webapps/community/app.py +++ b/lib/galaxy/webapps/community/app.py @@ -16,6 +16,7 @@ config.configure_logging( self.config ) # Set up datatypes registry self.datatypes_registry = galaxy.datatypes.registry.Registry() + # TODO: Handle datatypes included in repositories - the following will only load datatypes_conf.xml. self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config ) # Determine the database url if self.config.database_connection: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.