1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/865d998a693d/ changeset: 865d998a693d user: greg date: 2012-02-08 17:30:58 summary: Load proprietary datatypes from installed tool shed repositories before the datatypes in the Galaxy distribution are loaded. We do this because the distribution includes some extremely generic sniffers (eg text,xml) which will catch pretty much anything, making it impossible for proprietary sniffers to be used. Proprietary datatypes contained in installed repositories are loaded in order of oldest installation first, followed by next olderst installation, etc. affected #: 3 files diff -r d0e5fa20e4917aac9d058413dcf07b06f0dfec92 -r 865d998a693de676c01daea4128d20124d2015c0 lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -22,15 +22,6 @@ self.config = config.Configuration( **kwargs ) self.config.check() config.configure_logging( self.config ) - # Initialize the datatypes registry to the default data types included in self.config.datatypes_config. - self.datatypes_registry = galaxy.datatypes.registry.Registry() - self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config ) - galaxy.model.set_datatypes_registry( self.datatypes_registry ) - # Set up the tool sheds registry - if os.path.isfile( self.config.tool_sheds_config ): - self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config ) - else: - self.tool_shed_registry = None # Determine the database url if self.config.database_connection: db_url = self.config.database_connection @@ -48,6 +39,26 @@ self.config.database_engine_options, database_query_profiling_proxy = self.config.database_query_profiling_proxy, object_store = self.object_store ) + # Set up the tool sheds registry + if os.path.isfile( self.config.tool_sheds_config ): + self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config ) + else: + self.tool_shed_registry = None + # Manage installed tool shed repositories. + self.installed_repository_manager = galaxy.tool_shed.InstalledRepositoryManager( self ) + # Create an empty datatypes registry. + self.datatypes_registry = galaxy.datatypes.registry.Registry() + # Load proprietary datatypes defined in datatypes_conf.xml files in all installed tool shed repositories. We + # load proprietary datatypes before datatypes in the distribution because Galaxy's default sniffers include some + # generic sniffers (eg text,xml) which catch anything, so it's impossible for proprietary sniffers to be used. + # However, if there is a conflict (2 datatypes with the same extension) between a proprietary datatype and a datatype + # in the Galaxy distribution, the datatype in the Galaxy distribution will take precedence. If there is a conflict + # between 2 proprietary datatypes, the datatype from the repository that was installed earliest will take precedence. + # This will also load proprietary datatype converters and display applications. + self.installed_repository_manager.load_proprietary_datatypes() + # Load the data types in the Galaxy distribution, which are defined in self.config.datatypes_config. + self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config, override_conflicts=True ) + galaxy.model.set_datatypes_registry( self.datatypes_registry ) # Security helper self.security = security.SecurityHelper( id_secret=self.config.id_secret ) # Tag handler @@ -63,28 +74,22 @@ if self.config.get_bool( 'enable_tool_shed_install', False ): from tool_shed import install_manager self.install_manager = install_manager.InstallManager( self, self.config.tool_shed_install_config, self.config.install_tool_config ) - # If enabled, poll respective tool sheds to see if updates are - # available for any installed tool shed repositories. + # If enabled, poll respective tool sheds to see if updates are available for any installed tool shed repositories. if self.config.get_bool( 'enable_tool_shed_check', False ): from tool_shed import update_manager self.update_manager = update_manager.UpdateManager( self ) - # Manage installed tool shed repositories - self.installed_repository_manager = galaxy.tool_shed.InstalledRepositoryManager( self ) # Load datatype display applications defined in local datatypes_conf.xml self.datatypes_registry.load_display_applications() # Load datatype converters defined in local datatypes_conf.xml self.datatypes_registry.load_datatype_converters( self.toolbox ) - # Load history import/export tools - load_history_imp_exp_tools( self.toolbox ) # Load external metadata tool self.datatypes_registry.load_external_metadata_tool( self.toolbox ) - # Load proprietary datatypes defined in datatypes_conf.xml files in all installed tool shed - # repositories. This will also load all proprietary datatype converters and display_applications. - self.installed_repository_manager.load_proprietary_datatypes() - # Load security policy + # Load history import/export tools. + load_history_imp_exp_tools( self.toolbox ) + # Load security policy. self.security_agent = self.model.security_agent self.host_security_agent = galaxy.security.HostAgent( model=self.security_agent.model, permitted_actions=self.security_agent.permitted_actions ) - # Load quota management + # Load quota management. if self.config.enable_quotas: self.quota_agent = galaxy.quota.QuotaAgent( self.model ) else: diff -r d0e5fa20e4917aac9d058413dcf07b06f0dfec92 -r 865d998a693de676c01daea4128d20124d2015c0 lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -26,13 +26,14 @@ self.converter_deps = {} self.available_tracks = [] self.set_external_metadata_tool = None + # Keep a list of datatypes with loaded sniffers for handling potential conflicts. + self.datatype_sniffers = [] self.sniff_order = [] self.upload_file_formats = [] - # Datatype elements defined in local datatypes_conf.xml - # that contain display applications + # Datatype elements defined in local datatypes_conf.xml that contain display applications. self.display_app_containers = [] # Datatype elements in datatypes_conf.xml included in installed - # tool shed repositories that contain display applications + # tool shed repositories that contain display applications. self.proprietary_display_app_containers = [] # Map a display application id to a display application self.display_applications = odict() @@ -48,7 +49,7 @@ self.datatype_elems = [] self.sniffer_elems = [] self.xml_filename = None - def load_datatypes( self, root_dir=None, config=None, imported_modules=None, deactivate=False ): + def load_datatypes( self, root_dir=None, config=None, imported_modules=None, deactivate=False, override_conflicts=False ): """ Parse a datatypes XML file located at root_dir/config. If imported_modules is received, it is a list of imported datatypes class files included in an installed tool shed repository. @@ -99,13 +100,13 @@ else: # Keep an in-memory list of datatype elems to enable persistence. self.datatype_elems.append( elem ) - if extension and extension in self.datatypes_by_extension: + if extension and extension in self.datatypes_by_extension and not override_conflicts: if deactivate: # We are deactivating an installed tool shed repository, so eliminate the datatype from the registry. self.log.debug( "Removing datatype with extension '%s' from the registry." % extension ) del self.datatypes_by_extension[ extension ] else: - self.log.debug( "Ignoring datatype with extension '%s' from '%s' because the registry already contains a datatype with that extension." % ( extension, config ) ) + self.log.debug( "Ignoring datatype with extension '%s' from %s because the registry already contains a datatype with that extension." % ( extension, config ) ) elif extension and ( dtype or type_extension ): if dtype: fields = dtype.split( ':' ) @@ -130,52 +131,57 @@ datatype_class = self.datatypes_by_extension[type_extension].__class__ if make_subclass: datatype_class = type( datatype_class_name, (datatype_class,), {} ) - self.datatypes_by_extension[extension] = datatype_class() - if mimetype is None: - # Use default mime type as per datatype spec - mimetype = self.datatypes_by_extension[extension].get_mime() - self.mimetypes_by_extension[extension] = mimetype - if hasattr( datatype_class, "get_track_type" ): - self.available_tracks.append( extension ) - if display_in_upload: - self.upload_file_formats.append( extension ) - # Max file size cut off for setting optional metadata - self.datatypes_by_extension[extension].max_optional_metadata_filesize = elem.get( 'max_optional_metadata_filesize', None ) - for converter in elem.findall( 'converter' ): - # Build the list of datatype converters which will later be loaded - # into the calling app's toolbox. - converter_config = converter.get( 'file', None ) - target_datatype = converter.get( 'target_datatype', None ) - depends_on = converter.get( 'depends_on', None ) - if depends_on and target_datatype: - if extension not in self.converter_deps: - self.converter_deps[extension] = {} - self.converter_deps[extension][target_datatype] = depends_on.split(',') - if converter_config and target_datatype: + if extension not in self.datatypes_by_extension or override_conflicts: + if extension in self.datatypes_by_extension: + self.log.warning( "Overriding conflicting extension '%s', using datatype with same extension from %s." % ( extension, config ) ) + self.datatypes_by_extension[ extension ] = datatype_class() + if mimetype is None: + # Use default mime type as per datatype spec + mimetype = self.datatypes_by_extension[extension].get_mime() + self.mimetypes_by_extension[extension] = mimetype + if hasattr( datatype_class, "get_track_type" ): + self.available_tracks.append( extension ) + if display_in_upload: + self.upload_file_formats.append( extension ) + # Max file size cut off for setting optional metadata + self.datatypes_by_extension[extension].max_optional_metadata_filesize = elem.get( 'max_optional_metadata_filesize', None ) + for converter in elem.findall( 'converter' ): + # Build the list of datatype converters which will later be loaded + # into the calling app's toolbox. + converter_config = converter.get( 'file', None ) + target_datatype = converter.get( 'target_datatype', None ) + depends_on = converter.get( 'depends_on', None ) + if depends_on and target_datatype: + if extension not in self.converter_deps: + self.converter_deps[extension] = {} + self.converter_deps[extension][target_datatype] = depends_on.split(',') + if converter_config and target_datatype: + if imported_modules: + self.proprietary_converters.append( ( converter_config, extension, target_datatype ) ) + else: + self.converters.append( ( converter_config, extension, target_datatype ) ) + for composite_file in elem.findall( 'composite_file' ): + # add composite files + name = composite_file.get( 'name', None ) + if name is None: + self.log.warning( "You must provide a name for your composite_file (%s)." % composite_file ) + optional = composite_file.get( 'optional', False ) + mimetype = composite_file.get( 'mimetype', None ) + self.datatypes_by_extension[extension].add_composite_file( name, optional=optional, mimetype=mimetype ) + for display_app in elem.findall( 'display' ): if imported_modules: - self.proprietary_converters.append( ( converter_config, extension, target_datatype ) ) + if elem not in self.proprietary_display_app_containers: + self.proprietary_display_app_containers.append( elem ) else: - self.converters.append( ( converter_config, extension, target_datatype ) ) - for composite_file in elem.findall( 'composite_file' ): - # add composite files - name = composite_file.get( 'name', None ) - if name is None: - self.log.warning( "You must provide a name for your composite_file (%s)." % composite_file ) - optional = composite_file.get( 'optional', False ) - mimetype = composite_file.get( 'mimetype', None ) - self.datatypes_by_extension[extension].add_composite_file( name, optional=optional, mimetype=mimetype ) - for display_app in elem.findall( 'display' ): - if imported_modules: - if elem not in self.proprietary_display_app_containers: - self.proprietary_display_app_containers.append( elem ) - else: - if elem not in self.display_app_containers: - self.display_app_containers.append( elem ) + if elem not in self.display_app_containers: + self.display_app_containers.append( elem ) + else: + self.log.debug( "Ignoring datatype with extension '%s' from %s because the registry already contains a datatype with that extension." % ( extension, config ) ) except Exception, e: if deactivate: - self.log.warning( 'Error deactivating datatype "%s": %s' % ( extension, str( e ) ) ) + self.log.warning( "Error deactivating datatype with extension '%s': %s" % ( extension, str( e ) ) ) else: - self.log.warning( 'Error loading datatype "%s": %s' % ( extension, str( e ) ) ) + self.log.warning( "Error loading datatype with extension '%s': %s" % ( extension, str( e ) ) ) # Load datatype sniffers from the config sniffers = root.find( 'sniffers' ) if sniffers: @@ -204,15 +210,23 @@ aclass = getattr( module, datatype_class_name )() if deactivate: self.sniff_order.remove( aclass ) - self.log.debug( 'Deactivated sniffer for datatype: %s' % dtype ) + self.datatype_sniffers.remove( dtype ) + self.log.debug( "Deactivated sniffer for datatype '%s'" % dtype ) else: - self.sniff_order.append( aclass ) - self.log.debug( 'Loaded sniffer for datatype: %s' % dtype ) + if override_conflicts and dtype in self.datatype_sniffers: + self.sniff_order.append( aclass ) + self.log.debug( "Loaded additional sniffer for datatype '%s'" % dtype ) + elif dtype not in self.datatype_sniffers: + self.datatype_sniffers.append( dtype ) + self.sniff_order.append( aclass ) + self.log.debug( "Loaded sniffer for datatype '%s'" % dtype ) + else: + self.log.debug( "Ignoring sniffer for datatype '%s' because the registry already contains one." % dtype ) except Exception, exc: if deactivate: - self.log.warning( 'Error deactivating sniffer for datatype %s: %s' % ( dtype, str( exc ) ) ) + self.log.warning( "Error deactivating sniffer for datatype '%s': %s" % ( dtype, str( exc ) ) ) else: - self.log.warning( 'Error appending sniffer for datatype %s to sniff_order: %s' % ( dtype, str( exc ) ) ) + self.log.warning( "Error appending sniffer for datatype '%s' to sniff_order: %s" % ( dtype, str( exc ) ) ) # Persist the xml form of the registry into a temporary file so that it # can be loaded from the command line by tools and set_metadata processing. self.to_xml_file() diff -r d0e5fa20e4917aac9d058413dcf07b06f0dfec92 -r 865d998a693de676c01daea4128d20124d2015c0 lib/galaxy/tools/data/__init__.py --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -32,7 +32,7 @@ table = tool_data_table_types[ type ]( table_elem ) if table.name not in self.data_tables: self.data_tables[ table.name ] = table - log.debug( "Loaded tool data table '%s", table.name ) + log.debug( "Loaded tool data table '%s'", table.name ) return table_elems def add_new_entries_from_config_file( self, config_filename ): """ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.