1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/62fc9e053835/ changeset: 62fc9e053835 user: greg date: 2012-02-10 20:52:41 summary: Fixes for handling proprietary datatypes included in installed tool shed repositories - import proprietary class modules within the registry. affected #: 2 files diff -r ce17eb369f95ccaf2c8331e1a69fe34b1203cff1 -r 62fc9e05383514811e5add9db5f9ad6560a06b20 lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -1,8 +1,7 @@ """ Provides mapping between extensions and datatypes, mime-types, etc. """ -import os, tempfile -import logging +import os, sys, tempfile, threading, logging import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks, chrominfo, binary, assembly, ngsindex, wsf import galaxy.util from galaxy.util.odict import odict @@ -20,8 +19,7 @@ self.datatype_converters = odict() # Converters defined in local datatypes_conf.xml self.converters = [] - # Converters defined in datatypes_conf.xml included - # in installed tool shed repositories. + # Converters defined in datatypes_conf.xml included in installed tool shed repositories. self.proprietary_converters = [] self.converter_deps = {} self.available_tracks = [] @@ -44,17 +42,24 @@ # The 'default' display_path defined in local datatypes_conf.xml self.display_applications_path = None self.inherit_display_application_by_class = [] + # Keep a list of imported proprietary datatype class modules. + self.imported_modules = [] self.datatype_elems = [] self.sniffer_elems = [] self.xml_filename = None - def load_datatypes( self, root_dir=None, config=None, imported_modules=None, deactivate=False ): + def load_datatypes( self, root_dir=None, config=None, deactivate=False ): """ - Parse a datatypes XML file located at root_dir/config. If imported_modules is received, it - is a list of imported datatypes class files included in an installed tool shed repository. - If deactivate is received as True, an installed tool shed repository that includes proprietary - datatypes is being deactivated, so relevant loaded datatypes will be removed from the registry. + Parse a datatypes XML file located at root_dir/config. If deactivate is True, an installed + tool shed repository that includes proprietary datatypes is being deactivated, so appropriate + loaded datatypes will be removed from the registry. """ + def __import_module( full_path, datatype_module ): + sys.path.insert( 0, full_path ) + imported_module = __import__( datatype_module ) + sys.path.pop( 0 ) + return imported_module if root_dir and config: + handling_proprietary_datatypes = False # Parse datatypes_conf.xml tree = galaxy.util.parse_xml( config ) root = tree.getroot() @@ -73,6 +78,11 @@ if not self.display_applications_path: self.display_path_attr = registration.get( 'display_path', 'display_applications' ) self.display_applications_path = os.path.join( root_dir, self.display_path_attr ) + # Proprietary datatype's <registration> tag may have special attributes, proprietary_converter_path and proprietary_display_path. + proprietary_converter_path = registration.get( 'proprietary_converter_path', None ) + proprietary_display_path = registration.get( 'proprietary_display_path', None ) + if proprietary_converter_path or proprietary_display_path and not handling_proprietary_datatypes: + handling_proprietary_datatypes = True for elem in registration.findall( 'datatype' ): try: extension = elem.get( 'extension', None ) @@ -81,6 +91,12 @@ mimetype = elem.get( 'mimetype', None ) display_in_upload = elem.get( 'display_in_upload', False ) make_subclass = galaxy.util.string_as_bool( elem.get( 'subclass', False ) ) + # Proprietary datatypes included in installed tool shed repositories will include two special attributes + # (proprietary_path and proprietary_datatype_module) if they depend on proprietary datatypes classes. + proprietary_path = elem.get( 'proprietary_path', None ) + proprietary_datatype_module = elem.get( 'proprietary_datatype_module', None ) + if proprietary_path or proprietary_datatype_module and not handling_proprietary_datatypes: + handling_proprietary_datatypes = True if deactivate: # We are deactivating an installed tool shed repository, so eliminate the # datatype elem from the in-memory list of datatype elems. @@ -108,12 +124,21 @@ datatype_module = fields[0] datatype_class_name = fields[1] datatype_class = None - if imported_modules: - # See if one of the imported modules contains the datatype class name. - for imported_module in imported_modules: + if proprietary_path and proprietary_datatype_module: + # We need to change the value of sys.path, so do it in a way that is thread-safe. + lock = threading.Lock() + lock.acquire( True ) + try: + imported_module = __import_module( proprietary_path, proprietary_datatype_module ) + if imported_module not in self.imported_modules: + self.imported_modules.append( imported_module ) if hasattr( imported_module, datatype_class_name ): datatype_class = getattr( imported_module, datatype_class_name ) - break + except Exception, e: + full_path = os.path.join( full_path, proprietary_datatype_module ) + self.log.debug( "Exception importing proprietary code file %s: %s" % ( str( full_path ), str( e ) ) ) + finally: + lock.release() if datatype_class is None: # The datatype class name must be contained in one of the datatype modules in the Galaxy distribution. fields = datatype_module.split( '.' ) @@ -130,14 +155,14 @@ self.datatypes_by_extension[ extension ] = datatype_class() if mimetype is None: # Use default mime type as per datatype spec - mimetype = self.datatypes_by_extension[extension].get_mime() - self.mimetypes_by_extension[extension] = mimetype + mimetype = self.datatypes_by_extension[ extension ].get_mime() + self.mimetypes_by_extension[ extension ] = mimetype if hasattr( datatype_class, "get_track_type" ): self.available_tracks.append( extension ) if display_in_upload: self.upload_file_formats.append( extension ) # Max file size cut off for setting optional metadata - self.datatypes_by_extension[extension].max_optional_metadata_filesize = elem.get( 'max_optional_metadata_filesize', None ) + self.datatypes_by_extension[ extension ].max_optional_metadata_filesize = elem.get( 'max_optional_metadata_filesize', None ) for converter in elem.findall( 'converter' ): # Build the list of datatype converters which will later be loaded into the calling app's toolbox. converter_config = converter.get( 'file', None ) @@ -148,7 +173,8 @@ self.converter_deps[extension] = {} self.converter_deps[extension][target_datatype] = depends_on.split(',') if converter_config and target_datatype: - if imported_modules: + #if imported_modules: + if proprietary_converter_path: self.proprietary_converters.append( ( converter_config, extension, target_datatype ) ) else: self.converters.append( ( converter_config, extension, target_datatype ) ) @@ -161,7 +187,8 @@ mimetype = composite_file.get( 'mimetype', None ) self.datatypes_by_extension[extension].add_composite_file( name, optional=optional, mimetype=mimetype ) for display_app in elem.findall( 'display' ): - if imported_modules: + #if imported_modules: + if proprietary_display_path: if elem not in self.proprietary_display_app_containers: self.proprietary_display_app_containers.append( elem ) else: @@ -185,9 +212,10 @@ datatype_module = fields[0] datatype_class_name = fields[1] module = None - if imported_modules: + #if imported_modules: + if handling_proprietary_datatypes: # See if one of the imported modules contains the datatype class name. - for imported_module in imported_modules: + for imported_module in self.imported_modules: if hasattr( imported_module, datatype_class_name ): module = imported_module break @@ -197,13 +225,6 @@ for comp in datatype_module.split( '.' )[ 1: ]: module = getattr( module, comp ) aclass = getattr( module, datatype_class_name )() - # See if we have a conflicting sniffer already loaded. - conflict_loc = None - conflict = False - for conflict_loc, sniffer_class in enumerate( self.sniff_order ): - if sniffer_class.__class__ == aclass.__class__: - conflict = True - break if deactivate: for sniffer_class in self.sniff_order: if sniffer_class.__class__ == aclass.__class__: @@ -211,19 +232,21 @@ break self.log.debug( "Deactivated sniffer for datatype '%s'" % dtype ) else: - if conflict: - # We have a conflicting sniffer, so replace the one previously loaded. - del self.sniff_order[ conflict_loc ] - self.sniff_order.append( aclass ) - self.log.debug( "Replaced conflicting sniffer for datatype '%s'" % dtype ) - else: - self.sniff_order.append( aclass ) - self.log.debug( "Loaded sniffer for datatype '%s'" % dtype ) + # See if we have a conflicting sniffer already loaded. + for conflict_loc, sniffer_class in enumerate( self.sniff_order ): + if sniffer_class.__class__ == aclass.__class__: + # We have a conflicting sniffer, so replace the one previously loaded. + del self.sniff_order[ conflict_loc ] + self.log.debug( "Replaced conflicting sniffer for datatype '%s'" % dtype ) + break + self.sniff_order.append( aclass ) + self.log.debug( "Loaded sniffer for datatype '%s'" % dtype ) except Exception, exc: if deactivate: self.log.warning( "Error deactivating sniffer for datatype '%s': %s" % ( dtype, str( exc ) ) ) else: self.log.warning( "Error appending sniffer for datatype '%s' to sniff_order: %s" % ( dtype, str( exc ) ) ) + self.upload_file_formats.sort() # Persist the xml form of the registry into a temporary file so that it # can be loaded from the command line by tools and set_metadata processing. self.to_xml_file() @@ -385,8 +408,7 @@ app's toolbox. """ if installed_repository_dict: - # Load converters defined by datatypes_conf.xml - # included in installed tool shed repository. + # Load converters defined by datatypes_conf.xml included in installed tool shed repository. converters = self.proprietary_converters else: # Load converters defined by local datatypes_conf.xml. @@ -438,10 +460,9 @@ If deactivate is False, add display applications from self.display_app_containers or self.proprietary_display_app_containers to appropriate datatypes. If deactivate is True, eliminates relevant display applications from appropriate datatypes. - """ + """ if installed_repository_dict: - # Load display applications defined by datatypes_conf.xml - # included in installed tool shed repository. + # Load display applications defined by datatypes_conf.xml included in installed tool shed repository. datatype_elems = self.proprietary_display_app_containers else: # Load display applications defined by local datatypes_conf.xml. @@ -452,7 +473,8 @@ display_file = display_app.get( 'file', None ) if installed_repository_dict: display_path = installed_repository_dict[ 'display_path' ] - config_path = os.path.join( display_path, display_file ) + display_file_head, display_file_tail = os.path.split( display_file ) + config_path = os.path.join( display_path, display_file_tail ) else: config_path = os.path.join( self.display_applications_path, display_file ) try: diff -r ce17eb369f95ccaf2c8331e1a69fe34b1203cff1 -r 62fc9e05383514811e5add9db5f9ad6560a06b20 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -1,4 +1,4 @@ -import os, sys, tempfile, shutil, subprocess, threading, logging +import os, tempfile, shutil, subprocess, logging from datetime import date, datetime, timedelta from time import strftime from galaxy import util @@ -511,7 +511,7 @@ metadata = repository.metadata datatypes_config = metadata.get( 'datatypes_config', None ) if datatypes_config: - converter_path, display_path = load_datatypes( app, datatypes_config, relative_install_dir, deactivate=deactivate ) + converter_path, display_path = alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=deactivate ) if converter_path or display_path: # Create a dictionary of tool shed repository related information. repository_dict = create_repository_dict_for_proprietary_datatypes( tool_shed=repository.tool_shed, @@ -527,15 +527,13 @@ if display_path: # Load or deactivate proprietary datatype display applications app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict, deactivate=deactivate ) -def load_datatypes( app, datatypes_config, relative_install_dir, deactivate=False ): - # This method is used by the InstallManager, which does not have access to trans. - def __import_module( relative_path, datatype_module ): - sys.path.insert( 0, relative_path ) - imported_module = __import__( datatype_module ) - sys.path.pop( 0 ) - return imported_module - imported_modules = [] - # Parse datatypes_config. +def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=False ): + """ + Parse a proprietary datatypes config (a datatypes_conf.xml file included in an installed tool shed repository) and + add information to appropriate elements that will enable proprietary datatype class modules, datatypes converters + and display application to be discovered and properly imported by the datatypes registry. This method is used by + the InstallManager, which does not have access to trans. + """ tree = util.parse_xml( datatypes_config ) datatypes_config_root = tree.getroot() # Path to datatype converters @@ -551,6 +549,7 @@ # <datatype_file name="gmap.py"/> # <datatype_file name="metagenomics.py"/> # </datatype_files> + # We'll add attributes to the datatype tag sets so that the modules can be properly imported by the datatypes registry. for elem in datatype_files.findall( 'datatype_file' ): datatype_file_name = elem.get( 'name', None ) if datatype_file_name: @@ -563,76 +562,91 @@ break break if datatype_class_modules: - # Import each of the datatype class modules. + registration = datatypes_config_root.find( 'registration' ) + converter_path, display_path = get_converter_and_display_paths( registration, relative_install_dir ) + if converter_path: + registration.attrib[ 'proprietary_converter_path' ] = converter_path + if display_path: + registration.attrib[ 'proprietary_display_path' ] = display_path for relative_path_to_datatype_file_name in datatype_class_modules: relative_head, relative_tail = os.path.split( relative_path_to_datatype_file_name ) - registration = datatypes_config_root.find( 'registration' ) - # Get the module by parsing the <datatype> tag. for elem in registration.findall( 'datatype' ): - # A 'type' attribute is currently required. The attribute - # should be something like one of the following: + # Handle 'type' attribute which should be something like one of the following: # type="gmap:GmapDB" # type="galaxy.datatypes.gmap:GmapDB" dtype = elem.get( 'type', None ) if dtype: fields = dtype.split( ':' ) - datatype_module = fields[ 0 ] - if datatype_module.find( '.' ) >= 0: - # Handle the case where datatype_module is "galaxy.datatypes.gmap" - datatype_module = datatype_module.split( '.' )[ -1 ] - datatype_class_name = fields[ 1 ] - # We need to change the value of sys.path, so do it in a way that is thread-safe. - lock = threading.Lock() - lock.acquire( True ) - try: - imported_module = __import_module( relative_head, datatype_module ) - if imported_module not in imported_modules: - imported_modules.append( imported_module ) - except Exception, e: - log.debug( "Exception importing datatypes code file %s: %s" % ( str( relative_path_to_datatype_file_name ), str( e ) ) ) - finally: - lock.release() - # Handle data type converters and display applications. - for elem in registration.findall( 'datatype' ): - if not converter_path: - # If any of the <datatype> tag sets contain <converter> tags, set the converter_path - # if it is not already set. This requires developers to place all converters in the - # same subdirectory within the repository hierarchy. - for converter in elem.findall( 'converter' ): - converter_config = converter.get( 'file', None ) - if converter_config: - for root, dirs, files in os.walk( relative_install_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == converter_config: - converter_path = root - break - if converter_path: - break - if not display_path: - # If any of the <datatype> tag sets contain <display> tags, set the display_path - # if it is not already set. This requires developers to place all display acpplications - # in the same subdirectory within the repository hierarchy. - for display_app in elem.findall( 'display' ): - display_config = display_app.get( 'file', None ) - if display_config: - for root, dirs, files in os.walk( relative_install_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == display_config: - display_path = root - break - if display_path: - break - if converter_path and display_path: - break + proprietary_datatype_module = fields[ 0 ] + if proprietary_datatype_module.find( '.' ) >= 0: + # Handle the case where datatype_module is "galaxy.datatypes.gmap". + proprietary_datatype_module = proprietary_datatype_module.split( '.' )[ -1 ] + # The value of proprietary_path must be an absolute path due to job_working_directory. + elem.attrib[ 'proprietary_path' ] = os.path.abspath( relative_head ) + elem.attrib[ 'proprietary_datatype_module' ] = proprietary_datatype_module + + sniffers = datatypes_config_root.find( 'sniffers' ) + fd, proprietary_datatypes_config = tempfile.mkstemp() + os.write( fd, '<?xml version="1.0"?>\n' ) + os.write( fd, '<datatypes>\n' ) + os.write( fd, '%s' % util.xml_to_string( registration ) ) + os.write( fd, '%s' % util.xml_to_string( sniffers ) ) + os.write( fd, '</datatypes>\n' ) + os.close( fd ) + os.chmod( proprietary_datatypes_config, 0644 ) else: - # The repository includes a dataypes_conf.xml file, but no code file that - # contains data type classes. This implies that the data types in datayptes_conf.xml - # are all subclasses of data types that are in the distribution. - imported_modules = [] + proprietary_datatypes_config = datatypes_config # Load proprietary datatypes - app.datatypes_registry.load_datatypes( root_dir=app.config.root, config=datatypes_config, imported_modules=imported_modules, deactivate=deactivate ) + app.datatypes_registry.load_datatypes( root_dir=app.config.root, config=proprietary_datatypes_config, deactivate=deactivate ) + try: + os.unlink( proprietary_datatypes_config ) + except: + pass + return converter_path, display_path +def get_converter_and_display_paths( registration_elem, relative_install_dir ): + """ + Find the relative path to data type converters and display + applications included in installed tool shed repositories. + """ + converter_path = None + display_path = None + for elem in registration_elem.findall( 'datatype' ): + if not converter_path: + # If any of the <datatype> tag sets contain <converter> tags, set the converter_path + # if it is not already set. This requires developers to place all converters in the + # same subdirectory within the repository hierarchy. + for converter in elem.findall( 'converter' ): + converter_config = converter.get( 'file', None ) + if converter_config: + relative_head, relative_tail = os.path.split( converter_config ) + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == relative_tail: + # The value of converter_path must be absolute due to job_working_directory. + converter_path = os.path.abspath( root ) + break + if converter_path: + break + if not display_path: + # If any of the <datatype> tag sets contain <display> tags, set the display_path + # if it is not already set. This requires developers to place all display acpplications + # in the same subdirectory within the repository hierarchy. + for display_app in elem.findall( 'display' ): + display_config = display_app.get( 'file', None ) + if display_config: + relative_head, relative_tail = os.path.split( display_config ) + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == relative_tail: + # The value of display_path must be absolute due to job_working_directory. + display_path = os.path.abspath( root ) + break + if display_path: + break + if converter_path and display_path: + break return converter_path, display_path def load_repository_contents( app, repository_name, description, owner, changeset_revision, tool_path, repository_clone_url, relative_install_dir, current_working_dir, tmp_name, tool_shed=None, tool_section=None, shed_tool_conf=None, new_install=True, dist_to_shed=False ): @@ -714,7 +728,7 @@ if 'datatypes_config' in metadata_dict: datatypes_config = os.path.abspath( metadata_dict[ 'datatypes_config' ] ) # Load data types required by tools. - converter_path, display_path = load_datatypes( app, datatypes_config, relative_install_dir ) + converter_path, display_path = alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir ) if converter_path or display_path: # Create a dictionary of tool shed repository related information. repository_dict = create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.