1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/4d131422777f/ changeset: 4d131422777f user: greg date: 2012-01-05 18:24:45 summary: Fixes and enhancements for supporting tool shed repositories that include data types that use class modules included in the repository. More than 1 class modules can now be included in the repository. affected #: 4 files diff -r 82af6b933a0af1c4af0958aa378af846e7efc449 -r 4d131422777f436f3b3eebfc7a9f61cec61810b1 lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -71,7 +71,7 @@ # Manage installed tool shed repositories self.installed_repository_manager = galaxy.tool_shed.InstalledRepositoryManager( self ) # Add additional datatypes from installed tool shed repositories to the datatypes registry. - self.installed_repository_manager.load_datatypes() + self.installed_repository_manager.load_proprietary_datatypes() # Load datatype converters self.datatypes_registry.load_datatype_converters( self.toolbox ) # Load history import/export tools diff -r 82af6b933a0af1c4af0958aa378af846e7efc449 -r 4d131422777f436f3b3eebfc7a9f61cec61810b1 lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -36,7 +36,7 @@ self.datatype_elems = [] self.sniffer_elems = [] self.xml_filename = None - def load_datatypes( self, root_dir=None, config=None, imported_module=None ): + def load_datatypes( self, root_dir=None, config=None, imported_modules=None ): if root_dir and config: inherit_display_application_by_class = [] # Parse datatypes_conf.xml @@ -81,8 +81,11 @@ fields = dtype.split( ':' ) datatype_module = fields[0] datatype_class_name = fields[1] - if imported_module: - datatype_class = getattr( imported_module, datatype_class_name ) + if imported_modules: + for imported_module in imported_modules: + if hasattr( imported_module, datatype_class_name ): + datatype_class = getattr( imported_module, datatype_class_name ) + break else: fields = datatype_module.split( '.' ) module = __import__( fields.pop(0) ) diff -r 82af6b933a0af1c4af0958aa378af846e7efc449 -r 4d131422777f436f3b3eebfc7a9f61cec61810b1 lib/galaxy/tool_shed/__init__.py --- a/lib/galaxy/tool_shed/__init__.py +++ b/lib/galaxy/tool_shed/__init__.py @@ -2,6 +2,7 @@ Classes encapsulating the management of repositories installed from Galaxy tool sheds. """ import os, logging +from galaxy.util.shed_util import load_datatypes from galaxy.model.orm import * log = logging.getLogger(__name__) @@ -11,13 +12,16 @@ self.app = app self.model = self.app.model self.sa_session = self.model.context.current - def load_datatypes( self ): + def load_proprietary_datatypes( self ): for tool_shed_repository in self.sa_session.query( self.model.ToolShedRepository ) \ .filter( and_( self.model.ToolShedRepository.table.c.includes_datatypes==True, self.model.ToolShedRepository.table.c.deleted==False ) ) \ .order_by( self.model.ToolShedRepository.table.c.id ): metadata = tool_shed_repository.metadata datatypes_config = metadata[ 'datatypes_config' ] - full_path = os.path.abspath( datatypes_config ) - self.app.datatypes_registry.load_datatypes( self.app.config.root, full_path ) + # We need the repository installation directory, which we can get from the path to the datatypes config. + path_items = datatypes_config.split( 'repos' ) + relative_install_dir = '%srepos/%s/%s/%s' % \ + ( path_items[0], tool_shed_repository.owner, tool_shed_repository.name, tool_shed_repository.installed_changeset_revision ) + load_datatypes( self.app, datatypes_config, relative_install_dir ) \ No newline at end of file diff -r 82af6b933a0af1c4af0958aa378af846e7efc449 -r 4d131422777f436f3b3eebfc7a9f61cec61810b1 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -1,4 +1,4 @@ -import os, tempfile, shutil, subprocess, logging +import os, sys, tempfile, shutil, subprocess, threading, logging from datetime import date, datetime, timedelta from time import strftime from galaxy import util @@ -439,14 +439,20 @@ error = tmp_stderr.read() tmp_stderr.close() log.debug( 'Problem installing dependencies for tool "%s"\n%s' % ( repository_tool.name, error ) ) -def load_datatypes( app, datatypes_config, relative_intall_dir ): +def load_datatypes( app, datatypes_config, relative_install_dir ): # This method is used by the InstallManager, which does not have access to trans. - imported_module = None + def __import_module( relative_path, datatype_module ): + sys.path.insert( 0, relative_path ) + imported_module = __import__( datatype_module ) + sys.path.pop( 0 ) + return imported_module + imported_modules = [] # Parse datatypes_config. tree = util.parse_xml( datatypes_config ) datatypes_config_root = tree.getroot() relative_path_to_datatype_file_name = None datatype_files = datatypes_config_root.find( 'datatype_files' ) + datatype_class_modules = [] if datatype_files: # Currently only a single datatype_file is supported. For example: # <datatype_files> @@ -456,40 +462,47 @@ datatype_file_name = elem.get( 'name', None ) if datatype_file_name: # Find the file in the installed repository. - for root, dirs, files in os.walk( relative_intall_dir ): + for root, dirs, files in os.walk( relative_install_dir ): if root.find( '.hg' ) < 0: for name in files: if name == datatype_file_name: - relative_path_to_datatype_file_name = os.path.join( root, name ) + datatype_class_modules.append( os.path.join( root, name ) ) break break - if relative_path_to_datatype_file_name: - relative_head, relative_tail = os.path.split( relative_path_to_datatype_file_name ) - registration = datatypes_config_root.find( 'registration' ) - # Get the module by parsing the <datatype> tag. - for elem in registration.findall( 'datatype' ): - # A 'type' attribute is currently required. The attribute - # should be something like: type="gmap:GmapDB". - dtype = elem.get( 'type', None ) - if dtype: - fields = dtype.split( ':' ) - datatype_module = fields[0] - datatype_class_name = fields[1] - # Since we currently support only a single datatype_file, - # we have what we need. - break - try: - sys.path.insert( 0, relative_head ) - imported_module = __import__( datatype_module ) - sys.path.pop( 0 ) - except Exception, e: - log.debug( "Exception importing datatypes code file included in installed repository: %s" % str( e ) ) + if datatype_class_modules: + for relative_path_to_datatype_file_name in datatype_class_modules: + relative_head, relative_tail = os.path.split( relative_path_to_datatype_file_name ) + registration = datatypes_config_root.find( 'registration' ) + # Get the module by parsing the <datatype> tag. + for elem in registration.findall( 'datatype' ): + # A 'type' attribute is currently required. The attribute + # should be something like one of the following: + # type="gmap:GmapDB" + # type="galaxy.datatypes.gmap:GmapDB" + dtype = elem.get( 'type', None ) + if dtype: + fields = dtype.split( ':' ) + datatype_module = fields[ 0 ] + if datatype_module.find( '.' ) >= 0: + # Handle the case where datatype_module is "galaxy.datatypes.gmap" + datatype_module = datatype_module.split( '.' )[ -1 ] + datatype_class_name = fields[ 1 ] + # We need to change the value of sys.path, so do it in a way that is thread-safe. + lock = threading.Lock() + lock.acquire( True ) + try: + imported_module = __import_module( relative_head, datatype_module ) + imported_modules.append( imported_module ) + except Exception, e: + log.debug( "Exception importing datatypes code file %s: %s" % ( str( relative_path_to_datatype_file_name ), str( e ) ) ) + finally: + lock.release() else: - # The repository includes a datayptes_conf.xml file, but no code file that + # The repository includes a dataypes_conf.xml file, but no code file that # contains data type classes. This implies that the data types in datayptes_conf.xml # are all subclasses of data types that are in the distribution. - imported_module = None - app.datatypes_registry.load_datatypes( root_dir=app.config.root, config=datatypes_config, imported_module=imported_module ) + imported_modules = [] + app.datatypes_registry.load_datatypes( root_dir=app.config.root, config=datatypes_config, imported_modules=imported_modules ) def load_repository_contents( app, name, description, owner, changeset_revision, tool_path, repository_clone_url, relative_install_dir, current_working_dir, tmp_name, tool_section=None, shed_tool_conf=None, new_install=True ): # This method is used by the InstallManager, which does not have access to trans. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.