1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/74d7ff952ec9/ Changeset: 74d7ff952ec9 User: greg Date: 2014-07-23 17:26:49 Summary: Add a CustomDatatypeLoader for Galaxy installs, eliminate the use of the tool shed's datatyps_util module. Affected #: 7 files diff -r 053943b668af334c480b464fe7351b50284b7099 -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -12,7 +12,6 @@ import tool_shed.repository_types.util as rt_util from tool_shed.util import common_util -from tool_shed.util import datatype_util from tool_shed.util import encoding_util from tool_shed.util import hg_util from tool_shed.util import readme_util @@ -25,6 +24,7 @@ from tool_shed.galaxy_install import dependency_display from tool_shed.galaxy_install import install_manager +from tool_shed.galaxy_install.datatypes import custom_datatype_manager from tool_shed.galaxy_install.grids import admin_toolshed_grids from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager @@ -256,17 +256,17 @@ dmh.remove_from_data_manager( tool_shed_repository ) if tool_shed_repository.includes_datatypes: # Deactivate proprietary datatypes. - installed_repository_dict = datatype_util.load_installed_datatypes( trans.app, - tool_shed_repository, - repository_install_dir, - deactivate=True ) + cdl = custom_datatype_manager.CustomDatatypeLoader( trans.app ) + installed_repository_dict = cdl.load_installed_datatypes( tool_shed_repository, + repository_install_dir, + deactivate=True ) if installed_repository_dict: converter_path = installed_repository_dict.get( 'converter_path' ) if converter_path is not None: - datatype_util.load_installed_datatype_converters( trans.app, installed_repository_dict, deactivate=True ) + cdl.load_installed_datatype_converters( installed_repository_dict, deactivate=True ) display_path = installed_repository_dict.get( 'display_path' ) if display_path is not None: - datatype_util.load_installed_display_applications( trans.app, installed_repository_dict, deactivate=True ) + cdl.load_installed_display_applications( installed_repository_dict, deactivate=True ) if remove_from_disk_checked: try: # Remove the repository from disk. diff -r 053943b668af334c480b464fe7351b50284b7099 -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 lib/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py --- /dev/null +++ b/lib/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py @@ -0,0 +1,218 @@ +import logging +import os +import tempfile + +from galaxy.util import asbool + +from tool_shed.util import basic_util +from tool_shed.util import hg_util +from tool_shed.util import tool_util +from tool_shed.util import shed_util_common as suc +from tool_shed.util import xml_util + +log = logging.getLogger( __name__ ) + + +class CustomDatatypeLoader( object ): + + def __init__( self, app ): + self.app = app + + def alter_config_and_load_prorietary_datatypes( self, datatypes_config, relative_install_dir, + deactivate=False, override=True ): + """ + Parse a custom datatypes config (a datatypes_conf.xml file included in an installed + tool shed repository) and add information to appropriate element attributes that will + enable custom datatype class modules, datatypes converters and display applications + to be discovered and properly imported by the datatypes registry. The value of override + will be False when a tool shed repository is being installed. Since installation is + occurring after the datatypes registry has been initialized, the registry's contents + cannot be overridden by conflicting data types. + """ + tree, error_message = xml_util.parse_xml( datatypes_config ) + if tree is None: + return None, None + datatypes_config_root = tree.getroot() + registration = datatypes_config_root.find( 'registration' ) + if registration is None: + # We have valid XML, but not a valid custom datatypes definition. + return None, None + sniffers = datatypes_config_root.find( 'sniffers' ) + converter_path, display_path = self.get_converter_and_display_paths( registration, + relative_install_dir ) + if converter_path: + # Path to datatype converters + registration.attrib[ 'proprietary_converter_path' ] = converter_path + if display_path: + # Path to datatype display applications + registration.attrib[ 'proprietary_display_path' ] = display_path + relative_path_to_datatype_file_name = None + datatype_files = datatypes_config_root.find( 'datatype_files' ) + datatype_class_modules = [] + if datatype_files is not None: + # The <datatype_files> tag set contains any number of <datatype_file> tags. + # <datatype_files> + # <datatype_file name="gmap.py"/> + # <datatype_file name="metagenomics.py"/> + # </datatype_files> + # We'll add attributes to the datatype tag sets so that the modules can be properly imported + # by the datatypes registry. + for elem in datatype_files.findall( 'datatype_file' ): + datatype_file_name = elem.get( 'name', None ) + if datatype_file_name: + # Find the file in the installed repository. + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == datatype_file_name: + datatype_class_modules.append( os.path.join( root, name ) ) + break + break + if datatype_class_modules: + for relative_path_to_datatype_file_name in datatype_class_modules: + datatype_file_name_path, datatype_file_name = os.path.split( relative_path_to_datatype_file_name ) + for elem in registration.findall( 'datatype' ): + # Handle 'type' attribute which should be something like one of the following: + # type="gmap:GmapDB" + # type="galaxy.datatypes.gmap:GmapDB" + dtype = elem.get( 'type', None ) + if dtype: + fields = dtype.split( ':' ) + proprietary_datatype_module = fields[ 0 ] + if proprietary_datatype_module.find( '.' ) >= 0: + # Handle the case where datatype_module is "galaxy.datatypes.gmap". + proprietary_datatype_module = proprietary_datatype_module.split( '.' )[ -1 ] + # The value of proprietary_path must be an absolute path due to job_working_directory. + elem.attrib[ 'proprietary_path' ] = os.path.abspath( datatype_file_name_path ) + elem.attrib[ 'proprietary_datatype_module' ] = proprietary_datatype_module + # Temporarily persist the custom datatypes configuration file so it can be loaded into the + # datatypes registry. + fd, proprietary_datatypes_config = tempfile.mkstemp( prefix="tmp-toolshed-acalpd" ) + os.write( fd, '<?xml version="1.0"?>\n' ) + os.write( fd, '<datatypes>\n' ) + os.write( fd, '%s' % xml_util.xml_to_string( registration ) ) + if sniffers is not None: + os.write( fd, '%s' % xml_util.xml_to_string( sniffers ) ) + os.write( fd, '</datatypes>\n' ) + os.close( fd ) + os.chmod( proprietary_datatypes_config, 0644 ) + # Load custom datatypes + self.app.datatypes_registry.load_datatypes( root_dir=self.app.config.root, + config=proprietary_datatypes_config, + deactivate=deactivate, + override=override ) + if deactivate: + # Reload the upload tool to eliminate deactivated datatype extensions from the file_type + # select list. + tool_util.reload_upload_tools( self.app ) + else: + self.append_to_datatypes_registry_upload_file_formats( registration ) + tool_util.reload_upload_tools( self.app ) + if datatype_files is not None: + try: + os.unlink( proprietary_datatypes_config ) + except: + pass + return converter_path, display_path + + def append_to_datatypes_registry_upload_file_formats( self, elem ): + # See if we have any datatypes that should be displayed in the upload tool's file_type select list. + for datatype_elem in elem.findall( 'datatype' ): + extension = datatype_elem.get( 'extension', None ) + display_in_upload = datatype_elem.get( 'display_in_upload', None ) + if extension is not None and display_in_upload is not None: + display_in_upload = asbool( str( display_in_upload ) ) + if display_in_upload and extension not in self.app.datatypes_registry.upload_file_formats: + self.app.datatypes_registry.upload_file_formats.append( extension ) + + def create_repository_dict_for_proprietary_datatypes( self, tool_shed, name, owner, installed_changeset_revision, + tool_dicts, converter_path=None, display_path=None ): + return dict( tool_shed=tool_shed, + repository_name=name, + repository_owner=owner, + installed_changeset_revision=installed_changeset_revision, + tool_dicts=tool_dicts, + converter_path=converter_path, + display_path=display_path ) + + def get_converter_and_display_paths( self, registration_elem, relative_install_dir ): + """ + Find the relative path to data type converters and display applications included + in installed tool shed repositories. + """ + converter_path = None + display_path = None + for elem in registration_elem.findall( 'datatype' ): + if not converter_path: + # If any of the <datatype> tag sets contain <converter> tags, set the converter_path + # if it is not already set. This requires developers to place all converters in the + # same subdirectory within the repository hierarchy. + for converter in elem.findall( 'converter' ): + converter_config = converter.get( 'file', None ) + if converter_config: + converter_config_file_name = basic_util.strip_path( converter_config ) + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == converter_config_file_name: + # The value of converter_path must be absolute due to job_working_directory. + converter_path = os.path.abspath( root ) + break + if converter_path: + break + if not display_path: + # If any of the <datatype> tag sets contain <display> tags, set the display_path + # if it is not already set. This requires developers to place all display acpplications + # in the same subdirectory within the repository hierarchy. + for display_app in elem.findall( 'display' ): + display_config = display_app.get( 'file', None ) + if display_config: + display_config_file_name = basic_util.strip_path( display_config ) + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == display_config_file_name: + # The value of display_path must be absolute due to job_working_directory. + display_path = os.path.abspath( root ) + break + if display_path: + break + if converter_path and display_path: + break + return converter_path, display_path + + def load_installed_datatype_converters( self, installed_repository_dict, deactivate=False ): + """Load or deactivate proprietary datatype converters.""" + self.app.datatypes_registry.load_datatype_converters( self.app.toolbox, + installed_repository_dict=installed_repository_dict, + deactivate=deactivate ) + + def load_installed_datatypes( self, repository, relative_install_dir, deactivate=False ): + """ + Load proprietary datatypes and return information needed for loading custom + datatypes converters and display applications later. + """ + metadata = repository.metadata + repository_dict = None + datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, relative_install_dir ) + if datatypes_config: + converter_path, display_path = \ + self.alter_config_and_load_prorietary_datatypes( datatypes_config, + relative_install_dir, + deactivate=deactivate ) + if converter_path or display_path: + # Create a dictionary of tool shed repository related information. + repository_dict = \ + self.create_repository_dict_for_proprietary_datatypes( tool_shed=repository.tool_shed, + name=repository.name, + owner=repository.owner, + installed_changeset_revision=repository.installed_changeset_revision, + tool_dicts=metadata.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) + return repository_dict + + def load_installed_display_applications( self, installed_repository_dict, deactivate=False ): + """Load or deactivate custom datatype display applications.""" + self.app.datatypes_registry.load_display_applications( installed_repository_dict=installed_repository_dict, + deactivate=deactivate ) diff -r 053943b668af334c480b464fe7351b50284b7099 -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -18,7 +18,6 @@ from tool_shed.util import basic_util from tool_shed.util import common_util -from tool_shed.util import datatype_util from tool_shed.util import encoding_util from tool_shed.util import hg_util from tool_shed.util import shed_util_common as suc @@ -26,6 +25,7 @@ from tool_shed.util import tool_util from tool_shed.util import xml_util +from tool_shed.galaxy_install.datatypes import custom_datatype_manager from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager from tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder @@ -594,18 +594,19 @@ files_dir = os.path.join( shed_config_dict[ 'tool_path' ], files_dir ) datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, files_dir ) # Load data types required by tools. + cdl = custom_datatype_manager.CustomDatatypeLoader( self.app ) converter_path, display_path = \ - datatype_util.alter_config_and_load_prorietary_datatypes( self.app, datatypes_config, files_dir, override=False ) + cdl.alter_config_and_load_prorietary_datatypes( datatypes_config, files_dir, override=False ) if converter_path or display_path: # Create a dictionary of tool shed repository related information. repository_dict = \ - datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, - name=tool_shed_repository.name, - owner=tool_shed_repository.owner, - installed_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dicts=metadata_dict.get( 'tools', [] ), - converter_path=converter_path, - display_path=display_path ) + cdl.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, + name=tool_shed_repository.name, + owner=tool_shed_repository.owner, + installed_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dicts=metadata_dict.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) if converter_path: # Load proprietary datatype converters self.app.datatypes_registry.load_datatype_converters( self.app.toolbox, installed_repository_dict=repository_dict ) diff -r 053943b668af334c480b464fe7351b50284b7099 -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 lib/tool_shed/galaxy_install/installed_repository_manager.py --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -7,12 +7,12 @@ from galaxy import util from tool_shed.util import common_util from tool_shed.util import container_util -from tool_shed.util import datatype_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import xml_util from galaxy.model.orm import and_ +from tool_shed.galaxy_install.datatypes import custom_datatype_manager from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager from tool_shed.galaxy_install.tools import data_manager @@ -114,17 +114,17 @@ else: repository_install_dir = os.path.abspath( relative_install_dir ) # Activate proprietary datatypes. - installed_repository_dict = datatype_util.load_installed_datatypes( self.app, - repository, - repository_install_dir, - deactivate=False ) + cdl = custom_datatype_manager.CustomDatatypeLoader( self.app ) + installed_repository_dict = cdl.load_installed_datatypes( repository, + repository_install_dir, + deactivate=False ) if installed_repository_dict: converter_path = installed_repository_dict.get( 'converter_path' ) if converter_path is not None: - datatype_util.load_installed_datatype_converters( self.app, installed_repository_dict, deactivate=False ) + cdl.load_installed_datatype_converters( installed_repository_dict, deactivate=False ) display_path = installed_repository_dict.get( 'display_path' ) if display_path is not None: - datatype_util.load_installed_display_applications( self.app, installed_repository_dict, deactivate=False ) + cdl.load_installed_display_applications( installed_repository_dict, deactivate=False ) def add_entry_to_installed_repository_dependencies_of_installed_repositories( self, repository ): """ @@ -732,22 +732,24 @@ self.add_entry_to_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies( tool_dependency ) def load_proprietary_datatypes( self ): + cdl = custom_datatype_manager.CustomDatatypeLoader( self.app ) for tool_shed_repository in self.context.query( self.install_model.ToolShedRepository ) \ - .filter( and_( self.install_model.ToolShedRepository.table.c.includes_datatypes==True, - self.install_model.ToolShedRepository.table.c.deleted==False ) ) \ - .order_by( self.install_model.ToolShedRepository.table.c.id ): + .filter( and_( self.install_model.ToolShedRepository.table.c.includes_datatypes==True, + self.install_model.ToolShedRepository.table.c.deleted==False ) ) \ + .order_by( self.install_model.ToolShedRepository.table.c.id ): relative_install_dir = self.get_repository_install_dir( tool_shed_repository ) if relative_install_dir: - installed_repository_dict = datatype_util.load_installed_datatypes( self.app, tool_shed_repository, relative_install_dir ) + installed_repository_dict = cdl.load_installed_datatypes( tool_shed_repository, relative_install_dir ) if installed_repository_dict: self.installed_repository_dicts.append( installed_repository_dict ) def load_proprietary_converters_and_display_applications( self, deactivate=False ): + cdl = custom_datatype_manager.CustomDatatypeLoader( self.app ) for installed_repository_dict in self.installed_repository_dicts: if installed_repository_dict[ 'converter_path' ]: - datatype_util.load_installed_datatype_converters( self.app, installed_repository_dict, deactivate=deactivate ) + cdl.load_installed_datatype_converters( installed_repository_dict, deactivate=deactivate ) if installed_repository_dict[ 'display_path' ]: - datatype_util.load_installed_display_applications( self.app, installed_repository_dict, deactivate=deactivate ) + cdl.load_installed_display_applications( installed_repository_dict, deactivate=deactivate ) def purge_repository( self, repository ): """Purge a repository with status New (a white ghost) from the database.""" diff -r 053943b668af334c480b464fe7351b50284b7099 -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -14,6 +14,7 @@ from galaxy.util.odict import odict from tool_shed.galaxy_install import install_manager +from tool_shed.galaxy_install.datatypes import custom_datatype_manager from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.tools import tool_panel_manager @@ -22,7 +23,6 @@ from tool_shed.util import basic_util from tool_shed.util import common_util -from tool_shed.util import datatype_util from tool_shed.util import hg_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util @@ -492,6 +492,7 @@ print '\nThe ToolMigrationManager returned the following error while installing tool dependency ', installed_tool_dependency.name, ':' print installed_tool_dependency.error_message, '\n\n' if 'datatypes' in metadata_dict: + cdl = custom_datatype_manager.CustomDatatypeLoader( self.app ) tool_shed_repository.status = self.app.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES if not tool_shed_repository.includes_datatypes: tool_shed_repository.includes_datatypes = True @@ -499,21 +500,27 @@ self.app.install_model.context.flush() work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-hrc" ) datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, repo_install_dir ) - # Load proprietary data types required by tools. The value of override is not important here since the Galaxy server will be started - # after this installation completes. - converter_path, display_path = datatype_util.alter_config_and_load_prorietary_datatypes( self.app, datatypes_config, repo_install_dir, override=False ) #repo_install_dir was relative_install_dir + # Load proprietary data types required by tools. The value of override is not + # important here since the Galaxy server will be started after this installation + #completes. + converter_path, display_path = \ + cdl.alter_config_and_load_prorietary_datatypes( datatypes_config, + repo_install_dir, + override=False ) if converter_path or display_path: # Create a dictionary of tool shed repository related information. - repository_dict = datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=self.tool_shed_url, - name=tool_shed_repository.name, - owner=self.repository_owner, - installed_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dicts=metadata_dict.get( 'tools', [] ), - converter_path=converter_path, - display_path=display_path ) + repository_dict = \ + cdl.create_repository_dict_for_proprietary_datatypes( tool_shed=self.tool_shed_url, + name=tool_shed_repository.name, + owner=self.repository_owner, + installed_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dicts=metadata_dict.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) if converter_path: # Load proprietary datatype converters - self.app.datatypes_registry.load_datatype_converters( self.toolbox, installed_repository_dict=repository_dict ) + self.app.datatypes_registry.load_datatype_converters( self.toolbox, + installed_repository_dict=repository_dict ) if display_path: # Load proprietary datatype display applications self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) diff -r 053943b668af334c480b464fe7351b50284b7099 -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 lib/tool_shed/util/datatype_util.py --- a/lib/tool_shed/util/datatype_util.py +++ /dev/null @@ -1,187 +0,0 @@ -import logging -import os -import tempfile -from galaxy import eggs -from galaxy.util import asbool -from tool_shed.util import basic_util -from tool_shed.util import hg_util -from tool_shed.util import tool_util -from tool_shed.util import xml_util -import tool_shed.util.shed_util_common as suc - -log = logging.getLogger( __name__ ) - -def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=False, override=True ): - """ - Parse a proprietary datatypes config (a datatypes_conf.xml file included in an installed tool shed repository) and - add information to appropriate element attributes that will enable proprietary datatype class modules, datatypes converters - and display applications to be discovered and properly imported by the datatypes registry. The value of override will - be False when a tool shed repository is being installed. Since installation is occurring after the datatypes registry - has been initialized, the registry's contents cannot be overridden by conflicting data types. - """ - tree, error_message = xml_util.parse_xml( datatypes_config ) - if tree is None: - return None, None - datatypes_config_root = tree.getroot() - registration = datatypes_config_root.find( 'registration' ) - if registration is None: - # We have valid XML, but not a valid proprietary datatypes definition. - return None, None - sniffers = datatypes_config_root.find( 'sniffers' ) - converter_path, display_path = get_converter_and_display_paths( registration, relative_install_dir ) - if converter_path: - # Path to datatype converters - registration.attrib[ 'proprietary_converter_path' ] = converter_path - if display_path: - # Path to datatype display applications - registration.attrib[ 'proprietary_display_path' ] = display_path - relative_path_to_datatype_file_name = None - datatype_files = datatypes_config_root.find( 'datatype_files' ) - datatype_class_modules = [] - if datatype_files is not None: - # The <datatype_files> tag set contains any number of <datatype_file> tags. - # <datatype_files> - # <datatype_file name="gmap.py"/> - # <datatype_file name="metagenomics.py"/> - # </datatype_files> - # We'll add attributes to the datatype tag sets so that the modules can be properly imported by the datatypes registry. - for elem in datatype_files.findall( 'datatype_file' ): - datatype_file_name = elem.get( 'name', None ) - if datatype_file_name: - # Find the file in the installed repository. - for root, dirs, files in os.walk( relative_install_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == datatype_file_name: - datatype_class_modules.append( os.path.join( root, name ) ) - break - break - if datatype_class_modules: - for relative_path_to_datatype_file_name in datatype_class_modules: - datatype_file_name_path, datatype_file_name = os.path.split( relative_path_to_datatype_file_name ) - for elem in registration.findall( 'datatype' ): - # Handle 'type' attribute which should be something like one of the following: - # type="gmap:GmapDB" - # type="galaxy.datatypes.gmap:GmapDB" - dtype = elem.get( 'type', None ) - if dtype: - fields = dtype.split( ':' ) - proprietary_datatype_module = fields[ 0 ] - if proprietary_datatype_module.find( '.' ) >= 0: - # Handle the case where datatype_module is "galaxy.datatypes.gmap". - proprietary_datatype_module = proprietary_datatype_module.split( '.' )[ -1 ] - # The value of proprietary_path must be an absolute path due to job_working_directory. - elem.attrib[ 'proprietary_path' ] = os.path.abspath( datatype_file_name_path ) - elem.attrib[ 'proprietary_datatype_module' ] = proprietary_datatype_module - # Temporarily persist the proprietary datatypes configuration file so it can be loaded into the datatypes registry. - fd, proprietary_datatypes_config = tempfile.mkstemp( prefix="tmp-toolshed-acalpd" ) - os.write( fd, '<?xml version="1.0"?>\n' ) - os.write( fd, '<datatypes>\n' ) - os.write( fd, '%s' % xml_util.xml_to_string( registration ) ) - if sniffers is not None: - os.write( fd, '%s' % xml_util.xml_to_string( sniffers ) ) - os.write( fd, '</datatypes>\n' ) - os.close( fd ) - os.chmod( proprietary_datatypes_config, 0644 ) - # Load proprietary datatypes - app.datatypes_registry.load_datatypes( root_dir=app.config.root, config=proprietary_datatypes_config, deactivate=deactivate, override=override ) - if deactivate: - # Reload the upload tool to eliminate deactivated datatype extensions from the file_type select list. - tool_util.reload_upload_tools( app ) - else: - append_to_datatypes_registry_upload_file_formats( app, registration ) - tool_util.reload_upload_tools( app ) - if datatype_files is not None: - try: - os.unlink( proprietary_datatypes_config ) - except: - pass - return converter_path, display_path - -def append_to_datatypes_registry_upload_file_formats( app, elem ): - # See if we have any datatypes that should be displayed in the upload tool's file_type select list. - for datatype_elem in elem.findall( 'datatype' ): - extension = datatype_elem.get( 'extension', None ) - display_in_upload = datatype_elem.get( 'display_in_upload', None ) - if extension is not None and display_in_upload is not None: - display_in_upload = asbool( str( display_in_upload ) ) - if display_in_upload and extension not in app.datatypes_registry.upload_file_formats: - app.datatypes_registry.upload_file_formats.append( extension ) - -def create_repository_dict_for_proprietary_datatypes( tool_shed, name, owner, installed_changeset_revision, tool_dicts, converter_path=None, display_path=None ): - return dict( tool_shed=tool_shed, - repository_name=name, - repository_owner=owner, - installed_changeset_revision=installed_changeset_revision, - tool_dicts=tool_dicts, - converter_path=converter_path, - display_path=display_path ) - -def get_converter_and_display_paths( registration_elem, relative_install_dir ): - """Find the relative path to data type converters and display applications included in installed tool shed repositories.""" - converter_path = None - display_path = None - for elem in registration_elem.findall( 'datatype' ): - if not converter_path: - # If any of the <datatype> tag sets contain <converter> tags, set the converter_path - # if it is not already set. This requires developers to place all converters in the - # same subdirectory within the repository hierarchy. - for converter in elem.findall( 'converter' ): - converter_config = converter.get( 'file', None ) - if converter_config: - converter_config_file_name = basic_util.strip_path( converter_config ) - for root, dirs, files in os.walk( relative_install_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == converter_config_file_name: - # The value of converter_path must be absolute due to job_working_directory. - converter_path = os.path.abspath( root ) - break - if converter_path: - break - if not display_path: - # If any of the <datatype> tag sets contain <display> tags, set the display_path - # if it is not already set. This requires developers to place all display acpplications - # in the same subdirectory within the repository hierarchy. - for display_app in elem.findall( 'display' ): - display_config = display_app.get( 'file', None ) - if display_config: - display_config_file_name = basic_util.strip_path( display_config ) - for root, dirs, files in os.walk( relative_install_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == display_config_file_name: - # The value of display_path must be absolute due to job_working_directory. - display_path = os.path.abspath( root ) - break - if display_path: - break - if converter_path and display_path: - break - return converter_path, display_path - -def load_installed_datatype_converters( app, installed_repository_dict, deactivate=False ): - # Load or deactivate proprietary datatype converters - app.datatypes_registry.load_datatype_converters( app.toolbox, installed_repository_dict=installed_repository_dict, deactivate=deactivate ) - -def load_installed_datatypes( app, repository, relative_install_dir, deactivate=False ): - # Load proprietary datatypes and return information needed for loading proprietary datatypes converters and display applications later. - metadata = repository.metadata - repository_dict = None - datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, relative_install_dir ) - if datatypes_config: - converter_path, display_path = alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=deactivate ) - if converter_path or display_path: - # Create a dictionary of tool shed repository related information. - repository_dict = create_repository_dict_for_proprietary_datatypes( tool_shed=repository.tool_shed, - name=repository.name, - owner=repository.owner, - installed_changeset_revision=repository.installed_changeset_revision, - tool_dicts=metadata.get( 'tools', [] ), - converter_path=converter_path, - display_path=display_path ) - return repository_dict - -def load_installed_display_applications( app, installed_repository_dict, deactivate=False ): - # Load or deactivate proprietary datatype display applications - app.datatypes_registry.load_display_applications( installed_repository_dict=installed_repository_dict, deactivate=deactivate ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.