[galaxy-commits] commit/galaxy-central: 2 new changesets

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/87f7abe35390/ Changeset: 87f7abe35390 User: jmchilton Date: 2014-09-22 01:48:58+00:00 Summary: Reduce code duplication between job metrics and dependency resolution plugins. Same code reused again downstream in workflow schedulers plugin framework. Affected #: 3 files diff -r 6fe929724b066bd8b61ee0149479aba1e4e42f56 -r 87f7abe35390d8822f04e3e03dcc0598b5bf076c lib/galaxy/jobs/metrics/__init__.py --- a/lib/galaxy/jobs/metrics/__init__.py +++ b/lib/galaxy/jobs/metrics/__init__.py @@ -3,7 +3,7 @@ from xml.etree import ElementTree -from galaxy.util.submodules import submodules +from galaxy.util import plugin_config from galaxy import util from ..metrics import formatting @@ -49,17 +49,8 @@ return self.job_instrumenters[ destination_id ].collect_properties( job_id, job_directory ) def __plugins_dict( self ): - plugin_dict = {} - for plugin_module in self.__plugin_modules(): - for clazz in plugin_module.__all__: - plugin_type = getattr( clazz, 'plugin_type', None ) - if plugin_type: - plugin_dict[ plugin_type ] = clazz - return plugin_dict - - def __plugin_modules( self ): import galaxy.jobs.metrics.instrumenters - return submodules( galaxy.jobs.metrics.instrumenters ) + return plugin_config.plugins_dict( galaxy.jobs.metrics.instrumenters, 'plugin_type' ) class NullJobInstrumenter( object ): @@ -117,14 +108,7 @@ return per_plugin_properites def __plugins_for_element( self, plugins_element ): - plugins = [] - for plugin_element in plugins_element.getchildren(): - plugin_type = plugin_element.tag - plugin_kwds = dict( plugin_element.items() ) - plugin_kwds.update( self.extra_kwargs ) - plugin = self.plugin_classes[ plugin_type ]( **plugin_kwds ) - plugins.append( plugin ) - return plugins + return plugin_config.load_plugins_from_element(self.plugin_classes, plugins_element, self.extra_kwargs) @staticmethod def from_file( plugin_classes, conf_file, **kwargs ): diff -r 6fe929724b066bd8b61ee0149479aba1e4e42f56 -r 87f7abe35390d8822f04e3e03dcc0598b5bf076c lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -12,7 +12,7 @@ from .resolvers import INDETERMINATE_DEPENDENCY from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver -from galaxy.util.submodules import submodules +from galaxy.util import plugin_config def build_dependency_manager( config ): @@ -110,24 +110,9 @@ :param root: Object representing the root ``<dependency_resolvers>`` object in the file. :type root: ``xml.etree.ElementTree.Element`` """ - resolvers = [] - resolvers_element = root - for resolver_element in resolvers_element.getchildren(): - resolver_type = resolver_element.tag - resolver_kwds = dict(resolver_element.items()) - resolver = self.resolver_classes[resolver_type](self, **resolver_kwds) - resolvers.append(resolver) - return resolvers + extra_kwds = dict( dependency_manager=self ) + return plugin_config.load_plugins_from_element( self.resolver_classes, root, extra_kwds ) def __resolvers_dict( self ): - resolver_dict = {} - for resolver_module in self.__resolver_modules(): - for clazz in resolver_module.__all__: - resolver_type = getattr(clazz, 'resolver_type', None) - if resolver_type: - resolver_dict[resolver_type] = clazz - return resolver_dict - - def __resolver_modules( self ): import galaxy.tools.deps.resolvers - return submodules( galaxy.tools.deps.resolvers ) + return plugin_config.plugins_dict( galaxy.tools.deps.resolvers, 'resolver_type' ) diff -r 6fe929724b066bd8b61ee0149479aba1e4e42f56 -r 87f7abe35390d8822f04e3e03dcc0598b5bf076c lib/galaxy/util/plugin_config.py --- /dev/null +++ b/lib/galaxy/util/plugin_config.py @@ -0,0 +1,32 @@ +from galaxy.util.submodules import submodules + + +def plugins_dict(module, plugin_type_identifier): + """ Walk through all classes in submodules of module and find ones labelled + with specified plugin_type_identifier and throw in a dictionary to allow + constructions from plugins by these types later on. + """ + plugin_dict = {} + + for plugin_module in submodules( module ): + # FIXME: this is not how one is suppose to use __all__ why did you do + # this past John? + for clazz in plugin_module.__all__: + plugin_type = getattr( clazz, plugin_type_identifier, None ) + if plugin_type: + plugin_dict[ plugin_type ] = clazz + + return plugin_dict + + +def load_plugins_from_element(plugins_dict, plugins_element, extra_kwds={}): + plugins = [] + + for plugin_element in plugins_element.getchildren(): + plugin_type = plugin_element.tag + plugin_kwds = dict( plugin_element.items() ) + plugin_kwds.update( extra_kwds ) + plugin = plugins_dict[ plugin_type ]( **plugin_kwds ) + plugins.append( plugin ) + + return plugins https://bitbucket.org/galaxy/galaxy-central/commits/5fac4d4b7d7d/ Changeset: 5fac4d4b7d7d User: jmchilton Date: 2014-09-22 01:48:58+00:00 Summary: Allow loading dependency resolvers and job metrics from YAML instead of XML. Mostly for downstream Pulsar configuration improvements - but I don't see a reason not to allow this in Galaxy as well. Besides simply being more pleasant to write - having everything configurable by Python dictionary should enable easier automation and programatic configuration of these job running systems. Affected #: 3 files diff -r 87f7abe35390d8822f04e3e03dcc0598b5bf076c -r 5fac4d4b7d7d6b056056bf2a6094c7777dc28a71 lib/galaxy/jobs/metrics/__init__.py --- a/lib/galaxy/jobs/metrics/__init__.py +++ b/lib/galaxy/jobs/metrics/__init__.py @@ -1,8 +1,6 @@ import collections import os -from xml.etree import ElementTree - from galaxy.util import plugin_config from galaxy import util @@ -37,7 +35,7 @@ self.set_destination_instrumenter( destination_id, instrumenter ) def set_destination_conf_element( self, destination_id, element ): - instrumenter = JobInstrumenter( self.plugin_classes, element ) + instrumenter = JobInstrumenter( self.plugin_classes, ('xml', element) ) self.set_destination_instrumenter( destination_id, instrumenter ) def set_destination_instrumenter( self, destination_id, job_instrumenter=None ): @@ -69,10 +67,10 @@ class JobInstrumenter( object ): - def __init__( self, plugin_classes, metrics_element, **kwargs ): + def __init__( self, plugin_classes, plugins_source, **kwargs ): self.extra_kwargs = kwargs self.plugin_classes = plugin_classes - self.plugins = self.__plugins_for_element( metrics_element ) + self.plugins = self.__plugins_from_source( plugins_source ) def pre_execute_commands( self, job_directory ): commands = [] @@ -107,12 +105,12 @@ log.exception( "Failed to collect job properties for plugin %s" % plugin ) return per_plugin_properites - def __plugins_for_element( self, plugins_element ): - return plugin_config.load_plugins_from_element(self.plugin_classes, plugins_element, self.extra_kwargs) + def __plugins_from_source( self, plugins_source ): + return plugin_config.load_plugins(self.plugin_classes, plugins_source, self.extra_kwargs) @staticmethod def from_file( plugin_classes, conf_file, **kwargs ): if not conf_file or not os.path.exists( conf_file ): return NULL_JOB_INSTRUMENTER - plugins_element = ElementTree.parse( conf_file ).getroot() - return JobInstrumenter( plugin_classes, plugins_element, **kwargs ) + plugins_source = plugin_config.plugin_source_from_path( conf_file ) + return JobInstrumenter( plugin_classes, plugins_source, **kwargs ) diff -r 87f7abe35390d8822f04e3e03dcc0598b5bf076c -r 5fac4d4b7d7d6b056056bf2a6094c7777dc28a71 lib/galaxy/tools/deps/__init__.py --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -7,8 +7,6 @@ import logging log = logging.getLogger( __name__ ) -from xml.etree import ElementTree - from .resolvers import INDETERMINATE_DEPENDENCY from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver @@ -94,8 +92,8 @@ def __build_dependency_resolvers( self, conf_file ): if not conf_file or not os.path.exists( conf_file ): return self.__default_dependency_resolvers() - root = ElementTree.parse( conf_file ).getroot() - return self.__parse_resolver_conf_xml( root ) + plugin_source = plugin_config.plugin_source_from_path( conf_file ) + return self.__parse_resolver_conf_xml( plugin_source ) def __default_dependency_resolvers( self ): return [ @@ -104,14 +102,11 @@ GalaxyPackageDependencyResolver(self, versionless=True), ] - def __parse_resolver_conf_xml(self, root): + def __parse_resolver_conf_xml(self, plugin_source): """ - - :param root: Object representing the root ``<dependency_resolvers>`` object in the file. - :type root: ``xml.etree.ElementTree.Element`` """ extra_kwds = dict( dependency_manager=self ) - return plugin_config.load_plugins_from_element( self.resolver_classes, root, extra_kwds ) + return plugin_config.load_plugins( self.resolver_classes, plugin_source, extra_kwds ) def __resolvers_dict( self ): import galaxy.tools.deps.resolvers diff -r 87f7abe35390d8822f04e3e03dcc0598b5bf076c -r 5fac4d4b7d7d6b056056bf2a6094c7777dc28a71 lib/galaxy/util/plugin_config.py --- a/lib/galaxy/util/plugin_config.py +++ b/lib/galaxy/util/plugin_config.py @@ -1,3 +1,16 @@ +from xml.etree import ElementTree + +try: + from galaxy import eggs + eggs.require('PyYAML') +except Exception: + # If not in Galaxy, ignore this. + pass +try: + import yaml +except ImportError: + yaml = None + from galaxy.util.submodules import submodules @@ -19,7 +32,15 @@ return plugin_dict -def load_plugins_from_element(plugins_dict, plugins_element, extra_kwds={}): +def load_plugins(plugins_dict, plugin_source, extra_kwds={}): + source_type, source = plugin_source + if source_type == "xml": + return __load_plugins_from_element(plugins_dict, source, extra_kwds) + else: + return __load_plugins_from_dicts(plugins_dict, source, extra_kwds) + + +def __load_plugins_from_element(plugins_dict, plugins_element, extra_kwds): plugins = [] for plugin_element in plugins_element.getchildren(): @@ -30,3 +51,31 @@ plugins.append( plugin ) return plugins + + +def __load_plugins_from_dicts(plugins_dict, configs, extra_kwds): + plugins = [] + + for config in configs: + plugin_type = config[ "type" ] + plugin_kwds = config + plugin_kwds.update( extra_kwds ) + plugin = plugins_dict[ plugin_type ]( **plugin_kwds ) + plugins.append( plugin ) + + return plugins + + +def plugin_source_from_path(path): + if path.endswith(".yaml") or path.endswith(".yml"): + return ('dict', __read_yaml(path)) + else: + return ('xml', ElementTree.parse( path ).getroot()) + + +def __read_yaml(path): + if yaml is None: + raise ImportError("Attempting to read YAML configuration file - but PyYAML dependency unavailable.") + + with open(path, "rb") as f: + return yaml.load(f) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.