1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/650d2307e5b5/ Changeset: 650d2307e5b5 User: jmchilton Date: 2014-09-30 16:52:47+00:00 Summary: Merged in jmchilton/galaxy-central-fork-1 (pull request #509) Options to load and watch directories of tools, auto reload on update of any tool. Affected #: 5 files diff -r c40b29fb9fbc96c8ca2efb8b3dc4e789e2063d0b -r 650d2307e5b5688ca8137fc4af577308fac54833 config/galaxy.ini.sample --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -172,6 +172,12 @@ # install from in the admin interface (.sample used if default does not exist). #tool_sheds_config_file = config/tool_sheds_conf.xml +# If the following option is set to True - Galaxy will monitor individual tools +# and tool directories specified in tool_conf.xml for changes and reload these +# tools. Watchdog must be installed and available to Galaxy to use this option. +# See https://pypi.python.org/pypi/watchdog. +#watch_tools = False + # Enable automatic polling of relative tool sheds to see if any updates # are available for installed repositories. Ideally only one Galaxy # server process should be able to check for repository updates. The diff -r c40b29fb9fbc96c8ca2efb8b3dc4e789e2063d0b -r 650d2307e5b5688ca8137fc4af577308fac54833 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -229,6 +229,7 @@ self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None ) self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False ) self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' ) + self.watch_tools = kwargs.get( 'watch_tools', False ) # Location for tool dependencies. if 'tool_dependency_dir' in kwargs: self.tool_dependency_dir = resolve_path( kwargs.get( "tool_dependency_dir" ), self.root ) diff -r c40b29fb9fbc96c8ca2efb8b3dc4e789e2063d0b -r 650d2307e5b5688ca8137fc4af577308fac54833 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -39,6 +39,7 @@ from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper from galaxy import exceptions from galaxy.jobs import ParallelismInfo +from galaxy.tools import watcher from galaxy.tools.actions import DefaultToolAction from galaxy.tools.actions.data_source import DataSourceToolAction from galaxy.tools.actions.data_manager import DataManagerToolAction @@ -143,6 +144,7 @@ # (e.g., shed_tool_conf.xml) files include the tool_path attribute within the <toolbox> tag. self.tool_root_dir = tool_root_dir self.app = app + self.tool_watcher = watcher.get_watcher( self, app.config ) self.filter_factory = FilterFactory( self ) self.init_dependency_manager() config_filenames = listify( config_filenames ) @@ -232,6 +234,8 @@ self.load_section_tag_set( elem, tool_path, load_panel_dict, index=index ) elif elem.tag == 'label': self.load_label_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index ) + elif elem.tag == 'tool_dir': + self.__watch_directory( elem, self.tool_panel, self.integrated_tool_panel, tool_path, load_panel_dict ) if parsing_shed_tool_conf: shed_tool_conf_dict = dict( config_filename=config_filename, tool_path=tool_path, @@ -614,15 +618,7 @@ tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id ) self.sa_session.add( tta ) self.sa_session.flush() - # Allow for the same tool to be loaded into multiple places in the tool panel. We have to handle - # the case where the tool is contained in a repository installed from the tool shed, and the Galaxy - # administrator has retrieved updates to the installed repository. In this case, the tool may have - # been updated, but the version was not changed, so the tool should always be reloaded here. We used - # to only load the tool if it was not found in self.tools_by_id, but performing that check did - # not enable this scenario. - self.tools_by_id[ tool.id ] = tool - if load_panel_dict: - self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) ) + self.__add_tool( tool, load_panel_dict, panel_dict ) # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. if key in integrated_panel_dict or index is None: integrated_panel_dict[ key ] = tool @@ -631,6 +627,17 @@ except: log.exception( "Error reading tool from path: %s" % path ) + def __add_tool( self, tool, load_panel_dict, panel_dict ): + # Allow for the same tool to be loaded into multiple places in the tool panel. We have to handle + # the case where the tool is contained in a repository installed from the tool shed, and the Galaxy + # administrator has retrieved updates to the installed repository. In this case, the tool may have + # been updated, but the version was not changed, so the tool should always be reloaded here. We used + # to only load the tool if it was not found in self.tools_by_id, but performing that check did + # not enable this scenario. + self.tools_by_id[ tool.id ] = tool + if load_panel_dict: + self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) ) + def load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): try: # TODO: should id be encoded? @@ -679,6 +686,8 @@ self.load_workflow_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index ) elif sub_elem.tag == 'label': self.load_label_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index ) + elif sub_elem.tag == 'tool_dir': + self.__watch_directory( sub_elem, elems, tool_path, integrated_elems, load_panel_dict ) if load_panel_dict: self.tool_panel[ key ] = section # Always load sections into the integrated_tool_panel. @@ -687,6 +696,38 @@ else: self.integrated_tool_panel.insert( index, key, integrated_section ) + def __watch_directory( self, sub_elem, elems, tool_path, integrated_elems, load_panel_dict ): + directory = os.path.join( tool_path, sub_elem.attrib.get("dir") ) + + def quick_load( tool_file, async=True ): + try: + tool = self.load_tool( tool_file ) + self.__add_tool( tool, load_panel_dict, elems ) + # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. + key = 'tool_%s' % str( tool.id ) + integrated_elems[ key ] = tool + + if async: + self.load_tool_panel() + + if self.app.config.update_integrated_tool_panel: + # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. + # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel + # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel + # will be False when things like functional tests are the caller. + self.fix_integrated_tool_panel_dict() + self.write_integrated_tool_panel_config_file() + + return tool.id + except Exception: + log.exception("Failed to load potential tool %s." % tool_file) + return None + + for name in os.listdir( directory ): + if name.endswith( ".xml" ): + quick_load( os.path.join( directory, name), async=False ) + self.tool_watcher.watch_directory( directory, quick_load ) + def load_tool( self, config_file, guid=None, repository_id=None, **kwds ): """Load a single tool from the file named by `config_file` and return an instance of `Tool`.""" # Parse XML configuration file and get the root element @@ -721,7 +762,13 @@ inputs.append( conditional_element ) ToolClass = Tool - return ToolClass( config_file, root, self.app, guid=guid, repository_id=repository_id, **kwds ) + tool = ToolClass( config_file, root, self.app, guid=guid, repository_id=repository_id, **kwds ) + tool_id = tool.id + if not tool_id.startswith("__"): + # do not monitor special tools written to tmp directory - no reason + # to monitor such a large directory. + self.tool_watcher.watch_file( config_file, tool.id ) + return tool def package_tool( self, trans, tool_id ): """ diff -r c40b29fb9fbc96c8ca2efb8b3dc4e789e2063d0b -r 650d2307e5b5688ca8137fc4af577308fac54833 lib/galaxy/tools/watcher.py --- /dev/null +++ b/lib/galaxy/tools/watcher.py @@ -0,0 +1,99 @@ +import os.path +try: + from watchdog.events import FileSystemEventHandler + from watchdog.observers import Observer + can_watch = True +except ImportError: + FileSystemEventHandler = object + can_watch = False + +import logging +log = logging.getLogger( __name__ ) + + +def get_watcher(toolbox, config): + watch_tools = getattr(config, "watch_tools", False) + if watch_tools: + return ToolWatcher(toolbox) + else: + return NullWatcher() + + +class ToolWatcher(object): + + def __init__(self, toolbox): + if not can_watch: + raise Exception("Watchdog library unavailble, cannot watch tools.") + self.toolbox = toolbox + self.tool_file_ids = {} + self.tool_dir_callbacks = {} + self.monitored_dirs = {} + self.observer = Observer() + self.event_handler = ToolFileEventHandler(self) + self.start() + + def start(self): + self.observer.start() + + def monitor(self, dir): + self.observer.schedule(self.event_handler, dir, recursive=False) + + def watch_file(self, tool_file, tool_id): + tool_file = os.path.abspath( tool_file ) + self.tool_file_ids[tool_file] = tool_id + tool_dir = os.path.dirname( tool_file ) + if tool_dir not in self.monitored_dirs: + self.monitored_dirs[ tool_dir ] = tool_dir + self.monitor( tool_dir ) + + def watch_directory(self, tool_dir, callback): + tool_dir = os.path.abspath( tool_dir ) + self.tool_dir_callbacks[tool_dir] = callback + if tool_dir not in self.monitored_dirs: + self.monitored_dirs[ tool_dir ] = tool_dir + self.restart( tool_dir ) + + +class ToolFileEventHandler(FileSystemEventHandler): + + def __init__(self, tool_watcher): + self.tool_watcher = tool_watcher + + def on_any_event(self, event): + self._handle(event) + + def _handle(self, event): + # modified events will only have src path, move events will + # have dest_path and src_path but we only care about dest. So + # look at dest if it exists else use src. + path = getattr( event, 'dest_path', None ) or event.src_path + path = os.path.abspath( path ) + tool_id = self.tool_watcher.tool_file_ids.get( path, None ) + if tool_id: + try: + self.tool_watcher.toolbox.reload_tool_by_id(tool_id) + except Exception: + pass + elif path.endswith(".xml"): + directory = os.path.dirname( path ) + dir_callback = self.tool_watcher.tool_dir_callbacks.get( directory, None ) + if dir_callback: + tool_file = event.src_path + tool_id = dir_callback( tool_file ) + if tool_id: + self.tool_watcher.tool_file_ids[ tool_file ] = tool_id + + +class NullWatcher(object): + + def start(self): + pass + + def shutdown(self): + pass + + def watch_file(self, tool_file, tool_id): + pass + + def watch_directory(self, tool_dir, callback): + pass diff -r c40b29fb9fbc96c8ca2efb8b3dc4e789e2063d0b -r 650d2307e5b5688ca8137fc4af577308fac54833 test/unit/tools/test_watcher.py --- /dev/null +++ b/test/unit/tools/test_watcher.py @@ -0,0 +1,52 @@ +from contextlib import contextmanager +from os import path +from shutil import rmtree +import tempfile +import time + +from galaxy.tools import watcher +from galaxy.util import bunch + + +def test_watcher(): + with __test_directory() as t: + tool_path = path.join(t, "test.xml") + toolbox = Toolbox() + open(tool_path, "w").write("a") + tool_watcher = watcher.get_watcher(toolbox, bunch.Bunch( + watch_tools=True + )) + tool_watcher.watch_file(tool_path, "cool_tool") + open(tool_path, "w").write("b") + time.sleep(2) + toolbox.assert_reloaded("cool_tool") + + +class Toolbox(object): + + def __init__(self): + self.reloaded = {} + + def reload_tool_by_id( self, tool_id ): + self.reloaded[ tool_id ] = True + + def assert_reloaded(self, tool_id): + assert self.reloaded.get( tool_id, False ) + + +class CallbackRecorder(object): + + def __init__(self): + self.called = False + + def call(self): + self.called = True + + +@contextmanager +def __test_directory(): + base_path = tempfile.mkdtemp() + try: + yield base_path + finally: + rmtree(base_path) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.