1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/commits/99bc7d90156f/
Changeset: 99bc7d90156f
Branch: next-stable
User: martenson
Date: 2014-09-30 17:22:10+00:00
Summary: force qualified URL for tool sharing within the toolform
Affected #: 1 file
diff -r e59a2af27182123bd6ed96ab8a9bbfd4a83d11f9 -r 99bc7d90156facd280def33e6ad7e303ebe8d838 templates/webapps/galaxy/tool_form.mako
--- a/templates/webapps/galaxy/tool_form.mako
+++ b/templates/webapps/galaxy/tool_form.mako
@@ -337,7 +337,7 @@
</div>
%endif
<div class="icon-btn-group">
- <a href="#" data-link="${h.url_for( controller='root', action='index', tool_id=tool.id )}"
+ <a href="#" data-link="${h.url_for( controller='root', action='index', tool_id=tool.id, qualified=True )}"
class="icon-btn tool-share-link" title="Share this tool" data-toggle="tooltip" data-placement="bottom"><span class="fa fa-share"></span></a></div></span>
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/commits/55e6516c89be/
Changeset: 55e6516c89be
User: martenson
Date: 2014-09-30 17:22:10+00:00
Summary: force qualified URL for tool sharing within the toolform
Affected #: 1 file
diff -r 650d2307e5b5688ca8137fc4af577308fac54833 -r 55e6516c89bedc14605c31d48ed6eb055959a3f4 templates/webapps/galaxy/tool_form.mako
--- a/templates/webapps/galaxy/tool_form.mako
+++ b/templates/webapps/galaxy/tool_form.mako
@@ -337,7 +337,7 @@
</div>
%endif
<div class="icon-btn-group">
- <a href="#" data-link="${h.url_for( controller='root', action='index', tool_id=tool.id )}"
+ <a href="#" data-link="${h.url_for( controller='root', action='index', tool_id=tool.id, qualified=True )}"
class="icon-btn tool-share-link" title="Share this tool" data-toggle="tooltip" data-placement="bottom"><span class="fa fa-share"></span></a></div></span>
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/commits/650d2307e5b5/
Changeset: 650d2307e5b5
User: jmchilton
Date: 2014-09-30 16:52:47+00:00
Summary: Merged in jmchilton/galaxy-central-fork-1 (pull request #509)
Options to load and watch directories of tools, auto reload on update of any tool.
Affected #: 5 files
diff -r c40b29fb9fbc96c8ca2efb8b3dc4e789e2063d0b -r 650d2307e5b5688ca8137fc4af577308fac54833 config/galaxy.ini.sample
--- a/config/galaxy.ini.sample
+++ b/config/galaxy.ini.sample
@@ -172,6 +172,12 @@
# install from in the admin interface (.sample used if default does not exist).
#tool_sheds_config_file = config/tool_sheds_conf.xml
+# If the following option is set to True - Galaxy will monitor individual tools
+# and tool directories specified in tool_conf.xml for changes and reload these
+# tools. Watchdog must be installed and available to Galaxy to use this option.
+# See https://pypi.python.org/pypi/watchdog.
+#watch_tools = False
+
# Enable automatic polling of relative tool sheds to see if any updates
# are available for installed repositories. Ideally only one Galaxy
# server process should be able to check for repository updates. The
diff -r c40b29fb9fbc96c8ca2efb8b3dc4e789e2063d0b -r 650d2307e5b5688ca8137fc4af577308fac54833 lib/galaxy/config.py
--- a/lib/galaxy/config.py
+++ b/lib/galaxy/config.py
@@ -229,6 +229,7 @@
self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None )
self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False )
self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' )
+ self.watch_tools = kwargs.get( 'watch_tools', False )
# Location for tool dependencies.
if 'tool_dependency_dir' in kwargs:
self.tool_dependency_dir = resolve_path( kwargs.get( "tool_dependency_dir" ), self.root )
diff -r c40b29fb9fbc96c8ca2efb8b3dc4e789e2063d0b -r 650d2307e5b5688ca8137fc4af577308fac54833 lib/galaxy/tools/__init__.py
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -39,6 +39,7 @@
from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper
from galaxy import exceptions
from galaxy.jobs import ParallelismInfo
+from galaxy.tools import watcher
from galaxy.tools.actions import DefaultToolAction
from galaxy.tools.actions.data_source import DataSourceToolAction
from galaxy.tools.actions.data_manager import DataManagerToolAction
@@ -143,6 +144,7 @@
# (e.g., shed_tool_conf.xml) files include the tool_path attribute within the <toolbox> tag.
self.tool_root_dir = tool_root_dir
self.app = app
+ self.tool_watcher = watcher.get_watcher( self, app.config )
self.filter_factory = FilterFactory( self )
self.init_dependency_manager()
config_filenames = listify( config_filenames )
@@ -232,6 +234,8 @@
self.load_section_tag_set( elem, tool_path, load_panel_dict, index=index )
elif elem.tag == 'label':
self.load_label_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index )
+ elif elem.tag == 'tool_dir':
+ self.__watch_directory( elem, self.tool_panel, self.integrated_tool_panel, tool_path, load_panel_dict )
if parsing_shed_tool_conf:
shed_tool_conf_dict = dict( config_filename=config_filename,
tool_path=tool_path,
@@ -614,15 +618,7 @@
tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id )
self.sa_session.add( tta )
self.sa_session.flush()
- # Allow for the same tool to be loaded into multiple places in the tool panel. We have to handle
- # the case where the tool is contained in a repository installed from the tool shed, and the Galaxy
- # administrator has retrieved updates to the installed repository. In this case, the tool may have
- # been updated, but the version was not changed, so the tool should always be reloaded here. We used
- # to only load the tool if it was not found in self.tools_by_id, but performing that check did
- # not enable this scenario.
- self.tools_by_id[ tool.id ] = tool
- if load_panel_dict:
- self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) )
+ self.__add_tool( tool, load_panel_dict, panel_dict )
# Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file.
if key in integrated_panel_dict or index is None:
integrated_panel_dict[ key ] = tool
@@ -631,6 +627,17 @@
except:
log.exception( "Error reading tool from path: %s" % path )
+ def __add_tool( self, tool, load_panel_dict, panel_dict ):
+ # Allow for the same tool to be loaded into multiple places in the tool panel. We have to handle
+ # the case where the tool is contained in a repository installed from the tool shed, and the Galaxy
+ # administrator has retrieved updates to the installed repository. In this case, the tool may have
+ # been updated, but the version was not changed, so the tool should always be reloaded here. We used
+ # to only load the tool if it was not found in self.tools_by_id, but performing that check did
+ # not enable this scenario.
+ self.tools_by_id[ tool.id ] = tool
+ if load_panel_dict:
+ self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) )
+
def load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ):
try:
# TODO: should id be encoded?
@@ -679,6 +686,8 @@
self.load_workflow_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index )
elif sub_elem.tag == 'label':
self.load_label_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index )
+ elif sub_elem.tag == 'tool_dir':
+ self.__watch_directory( sub_elem, elems, tool_path, integrated_elems, load_panel_dict )
if load_panel_dict:
self.tool_panel[ key ] = section
# Always load sections into the integrated_tool_panel.
@@ -687,6 +696,38 @@
else:
self.integrated_tool_panel.insert( index, key, integrated_section )
+ def __watch_directory( self, sub_elem, elems, tool_path, integrated_elems, load_panel_dict ):
+ directory = os.path.join( tool_path, sub_elem.attrib.get("dir") )
+
+ def quick_load( tool_file, async=True ):
+ try:
+ tool = self.load_tool( tool_file )
+ self.__add_tool( tool, load_panel_dict, elems )
+ # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file.
+ key = 'tool_%s' % str( tool.id )
+ integrated_elems[ key ] = tool
+
+ if async:
+ self.load_tool_panel()
+
+ if self.app.config.update_integrated_tool_panel:
+ # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file.
+ # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel
+ # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel
+ # will be False when things like functional tests are the caller.
+ self.fix_integrated_tool_panel_dict()
+ self.write_integrated_tool_panel_config_file()
+
+ return tool.id
+ except Exception:
+ log.exception("Failed to load potential tool %s." % tool_file)
+ return None
+
+ for name in os.listdir( directory ):
+ if name.endswith( ".xml" ):
+ quick_load( os.path.join( directory, name), async=False )
+ self.tool_watcher.watch_directory( directory, quick_load )
+
def load_tool( self, config_file, guid=None, repository_id=None, **kwds ):
"""Load a single tool from the file named by `config_file` and return an instance of `Tool`."""
# Parse XML configuration file and get the root element
@@ -721,7 +762,13 @@
inputs.append( conditional_element )
ToolClass = Tool
- return ToolClass( config_file, root, self.app, guid=guid, repository_id=repository_id, **kwds )
+ tool = ToolClass( config_file, root, self.app, guid=guid, repository_id=repository_id, **kwds )
+ tool_id = tool.id
+ if not tool_id.startswith("__"):
+ # do not monitor special tools written to tmp directory - no reason
+ # to monitor such a large directory.
+ self.tool_watcher.watch_file( config_file, tool.id )
+ return tool
def package_tool( self, trans, tool_id ):
"""
diff -r c40b29fb9fbc96c8ca2efb8b3dc4e789e2063d0b -r 650d2307e5b5688ca8137fc4af577308fac54833 lib/galaxy/tools/watcher.py
--- /dev/null
+++ b/lib/galaxy/tools/watcher.py
@@ -0,0 +1,99 @@
+import os.path
+try:
+ from watchdog.events import FileSystemEventHandler
+ from watchdog.observers import Observer
+ can_watch = True
+except ImportError:
+ FileSystemEventHandler = object
+ can_watch = False
+
+import logging
+log = logging.getLogger( __name__ )
+
+
+def get_watcher(toolbox, config):
+ watch_tools = getattr(config, "watch_tools", False)
+ if watch_tools:
+ return ToolWatcher(toolbox)
+ else:
+ return NullWatcher()
+
+
+class ToolWatcher(object):
+
+ def __init__(self, toolbox):
+ if not can_watch:
+ raise Exception("Watchdog library unavailble, cannot watch tools.")
+ self.toolbox = toolbox
+ self.tool_file_ids = {}
+ self.tool_dir_callbacks = {}
+ self.monitored_dirs = {}
+ self.observer = Observer()
+ self.event_handler = ToolFileEventHandler(self)
+ self.start()
+
+ def start(self):
+ self.observer.start()
+
+ def monitor(self, dir):
+ self.observer.schedule(self.event_handler, dir, recursive=False)
+
+ def watch_file(self, tool_file, tool_id):
+ tool_file = os.path.abspath( tool_file )
+ self.tool_file_ids[tool_file] = tool_id
+ tool_dir = os.path.dirname( tool_file )
+ if tool_dir not in self.monitored_dirs:
+ self.monitored_dirs[ tool_dir ] = tool_dir
+ self.monitor( tool_dir )
+
+ def watch_directory(self, tool_dir, callback):
+ tool_dir = os.path.abspath( tool_dir )
+ self.tool_dir_callbacks[tool_dir] = callback
+ if tool_dir not in self.monitored_dirs:
+ self.monitored_dirs[ tool_dir ] = tool_dir
+ self.restart( tool_dir )
+
+
+class ToolFileEventHandler(FileSystemEventHandler):
+
+ def __init__(self, tool_watcher):
+ self.tool_watcher = tool_watcher
+
+ def on_any_event(self, event):
+ self._handle(event)
+
+ def _handle(self, event):
+ # modified events will only have src path, move events will
+ # have dest_path and src_path but we only care about dest. So
+ # look at dest if it exists else use src.
+ path = getattr( event, 'dest_path', None ) or event.src_path
+ path = os.path.abspath( path )
+ tool_id = self.tool_watcher.tool_file_ids.get( path, None )
+ if tool_id:
+ try:
+ self.tool_watcher.toolbox.reload_tool_by_id(tool_id)
+ except Exception:
+ pass
+ elif path.endswith(".xml"):
+ directory = os.path.dirname( path )
+ dir_callback = self.tool_watcher.tool_dir_callbacks.get( directory, None )
+ if dir_callback:
+ tool_file = event.src_path
+ tool_id = dir_callback( tool_file )
+ if tool_id:
+ self.tool_watcher.tool_file_ids[ tool_file ] = tool_id
+
+
+class NullWatcher(object):
+
+ def start(self):
+ pass
+
+ def shutdown(self):
+ pass
+
+ def watch_file(self, tool_file, tool_id):
+ pass
+
+ def watch_directory(self, tool_dir, callback):
+ pass
diff -r c40b29fb9fbc96c8ca2efb8b3dc4e789e2063d0b -r 650d2307e5b5688ca8137fc4af577308fac54833 test/unit/tools/test_watcher.py
--- /dev/null
+++ b/test/unit/tools/test_watcher.py
@@ -0,0 +1,52 @@
+from contextlib import contextmanager
+from os import path
+from shutil import rmtree
+import tempfile
+import time
+
+from galaxy.tools import watcher
+from galaxy.util import bunch
+
+
+def test_watcher():
+ with __test_directory() as t:
+ tool_path = path.join(t, "test.xml")
+ toolbox = Toolbox()
+ open(tool_path, "w").write("a")
+ tool_watcher = watcher.get_watcher(toolbox, bunch.Bunch(
+ watch_tools=True
+ ))
+ tool_watcher.watch_file(tool_path, "cool_tool")
+ open(tool_path, "w").write("b")
+ time.sleep(2)
+ toolbox.assert_reloaded("cool_tool")
+
+
+class Toolbox(object):
+
+ def __init__(self):
+ self.reloaded = {}
+
+ def reload_tool_by_id( self, tool_id ):
+ self.reloaded[ tool_id ] = True
+
+ def assert_reloaded(self, tool_id):
+ assert self.reloaded.get( tool_id, False )
+
+
+class CallbackRecorder(object):
+
+ def __init__(self):
+ self.called = False
+
+ def call(self):
+ self.called = True
+
+
+@contextmanager
+def __test_directory():
+ base_path = tempfile.mkdtemp()
+ try:
+ yield base_path
+ finally:
+ rmtree(base_path)
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
2 new commits in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/commits/9d2ccd30d905/
Changeset: 9d2ccd30d905
User: jmchilton
Date: 2014-09-28 21:25:12+00:00
Summary: Options to load and watch directories of tools, auto reload on update of any tool.
Allow tool panel to contain a <tool_dir dir="foo" /> element. Tools in such a directory will be loaded at startup time.
Additionally, if watch_tools is set to True in config/galaxy.ini and watchdog (http://pythonhosted.org/watchdog/) is available to Galaxy - all tools will be dynamically reloaded as they are modified and new tools that are added to the tool_dir directories will be dynamically populated (no need to restart Galaxy).
Affected #: 5 files
diff -r 0ef7a9e6973794876c3aee484512fb6c23de517d -r 9d2ccd30d905bf03ca11da02260468f85bad0509 config/galaxy.ini.sample
--- a/config/galaxy.ini.sample
+++ b/config/galaxy.ini.sample
@@ -172,6 +172,12 @@
# install from in the admin interface (.sample used if default does not exist).
#tool_sheds_config_file = config/tool_sheds_conf.xml
+# If the following option is set to True - Galaxy will monitor individual tools
+# and tool directories specified in tool_conf.xml for changes and reload these
+# tools. Watchdog must be installed and available to Galaxy to use this option.
+# See https://pypi.python.org/pypi/watchdog.
+#watch_tools = False
+
# Enable automatic polling of relative tool sheds to see if any updates
# are available for installed repositories. Ideally only one Galaxy
# server process should be able to check for repository updates. The
diff -r 0ef7a9e6973794876c3aee484512fb6c23de517d -r 9d2ccd30d905bf03ca11da02260468f85bad0509 lib/galaxy/config.py
--- a/lib/galaxy/config.py
+++ b/lib/galaxy/config.py
@@ -229,6 +229,7 @@
self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None )
self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False )
self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' )
+ self.watch_tools = kwargs.get( 'watch_tools', False )
# Location for tool dependencies.
if 'tool_dependency_dir' in kwargs:
self.tool_dependency_dir = resolve_path( kwargs.get( "tool_dependency_dir" ), self.root )
diff -r 0ef7a9e6973794876c3aee484512fb6c23de517d -r 9d2ccd30d905bf03ca11da02260468f85bad0509 lib/galaxy/tools/__init__.py
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -39,6 +39,7 @@
from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper
from galaxy import exceptions
from galaxy.jobs import ParallelismInfo
+from galaxy.tools import watcher
from galaxy.tools.actions import DefaultToolAction
from galaxy.tools.actions.data_source import DataSourceToolAction
from galaxy.tools.actions.data_manager import DataManagerToolAction
@@ -143,6 +144,7 @@
# (e.g., shed_tool_conf.xml) files include the tool_path attribute within the <toolbox> tag.
self.tool_root_dir = tool_root_dir
self.app = app
+ self.tool_watcher = watcher.get_watcher( self, app.config )
self.filter_factory = FilterFactory( self )
self.init_dependency_manager()
config_filenames = listify( config_filenames )
@@ -232,6 +234,8 @@
self.load_section_tag_set( elem, tool_path, load_panel_dict, index=index )
elif elem.tag == 'label':
self.load_label_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index )
+ elif elem.tag == 'tool_dir':
+ self.__watch_directory( elem, self.tool_panel, self.integrated_tool_panel, tool_path, load_panel_dict )
if parsing_shed_tool_conf:
shed_tool_conf_dict = dict( config_filename=config_filename,
tool_path=tool_path,
@@ -614,15 +618,7 @@
tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id )
self.sa_session.add( tta )
self.sa_session.flush()
- # Allow for the same tool to be loaded into multiple places in the tool panel. We have to handle
- # the case where the tool is contained in a repository installed from the tool shed, and the Galaxy
- # administrator has retrieved updates to the installed repository. In this case, the tool may have
- # been updated, but the version was not changed, so the tool should always be reloaded here. We used
- # to only load the tool if it was not found in self.tools_by_id, but performing that check did
- # not enable this scenario.
- self.tools_by_id[ tool.id ] = tool
- if load_panel_dict:
- self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) )
+ self.__add_tool( tool, load_panel_dict, panel_dict )
# Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file.
if key in integrated_panel_dict or index is None:
integrated_panel_dict[ key ] = tool
@@ -631,6 +627,17 @@
except:
log.exception( "Error reading tool from path: %s" % path )
+ def __add_tool( self, tool, load_panel_dict, panel_dict ):
+ # Allow for the same tool to be loaded into multiple places in the tool panel. We have to handle
+ # the case where the tool is contained in a repository installed from the tool shed, and the Galaxy
+ # administrator has retrieved updates to the installed repository. In this case, the tool may have
+ # been updated, but the version was not changed, so the tool should always be reloaded here. We used
+ # to only load the tool if it was not found in self.tools_by_id, but performing that check did
+ # not enable this scenario.
+ self.tools_by_id[ tool.id ] = tool
+ if load_panel_dict:
+ self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) )
+
def load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ):
try:
# TODO: should id be encoded?
@@ -679,6 +686,8 @@
self.load_workflow_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index )
elif sub_elem.tag == 'label':
self.load_label_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index )
+ elif sub_elem.tag == 'tool_dir':
+ self.__watch_directory( sub_elem, elems, tool_path, integrated_elems, load_panel_dict )
if load_panel_dict:
self.tool_panel[ key ] = section
# Always load sections into the integrated_tool_panel.
@@ -687,6 +696,38 @@
else:
self.integrated_tool_panel.insert( index, key, integrated_section )
+ def __watch_directory( self, sub_elem, elems, tool_path, integrated_elems, load_panel_dict ):
+ directory = os.path.join( tool_path, sub_elem.attrib.get("dir") )
+
+ def quick_load( tool_file, async=True ):
+ try:
+ tool = self.load_tool( tool_file )
+ self.__add_tool( tool, load_panel_dict, elems )
+ # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file.
+ key = 'tool_%s' % str( tool.id )
+ integrated_elems[ key ] = tool
+
+ if async:
+ self.load_tool_panel()
+
+ if self.app.config.update_integrated_tool_panel:
+ # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file.
+ # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel
+ # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel
+ # will be False when things like functional tests are the caller.
+ self.fix_integrated_tool_panel_dict()
+ self.write_integrated_tool_panel_config_file()
+
+ return tool.id
+ except Exception:
+ log.exception("Failed to load potential tool %s." % tool_file)
+ return None
+
+ for name in os.listdir( directory ):
+ if name.endswith( ".xml" ):
+ quick_load( os.path.join( directory, name), async=False )
+ self.tool_watcher.watch_directory( directory, quick_load )
+
def load_tool( self, config_file, guid=None, repository_id=None, **kwds ):
"""Load a single tool from the file named by `config_file` and return an instance of `Tool`."""
# Parse XML configuration file and get the root element
@@ -721,7 +762,13 @@
inputs.append( conditional_element )
ToolClass = Tool
- return ToolClass( config_file, root, self.app, guid=guid, repository_id=repository_id, **kwds )
+ tool = ToolClass( config_file, root, self.app, guid=guid, repository_id=repository_id, **kwds )
+ tool_id = tool.id
+ if not tool_id.startswith("__"):
+ # do not monitor special tools written to tmp directory - no reason
+ # to monitor such a large directory.
+ self.tool_watcher.watch_file( config_file, tool.id )
+ return tool
def package_tool( self, trans, tool_id ):
"""
diff -r 0ef7a9e6973794876c3aee484512fb6c23de517d -r 9d2ccd30d905bf03ca11da02260468f85bad0509 lib/galaxy/tools/watcher.py
--- /dev/null
+++ b/lib/galaxy/tools/watcher.py
@@ -0,0 +1,99 @@
+import os.path
+try:
+ from watchdog.events import FileSystemEventHandler
+ from watchdog.observers import Observer
+ can_watch = True
+except ImportError:
+ FileSystemEventHandler = object
+ can_watch = False
+
+import logging
+log = logging.getLogger( __name__ )
+
+
+def get_watcher(toolbox, config):
+ watch_tools = getattr(config, "watch_tools", False)
+ if watch_tools:
+ return ToolWatcher(toolbox)
+ else:
+ return NullWatcher()
+
+
+class ToolWatcher(object):
+
+ def __init__(self, toolbox):
+ if not can_watch:
+ raise Exception("Watchdog library unavailble, cannot watch tools.")
+ self.toolbox = toolbox
+ self.tool_file_ids = {}
+ self.tool_dir_callbacks = {}
+ self.monitored_dirs = {}
+ self.observer = Observer()
+ self.event_handler = ToolFileEventHandler(self)
+ self.start()
+
+ def start(self):
+ self.observer.start()
+
+ def monitor(self, dir):
+ self.observer.schedule(self.event_handler, dir, recursive=False)
+
+ def watch_file(self, tool_file, tool_id):
+ tool_file = os.path.abspath( tool_file )
+ self.tool_file_ids[tool_file] = tool_id
+ tool_dir = os.path.dirname( tool_file )
+ if tool_dir not in self.monitored_dirs:
+ self.monitored_dirs[ tool_dir ] = tool_dir
+ self.monitor( tool_dir )
+
+ def watch_directory(self, tool_dir, callback):
+ tool_dir = os.path.abspath( tool_dir )
+ self.tool_dir_callbacks[tool_dir] = callback
+ if tool_dir not in self.monitored_dirs:
+ self.monitored_dirs[ tool_dir ] = tool_dir
+ self.restart( tool_dir )
+
+
+class ToolFileEventHandler(FileSystemEventHandler):
+
+ def __init__(self, tool_watcher):
+ self.tool_watcher = tool_watcher
+
+ def on_any_event(self, event):
+ self._handle(event)
+
+ def _handle(self, event):
+ # modified events will only have src path, move events will
+ # have dest_path and src_path but we only care about dest. So
+ # look at dest if it exists else use src.
+ path = getattr( event, 'dest_path', None ) or event.src_path
+ path = os.path.abspath( path )
+ tool_id = self.tool_watcher.tool_file_ids.get( path, None )
+ if tool_id:
+ try:
+ self.tool_watcher.toolbox.reload_tool_by_id(tool_id)
+ except Exception:
+ pass
+ elif path.endswith(".xml"):
+ directory = os.path.dirname( path )
+ dir_callback = self.tool_watcher.tool_dir_callbacks.get( directory, None )
+ if dir_callback:
+ tool_file = event.src_path
+ tool_id = dir_callback( tool_file )
+ if tool_id:
+ self.tool_watcher.tool_file_ids[ tool_file ] = tool_id
+
+
+class NullWatcher(object):
+
+ def start(self):
+ pass
+
+ def shutdown(self):
+ pass
+
+ def watch_file(self, tool_file, tool_id):
+ pass
+
+ def watch_directory(self, tool_dir, callback):
+ pass
diff -r 0ef7a9e6973794876c3aee484512fb6c23de517d -r 9d2ccd30d905bf03ca11da02260468f85bad0509 test/unit/tools/test_watcher.py
--- /dev/null
+++ b/test/unit/tools/test_watcher.py
@@ -0,0 +1,52 @@
+from contextlib import contextmanager
+from os import path
+from shutil import rmtree
+import tempfile
+import time
+
+from galaxy.tools import watcher
+from galaxy.util import bunch
+
+
+def test_watcher():
+ with __test_directory() as t:
+ tool_path = path.join(t, "test.xml")
+ toolbox = Toolbox()
+ open(tool_path, "w").write("a")
+ tool_watcher = watcher.get_watcher(toolbox, bunch.Bunch(
+ watch_tools=True
+ ))
+ tool_watcher.watch_file(tool_path, "cool_tool")
+ open(tool_path, "w").write("b")
+ time.sleep(2)
+ toolbox.assert_reloaded("cool_tool")
+
+
+class Toolbox(object):
+
+ def __init__(self):
+ self.reloaded = {}
+
+ def reload_tool_by_id( self, tool_id ):
+ self.reloaded[ tool_id ] = True
+
+ def assert_reloaded(self, tool_id):
+ assert self.reloaded.get( tool_id, False )
+
+
+class CallbackRecorder(object):
+
+ def __init__(self):
+ self.called = False
+
+ def call(self):
+ self.called = True
+
+
+@contextmanager
+def __test_directory():
+ base_path = tempfile.mkdtemp()
+ try:
+ yield base_path
+ finally:
+ rmtree(base_path)
https://bitbucket.org/galaxy/galaxy-central/commits/650d2307e5b5/
Changeset: 650d2307e5b5
User: jmchilton
Date: 2014-09-30 16:52:47+00:00
Summary: Merged in jmchilton/galaxy-central-fork-1 (pull request #509)
Options to load and watch directories of tools, auto reload on update of any tool.
Affected #: 5 files
diff -r c40b29fb9fbc96c8ca2efb8b3dc4e789e2063d0b -r 650d2307e5b5688ca8137fc4af577308fac54833 config/galaxy.ini.sample
--- a/config/galaxy.ini.sample
+++ b/config/galaxy.ini.sample
@@ -172,6 +172,12 @@
# install from in the admin interface (.sample used if default does not exist).
#tool_sheds_config_file = config/tool_sheds_conf.xml
+# If the following option is set to True - Galaxy will monitor individual tools
+# and tool directories specified in tool_conf.xml for changes and reload these
+# tools. Watchdog must be installed and available to Galaxy to use this option.
+# See https://pypi.python.org/pypi/watchdog.
+#watch_tools = False
+
# Enable automatic polling of relative tool sheds to see if any updates
# are available for installed repositories. Ideally only one Galaxy
# server process should be able to check for repository updates. The
diff -r c40b29fb9fbc96c8ca2efb8b3dc4e789e2063d0b -r 650d2307e5b5688ca8137fc4af577308fac54833 lib/galaxy/config.py
--- a/lib/galaxy/config.py
+++ b/lib/galaxy/config.py
@@ -229,6 +229,7 @@
self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None )
self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False )
self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' )
+ self.watch_tools = kwargs.get( 'watch_tools', False )
# Location for tool dependencies.
if 'tool_dependency_dir' in kwargs:
self.tool_dependency_dir = resolve_path( kwargs.get( "tool_dependency_dir" ), self.root )
diff -r c40b29fb9fbc96c8ca2efb8b3dc4e789e2063d0b -r 650d2307e5b5688ca8137fc4af577308fac54833 lib/galaxy/tools/__init__.py
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -39,6 +39,7 @@
from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper
from galaxy import exceptions
from galaxy.jobs import ParallelismInfo
+from galaxy.tools import watcher
from galaxy.tools.actions import DefaultToolAction
from galaxy.tools.actions.data_source import DataSourceToolAction
from galaxy.tools.actions.data_manager import DataManagerToolAction
@@ -143,6 +144,7 @@
# (e.g., shed_tool_conf.xml) files include the tool_path attribute within the <toolbox> tag.
self.tool_root_dir = tool_root_dir
self.app = app
+ self.tool_watcher = watcher.get_watcher( self, app.config )
self.filter_factory = FilterFactory( self )
self.init_dependency_manager()
config_filenames = listify( config_filenames )
@@ -232,6 +234,8 @@
self.load_section_tag_set( elem, tool_path, load_panel_dict, index=index )
elif elem.tag == 'label':
self.load_label_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index )
+ elif elem.tag == 'tool_dir':
+ self.__watch_directory( elem, self.tool_panel, self.integrated_tool_panel, tool_path, load_panel_dict )
if parsing_shed_tool_conf:
shed_tool_conf_dict = dict( config_filename=config_filename,
tool_path=tool_path,
@@ -614,15 +618,7 @@
tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id )
self.sa_session.add( tta )
self.sa_session.flush()
- # Allow for the same tool to be loaded into multiple places in the tool panel. We have to handle
- # the case where the tool is contained in a repository installed from the tool shed, and the Galaxy
- # administrator has retrieved updates to the installed repository. In this case, the tool may have
- # been updated, but the version was not changed, so the tool should always be reloaded here. We used
- # to only load the tool if it was not found in self.tools_by_id, but performing that check did
- # not enable this scenario.
- self.tools_by_id[ tool.id ] = tool
- if load_panel_dict:
- self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) )
+ self.__add_tool( tool, load_panel_dict, panel_dict )
# Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file.
if key in integrated_panel_dict or index is None:
integrated_panel_dict[ key ] = tool
@@ -631,6 +627,17 @@
except:
log.exception( "Error reading tool from path: %s" % path )
+ def __add_tool( self, tool, load_panel_dict, panel_dict ):
+ # Allow for the same tool to be loaded into multiple places in the tool panel. We have to handle
+ # the case where the tool is contained in a repository installed from the tool shed, and the Galaxy
+ # administrator has retrieved updates to the installed repository. In this case, the tool may have
+ # been updated, but the version was not changed, so the tool should always be reloaded here. We used
+ # to only load the tool if it was not found in self.tools_by_id, but performing that check did
+ # not enable this scenario.
+ self.tools_by_id[ tool.id ] = tool
+ if load_panel_dict:
+ self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) )
+
def load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ):
try:
# TODO: should id be encoded?
@@ -679,6 +686,8 @@
self.load_workflow_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index )
elif sub_elem.tag == 'label':
self.load_label_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index )
+ elif sub_elem.tag == 'tool_dir':
+ self.__watch_directory( sub_elem, elems, tool_path, integrated_elems, load_panel_dict )
if load_panel_dict:
self.tool_panel[ key ] = section
# Always load sections into the integrated_tool_panel.
@@ -687,6 +696,38 @@
else:
self.integrated_tool_panel.insert( index, key, integrated_section )
+ def __watch_directory( self, sub_elem, elems, tool_path, integrated_elems, load_panel_dict ):
+ directory = os.path.join( tool_path, sub_elem.attrib.get("dir") )
+
+ def quick_load( tool_file, async=True ):
+ try:
+ tool = self.load_tool( tool_file )
+ self.__add_tool( tool, load_panel_dict, elems )
+ # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file.
+ key = 'tool_%s' % str( tool.id )
+ integrated_elems[ key ] = tool
+
+ if async:
+ self.load_tool_panel()
+
+ if self.app.config.update_integrated_tool_panel:
+ # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file.
+ # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel
+ # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel
+ # will be False when things like functional tests are the caller.
+ self.fix_integrated_tool_panel_dict()
+ self.write_integrated_tool_panel_config_file()
+
+ return tool.id
+ except Exception:
+ log.exception("Failed to load potential tool %s." % tool_file)
+ return None
+
+ for name in os.listdir( directory ):
+ if name.endswith( ".xml" ):
+ quick_load( os.path.join( directory, name), async=False )
+ self.tool_watcher.watch_directory( directory, quick_load )
+
def load_tool( self, config_file, guid=None, repository_id=None, **kwds ):
"""Load a single tool from the file named by `config_file` and return an instance of `Tool`."""
# Parse XML configuration file and get the root element
@@ -721,7 +762,13 @@
inputs.append( conditional_element )
ToolClass = Tool
- return ToolClass( config_file, root, self.app, guid=guid, repository_id=repository_id, **kwds )
+ tool = ToolClass( config_file, root, self.app, guid=guid, repository_id=repository_id, **kwds )
+ tool_id = tool.id
+ if not tool_id.startswith("__"):
+ # do not monitor special tools written to tmp directory - no reason
+ # to monitor such a large directory.
+ self.tool_watcher.watch_file( config_file, tool.id )
+ return tool
def package_tool( self, trans, tool_id ):
"""
diff -r c40b29fb9fbc96c8ca2efb8b3dc4e789e2063d0b -r 650d2307e5b5688ca8137fc4af577308fac54833 lib/galaxy/tools/watcher.py
--- /dev/null
+++ b/lib/galaxy/tools/watcher.py
@@ -0,0 +1,99 @@
+import os.path
+try:
+ from watchdog.events import FileSystemEventHandler
+ from watchdog.observers import Observer
+ can_watch = True
+except ImportError:
+ FileSystemEventHandler = object
+ can_watch = False
+
+import logging
+log = logging.getLogger( __name__ )
+
+
+def get_watcher(toolbox, config):
+ watch_tools = getattr(config, "watch_tools", False)
+ if watch_tools:
+ return ToolWatcher(toolbox)
+ else:
+ return NullWatcher()
+
+
+class ToolWatcher(object):
+
+ def __init__(self, toolbox):
+ if not can_watch:
+ raise Exception("Watchdog library unavailble, cannot watch tools.")
+ self.toolbox = toolbox
+ self.tool_file_ids = {}
+ self.tool_dir_callbacks = {}
+ self.monitored_dirs = {}
+ self.observer = Observer()
+ self.event_handler = ToolFileEventHandler(self)
+ self.start()
+
+ def start(self):
+ self.observer.start()
+
+ def monitor(self, dir):
+ self.observer.schedule(self.event_handler, dir, recursive=False)
+
+ def watch_file(self, tool_file, tool_id):
+ tool_file = os.path.abspath( tool_file )
+ self.tool_file_ids[tool_file] = tool_id
+ tool_dir = os.path.dirname( tool_file )
+ if tool_dir not in self.monitored_dirs:
+ self.monitored_dirs[ tool_dir ] = tool_dir
+ self.monitor( tool_dir )
+
+ def watch_directory(self, tool_dir, callback):
+ tool_dir = os.path.abspath( tool_dir )
+ self.tool_dir_callbacks[tool_dir] = callback
+ if tool_dir not in self.monitored_dirs:
+ self.monitored_dirs[ tool_dir ] = tool_dir
+ self.restart( tool_dir )
+
+
+class ToolFileEventHandler(FileSystemEventHandler):
+
+ def __init__(self, tool_watcher):
+ self.tool_watcher = tool_watcher
+
+ def on_any_event(self, event):
+ self._handle(event)
+
+ def _handle(self, event):
+ # modified events will only have src path, move events will
+ # have dest_path and src_path but we only care about dest. So
+ # look at dest if it exists else use src.
+ path = getattr( event, 'dest_path', None ) or event.src_path
+ path = os.path.abspath( path )
+ tool_id = self.tool_watcher.tool_file_ids.get( path, None )
+ if tool_id:
+ try:
+ self.tool_watcher.toolbox.reload_tool_by_id(tool_id)
+ except Exception:
+ pass
+ elif path.endswith(".xml"):
+ directory = os.path.dirname( path )
+ dir_callback = self.tool_watcher.tool_dir_callbacks.get( directory, None )
+ if dir_callback:
+ tool_file = event.src_path
+ tool_id = dir_callback( tool_file )
+ if tool_id:
+ self.tool_watcher.tool_file_ids[ tool_file ] = tool_id
+
+
+class NullWatcher(object):
+
+ def start(self):
+ pass
+
+ def shutdown(self):
+ pass
+
+ def watch_file(self, tool_file, tool_id):
+ pass
+
+ def watch_directory(self, tool_dir, callback):
+ pass
diff -r c40b29fb9fbc96c8ca2efb8b3dc4e789e2063d0b -r 650d2307e5b5688ca8137fc4af577308fac54833 test/unit/tools/test_watcher.py
--- /dev/null
+++ b/test/unit/tools/test_watcher.py
@@ -0,0 +1,52 @@
+from contextlib import contextmanager
+from os import path
+from shutil import rmtree
+import tempfile
+import time
+
+from galaxy.tools import watcher
+from galaxy.util import bunch
+
+
+def test_watcher():
+ with __test_directory() as t:
+ tool_path = path.join(t, "test.xml")
+ toolbox = Toolbox()
+ open(tool_path, "w").write("a")
+ tool_watcher = watcher.get_watcher(toolbox, bunch.Bunch(
+ watch_tools=True
+ ))
+ tool_watcher.watch_file(tool_path, "cool_tool")
+ open(tool_path, "w").write("b")
+ time.sleep(2)
+ toolbox.assert_reloaded("cool_tool")
+
+
+class Toolbox(object):
+
+ def __init__(self):
+ self.reloaded = {}
+
+ def reload_tool_by_id( self, tool_id ):
+ self.reloaded[ tool_id ] = True
+
+ def assert_reloaded(self, tool_id):
+ assert self.reloaded.get( tool_id, False )
+
+
+class CallbackRecorder(object):
+
+ def __init__(self):
+ self.called = False
+
+ def call(self):
+ self.called = True
+
+
+@contextmanager
+def __test_directory():
+ base_path = tempfile.mkdtemp()
+ try:
+ yield base_path
+ finally:
+ rmtree(base_path)
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.