1 new changeset in galaxy-central: http://bitbucket.org/galaxy/galaxy-central/changeset/ad227f6403bb/ changeset: ad227f6403bb user: greg date: 2011-09-02 20:57:33 summary: Enhance Galaxy tool initialization to do the following: 1. Support multiple tool_conf.xml files with tool_conf.xml being the default. Additional files must use the tool shed XML rules. 2. Allow new added tools to be loaded into a selected tool panel section wothout requiring a Galaxy server restart. 3. Add a new tool shed registry, enabling configuring of tool sheds for communication with the local Galaxy instance. Also add the version of the tool to the form heading when loaded into tool_fom.mako, and clean up the code for reloading a tool in the tool panel. affected #: 9 files (2.5 KB) --- a/lib/galaxy/app.py Fri Sep 02 14:37:09 2011 -0400 +++ b/lib/galaxy/app.py Fri Sep 02 14:57:33 2011 -0400 @@ -3,6 +3,7 @@ from galaxy import config, jobs, util, tools, web import galaxy.tools.search import galaxy.tools.data +import galaxy.tools.tool_shed_registry from galaxy.web import security import galaxy.model import galaxy.datatypes.registry @@ -23,6 +24,11 @@ # Set up datatypes registry self.datatypes_registry = galaxy.datatypes.registry.Registry( self.config.root, self.config.datatypes_config ) galaxy.model.set_datatypes_registry( self.datatypes_registry ) + # Set up the tool sheds registry + if os.path.isfile( self.config.tool_sheds_config ): + self.tool_shed_registry = galaxy.tools.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config ) + else: + self.tool_shed_registry = None # Determine the database url if self.config.database_connection: db_url = self.config.database_connection @@ -44,7 +50,7 @@ # Tool data tables self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_table_config_path ) # Initialize the tools - self.toolbox = tools.ToolBox( self.config.tool_config, self.config.tool_path, self ) + self.toolbox = tools.ToolBox( self.config.tool_configs, self.config.tool_path, self ) # Search support for tools self.toolbox_search = galaxy.tools.search.ToolBoxSearch( self.toolbox ) # Load datatype converters --- a/lib/galaxy/config.py Fri Sep 02 14:37:09 2011 -0400 +++ b/lib/galaxy/config.py Fri Sep 02 14:57:33 2011 -0400 @@ -5,7 +5,7 @@ import sys, os, tempfile import logging, logging.config import ConfigParser -from galaxy.util import string_as_bool +from galaxy.util import string_as_bool, listify from galaxy import eggs import pkg_resources @@ -46,11 +46,12 @@ self.enable_api = string_as_bool( kwargs.get( 'enable_api', False ) ) self.enable_openid = string_as_bool( kwargs.get( 'enable_openid', False ) ) self.enable_quotas = string_as_bool( kwargs.get( 'enable_quotas', False ) ) + self.tool_sheds_config = kwargs.get( 'tool_sheds_config_file', 'tool_sheds_conf.xml' ) self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root ) self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "tool-data" ), os.getcwd() ) self.len_file_path = kwargs.get( "len_file_path", resolve_path(os.path.join(self.tool_data_path, 'shared','ucsc','chrom'), self.root) ) self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root ) - self.tool_config = resolve_path( kwargs.get( 'tool_config_file', 'tool_conf.xml' ), self.root ) + self.tool_configs = [ resolve_path( p, self.root ) for p in listify( kwargs.get( 'tool_config_file', 'tool_conf.xml' ) ) ] self.tool_data_table_config_path = resolve_path( kwargs.get( 'tool_data_table_config_path', 'tool_data_table_conf.xml' ), self.root ) self.tool_secret = kwargs.get( "tool_secret", "" ) self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" ) @@ -194,9 +195,11 @@ except Exception, e: raise ConfigurationError( "Unable to create missing directory: %s\n%s" % ( path, e ) ) # Check that required files exist - for path in self.tool_config, self.datatypes_config: + for path in self.tool_configs: if not os.path.isfile(path): raise ConfigurationError("File not found: %s" % path ) + if not os.path.isfile( self.datatypes_config ): + raise ConfigurationError("File not found: %s" % path ) # Check for deprecated options. for key in self.config_dict.keys(): if key in self.deprecated_options: --- a/lib/galaxy/tools/__init__.py Fri Sep 02 14:37:09 2011 -0400 +++ b/lib/galaxy/tools/__init__.py Fri Sep 02 14:57:33 2011 -0400 @@ -28,6 +28,7 @@ from galaxy.datatypes import sniff from cgi import FieldStorage from galaxy.util.hash_util import * +from galaxy.util import listify log = logging.getLogger( __name__ ) @@ -39,23 +40,28 @@ Container for a collection of tools """ - def __init__( self, config_filename, tool_root_dir, app ): + def __init__( self, config_filenames, tool_root_dir, app ): """ Create a toolbox from the config file names by `config_filename`, using `tool_root_directory` as the base directory for finding individual tool config files. """ + # The shed_tool_confs dictionary contains shed_conf_filename : tool_path pairs. + self.shed_tool_confs = {} self.tools_by_id = {} self.workflows_by_id = {} self.tool_panel = odict() + # The following refers to the tool_path config setting for backward compatibility. + # Additional newer (e.g., shed_tool_conf.xml) files include the tool_path attribute + # within the <toolbox> tag. self.tool_root_dir = tool_root_dir self.app = app self.init_dependency_manager() - try: - self.init_tools( config_filename ) - except: - log.exception( "ToolBox error reading %s", config_filename ) - + for config_filename in listify( config_filenames ): + try: + self.init_tools( config_filename ) + except: + log.exception( "ToolBox error reading %s", config_filename ) def init_tools( self, config_filename ): """ Read the configuration file and load each tool. @@ -71,83 +77,99 @@ </section></toolbox> """ - def load_tool( elem, panel_dict ): - try: - path = elem.get( "file" ) - tool = self.load_tool( os.path.join( self.tool_root_dir, path ) ) - if self.app.config.get_bool( 'enable_tool_tags', False ): - tag_names = elem.get( "tags", "" ).split( "," ) - for tag_name in tag_names: - if tag_name == '': - continue - tag = self.sa_session.query( self.app.model.Tag ).filter_by( name=tag_name ).first() - if not tag: - tag = self.app.model.Tag( name=tag_name ) - self.sa_session.add( tag ) - self.sa_session.flush() - tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id ) - self.sa_session.add( tta ) - self.sa_session.flush() - else: - for tagged_tool in tag.tagged_tools: - if tagged_tool.tool_id == tool.id: - break - else: - tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id ) - self.sa_session.add( tta ) - self.sa_session.flush() - self.tools_by_id[ tool.id ] = tool - key = 'tool_' + tool.id - panel_dict[ key ] = tool - log.debug( "Loaded tool: %s %s" % ( tool.id, tool.version ) ) - except: - log.exception( "error reading tool from path: %s" % path ) - def load_workflow( elem, panel_dict ): - try: - # TODO: should id be encoded? - workflow_id = elem.get( 'id' ) - workflow = self.load_workflow( workflow_id ) - self.workflows_by_id[ workflow_id ] = workflow - key = 'workflow_' + workflow_id - panel_dict[ key ] = workflow - log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) - except: - log.exception( "error loading workflow: %s" % workflow_id ) - def load_label( elem, panel_dict ): - label = ToolSectionLabel( elem ) - key = 'label_' + label.id - panel_dict[ key ] = label - def load_section( elem, panel_dict ): - section = ToolSection( elem ) - log.debug( "Loading section: %s" % section.name ) - for section_elem in elem: - if section_elem.tag == 'tool': - load_tool( section_elem, section.elems ) - elif section_elem.tag == 'workflow': - load_workflow( section_elem, section.elems ) - elif section_elem.tag == 'label': - load_label( section_elem, section.elems ) - key = 'section_' + section.id - panel_dict[ key ] = section - if self.app.config.get_bool( 'enable_tool_tags', False ): log.info("removing all tool tag associations (" + str( self.sa_session.query( self.app.model.ToolTagAssociation ).count() ) + ")") self.sa_session.query( self.app.model.ToolTagAssociation ).delete() self.sa_session.flush() - log.info("parsing the tool configuration") + log.info( "parsing the tool configuration %s" % config_filename ) tree = util.parse_xml( config_filename ) root = tree.getroot() + tool_path = root.get( 'tool_path' ) + if tool_path: + # We're parsing a shed_tool_conf file since we have a tool_path attribute. + self.shed_tool_confs[ config_filename ] = tool_path + else: + # Default to backward compatible config setting. + tool_path = self.tool_root_dir for elem in root: if elem.tag == 'tool': - load_tool( elem, self.tool_panel ) + self.load_tool_tag_set( elem, self.tool_panel, tool_path, guid=elem.get( 'guid' ) ) elif elem.tag == 'workflow': - load_workflow( elem, self.tool_panel ) + self.load_workflow_tag_set( elem, self.tool_panel ) elif elem.tag == 'section' : - load_section( elem, self.tool_panel ) + self.load_section_tag_set( elem, self.tool_panel, tool_path ) elif elem.tag == 'label': - load_label( elem, self.tool_panel ) - - def load_tool( self, config_file ): + self.load_label_tag_set( elem, self.tool_panel ) + def load_tool_tag_set( self, elem, panel_dict, tool_path, guid=None ): + try: + path = elem.get( "file" ) + tool = self.load_tool( os.path.join( tool_path, path ), guid=guid ) + if self.app.config.get_bool( 'enable_tool_tags', False ): + tag_names = elem.get( "tags", "" ).split( "," ) + for tag_name in tag_names: + if tag_name == '': + continue + tag = self.sa_session.query( self.app.model.Tag ).filter_by( name=tag_name ).first() + if not tag: + tag = self.app.model.Tag( name=tag_name ) + self.sa_session.add( tag ) + self.sa_session.flush() + tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id ) + self.sa_session.add( tta ) + self.sa_session.flush() + else: + for tagged_tool in tag.tagged_tools: + if tagged_tool.tool_id == tool.id: + break + else: + tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id ) + self.sa_session.add( tta ) + self.sa_session.flush() + if tool.id in self.tools_by_id: + raise Exception( "Tool with id %s already loaded." % tool.id ) + else: + self.tools_by_id[ tool.id ] = tool + key = 'tool_' + tool.id + panel_dict[ key ] = tool + log.debug( "Loaded tool: %s %s" % ( tool.id, tool.version ) ) + except: + log.exception( "error reading tool from path: %s" % path ) + def load_workflow_tag_set( self, elem, panel_dict ): + try: + # TODO: should id be encoded? + workflow_id = elem.get( 'id' ) + workflow = self.load_workflow( workflow_id ) + self.workflows_by_id[ workflow_id ] = workflow + key = 'workflow_' + workflow_id + panel_dict[ key ] = workflow + log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) + except: + log.exception( "error loading workflow: %s" % workflow_id ) + def load_label_tag_set( self, elem, panel_dict ): + label = ToolSectionLabel( elem ) + key = 'label_' + label.id + panel_dict[ key ] = label + def load_section_tag_set( self, elem, panel_dict, tool_path ): + key = 'section_' + elem.get( "id" ) + if key in panel_dict: + # Appending a tool to an existing section in self.tool_panel + elems = panel_dict[ key ].elems + log.debug( "Appending to section: %s" % elem.get( "name" ) ) + else: + # Appending a new section to self.tool_panel + section = ToolSection( elem ) + elems = section.elems + log.debug( "Loading section: %s" % section.name ) + for section_elem in elem: + if section_elem.tag == 'tool': + self.load_tool_tag_set( section_elem, elems, tool_path, guid=section_elem.get( 'guid' ) ) + elif section_elem.tag == 'workflow': + self.load_workflow_tag_set( section_elem, elems ) + elif section_elem.tag == 'label': + self.load_label_tag_set( section_elem, elems ) + if key not in panel_dict: + panel_dict[ key ] = section + def load_tool( self, config_file, guid=None ): """ Load a single tool from the file named by `config_file` and return an instance of `Tool`. @@ -160,38 +182,43 @@ type_elem = root.find( "type" ) module = type_elem.get( 'module', 'galaxy.tools' ) cls = type_elem.get( 'class' ) - mod = __import__( module, globals(), locals(), [cls]) + mod = __import__( module, globals(), locals(), [cls] ) ToolClass = getattr( mod, cls ) elif root.get( 'tool_type', None ) is not None: ToolClass = tool_types.get( root.get( 'tool_type' ) ) else: ToolClass = Tool - return ToolClass( config_file, root, self.app ) - - def reload( self, tool_id ): + return ToolClass( config_file, root, self.app, guid=guid ) + def reload_tool_by_id( self, tool_id ): """ Attempt to reload the tool identified by 'tool_id', if successful replace the old tool. """ if tool_id not in self.tools_by_id: - raise ToolNotFoundException( "No tool with id %s" % tool_id ) - old_tool = self.tools_by_id[ tool_id ] - new_tool = self.load_tool( old_tool.config_file ) - # Replace old_tool with new_tool in self.tool_panel - tool_key = 'tool_' + tool_id - for key, val in self.tool_panel.items(): - if key == tool_key: - self.tool_panel[ key ] = new_tool - break - elif key.startswith( 'section' ): - section = val - for section_key, section_val in section.elems.items(): - if section_key == tool_key: - self.tool_panel[ key ].elems[ section_key ] = new_tool - break - self.tools_by_id[ tool_id ] = new_tool - log.debug( "Reloaded tool %s %s" %( old_tool.id, old_tool.version ) ) - + message = "No tool with id %s" % tool_id + status = 'error' + else: + old_tool = self.tools_by_id[ tool_id ] + new_tool = self.load_tool( old_tool.config_file ) + # Replace old_tool with new_tool in self.tool_panel + tool_key = 'tool_' + tool_id + for key, val in self.tool_panel.items(): + if key == tool_key: + self.tool_panel[ key ] = new_tool + break + elif key.startswith( 'section' ): + section = val + for section_key, section_val in section.elems.items(): + if section_key == tool_key: + self.tool_panel[ key ].elems[ section_key ] = new_tool + break + self.tools_by_id[ tool_id ] = new_tool + message = "Reloaded the tool:<br/>" + message += "<b>name:</b> %s<br/>" % old_tool.name + message += "<b>id:</b> %s<br/>" % old_tool.id + message += "<b>version:</b> %s" % old_tool.version + status = 'done' + return message, status def load_workflow( self, workflow_id ): """ Return an instance of 'Workflow' identified by `id`, @@ -328,7 +355,7 @@ tool_type = 'default' - def __init__( self, config_file, root, app ): + def __init__( self, config_file, root, app, guid=None ): """ Load a tool from the config named by `config_file` """ @@ -337,7 +364,7 @@ self.tool_dir = os.path.dirname( config_file ) self.app = app # Parse XML element containing configuration - self.parse( root ) + self.parse( root, guid=guid ) @property def sa_session( self ): @@ -346,7 +373,7 @@ """ return self.app.model.context - def parse( self, root ): + def parse( self, root, guid=None ): """ Read tool configuration from the element `root` and fill in `self`. """ @@ -356,7 +383,10 @@ raise Exception, "Missing tool 'name'" # Get the UNIQUE id for the tool # TODO: can this be generated automatically? - self.id = root.get( "id" ) + if guid is not None: + self.id = guid + else: + self.id = root.get( "id" ) if not self.id: raise Exception, "Missing tool 'id'" self.version = root.get( "version" ) --- a/templates/admin/reload_tool.mako Fri Sep 02 14:37:09 2011 -0400 +++ b/templates/admin/reload_tool.mako Fri Sep 02 14:57:33 2011 -0400 @@ -8,7 +8,7 @@ <div class="toolForm"><div class="toolFormTitle">Reload Tool</div><div class="toolFormBody"> - <form name="tool_reload" action="${h.url_for( controller='admin', action='tool_reload' )}" method="post" > + <form name="reload_tool" id="reload_tool" action="${h.url_for( controller='admin', action='reload_tool' )}" method="post" ><div class="form-row"><label> Tool to reload: @@ -30,7 +30,7 @@ </select></div><div class="form-row"> - <input type="submit" name="action" value="Reload"/> + <input type="submit" name="reload_tool_button" value="Reload"/></div></form></div> --- a/templates/tool_form.mako Fri Sep 02 14:37:09 2011 -0400 +++ b/templates/tool_form.mako Fri Sep 02 14:57:33 2011 -0400 @@ -260,7 +260,7 @@ %if tool.has_multiple_pages: <div class="toolFormTitle">${tool.name} (step ${tool_state.page+1} of ${tool.npages})</div> %else: - <div class="toolFormTitle">${tool.name}</div> + <div class="toolFormTitle">${tool.name} (version ${tool.version})</div> %endif <div class="toolFormBody"><form id="tool_form" name="tool_form" action="${tool_url}" enctype="${tool.enctype}" target="${tool.target}" method="${tool.method}"> --- a/universe_wsgi.ini.sample Fri Sep 02 14:37:09 2011 -0400 +++ b/universe_wsgi.ini.sample Fri Sep 02 14:57:33 2011 -0400 @@ -123,10 +123,12 @@ # Temporary files are stored in this directory. #new_file_path = database/tmp -# Tool config file, defines what tools are available in Galaxy. +# Tool config files, defines what tools are available in Galaxy. +# Tools can be locally developed or installed from tool sheds. #tool_config_file = tool_conf.xml -# Path to the directory containing the tools defined in the config. +# Default path to the directory containing the tools defined in tool_conf.xml. +# Other tool config files must include the tool_path as an attribute in the <toolbox> tag. #tool_path = tools # Directory where data used by tools is located, see the samples in that Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.