1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/bc5fa254bafc/ changeset: bc5fa254bafc user: greg date: 2012-11-07 22:36:28 summary: Maintain entries for Galaxy's ToolDataTableManager acquired from installed tool shed repositrories that contain a valid file named tool_data_table_conf.xmnl.sample in a separate config file name shed_tool_data_table_conf.xml. This will ensure that manual edits to the original tool_data_table_conf.xml will not be munged when Galaxy's tool shed installation process automatically adds entries into the file. This enhancement alos includes a bug fix where ToolDataTableEntries that should have been persisted to the XML file were not being handled correctly. affected #: 11 files diff -r dbff28bde968a05b787dd53cfca182c53a81eeb7 -r bc5fa254bafc981c70bdffd394e9c44c8dda4ab8 lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -76,8 +76,13 @@ self.genomes = Genomes( self ) # Data providers registry. self.data_provider_registry = DataProviderRegistry() - # Tool data tables - self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_path, self.config.tool_data_table_config_path ) + # Initialize tool data tables using the config defined by self.config.tool_data_table_config_path. + self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( tool_data_path=self.config.tool_data_path, + config_filename=self.config.tool_data_table_config_path ) + # Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables. + self.tool_data_tables.load_from_config_file( config_filename=self.config.shed_tool_data_table_config, + tool_data_path=self.tool_data_tables.tool_data_path, + from_shed_config=True ) # Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file. tool_configs = self.config.tool_configs if self.config.migrated_tools_config not in tool_configs: diff -r dbff28bde968a05b787dd53cfca182c53a81eeb7 -r bc5fa254bafc981c70bdffd394e9c44c8dda4ab8 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -66,6 +66,7 @@ tcf = 'tool_conf.xml' self.tool_configs = [ resolve_path( p, self.root ) for p in listify( tcf ) ] self.tool_data_table_config_path = resolve_path( kwargs.get( 'tool_data_table_config_path', 'tool_data_table_conf.xml' ), self.root ) + self.shed_tool_data_table_config = resolve_path( kwargs.get( 'shed_tool_data_table_config', 'shed_tool_data_table_conf.xml' ), self.root ) self.enable_tool_shed_check = string_as_bool( kwargs.get( 'enable_tool_shed_check', False ) ) try: self.hours_between_check = int( kwargs.get( 'hours_between_check', 12 ) ) diff -r dbff28bde968a05b787dd53cfca182c53a81eeb7 -r bc5fa254bafc981c70bdffd394e9c44c8dda4ab8 lib/galaxy/tool_shed/install_manager.py --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -184,7 +184,8 @@ relative_install_dir=relative_install_dir, repository_files_dir=None, resetting_all_metadata_on_repository=False, - updating_installed_repository=False ) + updating_installed_repository=False, + persist=True ) tool_shed_repository.metadata = metadata_dict self.app.sa_session.add( tool_shed_repository ) self.app.sa_session.flush() diff -r dbff28bde968a05b787dd53cfca182c53a81eeb7 -r bc5fa254bafc981c70bdffd394e9c44c8dda4ab8 lib/galaxy/tool_shed/migrate/common.py --- a/lib/galaxy/tool_shed/migrate/common.py +++ b/lib/galaxy/tool_shed/migrate/common.py @@ -41,8 +41,13 @@ self.datatypes_registry = galaxy.datatypes.registry.Registry() # Load the data types in the Galaxy distribution, which are defined in self.config.datatypes_config. self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config ) - # Tool data tables - self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_path, self.config.tool_data_table_config_path ) + # Initialize tool data tables using the config defined by self.config.tool_data_table_config_path. + self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( tool_data_path=self.config.tool_data_path, + config_filename=self.config.tool_data_table_config_path ) + # Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables. + self.tool_data_tables.load_from_config_file( config_filename=self.config.shed_tool_data_table_config, + tool_data_path=self.tool_data_tables.tool_data_path, + from_shed_config=True ) # Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file. tool_configs = self.config.tool_configs if self.config.migrated_tools_config not in tool_configs: diff -r dbff28bde968a05b787dd53cfca182c53a81eeb7 -r bc5fa254bafc981c70bdffd394e9c44c8dda4ab8 lib/galaxy/tools/data/__init__.py --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -15,17 +15,28 @@ """Manages a collection of tool data tables""" def __init__( self, tool_data_path, config_filename=None ): self.tool_data_path = tool_data_path - self.data_tables = {} - # Store config elements for on-the-fly persistence. - self.data_table_elems = [] + # This stores all defined data table entries from both the tool_data_table_conf.xml file and the shed_tool_data_table_conf.xml file + # at server startup. If tool shed repositories are installed that contain a valid file named tool_data_table_conf.xml.sample, entries + # from that file are inserted into this dict at the time of installation. + self.data_tables = {} + # Store config elements for on-the-fly persistence to the defined shed_tool_data_table_config file name. + self.shed_data_table_elems = [] self.data_table_elem_names = [] if config_filename: - self.load_from_config_file( config_filename, self.tool_data_path ) + self.load_from_config_file( config_filename, self.tool_data_path, from_shed_config=False ) def __getitem__( self, key ): return self.data_tables.__getitem__( key ) def __contains__( self, key ): return self.data_tables.__contains__( key ) - def load_from_config_file( self, config_filename, tool_data_path ): + def load_from_config_file( self, config_filename, tool_data_path, from_shed_config=False ): + """ + This method is called under 3 conditions: + 1) When the ToolDataTableManager is initialized (see __init__ above). + 2) Just after the ToolDataTableManager is initialized and the additional entries defined by shed_tool_data_table_conf.xml + are being loaded into the ToolDataTableManager.data_tables. + 3) When a tool shed repository that includes a tool_data_table_conf.xml.sample file is being installed into a local + Galaxy instance. In this case, we have 2 entry types to handle, files whose root tag is <tables>, for example: + """ tree = util.parse_xml( config_filename ) root = tree.getroot() table_elems = [] @@ -36,13 +47,14 @@ table_elem_name = table_elem.get( 'name', None ) if table_elem_name and table_elem_name not in self.data_table_elem_names: self.data_table_elem_names.append( table_elem_name ) - self.data_table_elems.append( table_elem ) + if from_shed_config: + self.shed_data_table_elems.append( table_elem ) table = tool_data_table_types[ type ]( table_elem, tool_data_path ) if table.name not in self.data_tables: self.data_tables[ table.name ] = table log.debug( "Loaded tool data table '%s'", table.name ) return table_elems - def add_new_entries_from_config_file( self, config_filename, tool_data_path, tool_data_table_config_path, persist=False ): + def add_new_entries_from_config_file( self, config_filename, tool_data_path, shed_tool_data_table_config, persist=False ): """ This method is called when a tool shed repository that includes a tool_data_table_conf.xml.sample file is being installed into a local galaxy instance. We have 2 cases to handle, files whose root tag is <tables>, for example: @@ -65,7 +77,9 @@ # Make a copy of the current list of data_table_elem_names so we can persist later if changes to the config file are necessary. original_data_table_elem_names = [ name for name in self.data_table_elem_names ] if root.tag == 'tables': - table_elems = self.load_from_config_file( config_filename, tool_data_path ) + table_elems = self.load_from_config_file( config_filename=config_filename, + tool_data_path=tool_data_path, + from_shed_config=True ) else: table_elems = [] type = root.get( 'type', 'tabular' ) @@ -74,23 +88,22 @@ table_elem_name = root.get( 'name', None ) if table_elem_name and table_elem_name not in self.data_table_elem_names: self.data_table_elem_names.append( table_elem_name ) - self.data_table_elems.append( root ) + self.shed_data_table_elems.append( root ) table = tool_data_table_types[ type ]( root, tool_data_path ) if table.name not in self.data_tables: self.data_tables[ table.name ] = table log.debug( "Added new tool data table '%s'", table.name ) if persist and self.data_table_elem_names != original_data_table_elem_names: # Persist Galaxy's version of the changed tool_data_table_conf.xml file. - self.to_xml_file( tool_data_table_config_path ) + self.to_xml_file( shed_tool_data_table_config ) return table_elems - def to_xml_file( self, tool_data_table_config_path ): - """Write the current in-memory version of the tool_data-table_conf.xml file to disk.""" - full_path = os.path.abspath( tool_data_table_config_path ) + def to_xml_file( self, shed_tool_data_table_config ): + """Write the current in-memory version of the shed_tool_data_table_conf.xml file to disk.""" + full_path = os.path.abspath( shed_tool_data_table_config ) fd, filename = tempfile.mkstemp() os.write( fd, '<?xml version="1.0"?>\n' ) - os.write( fd, "<!-- Use the file tool_data_table_conf.xml.oldlocstyle if you don't want to update your loc files as changed in revision 4550:535d276c92bc-->\n" ) os.write( fd, '<tables>\n' ) - for elem in self.data_table_elems: + for elem in self.shed_data_table_elems: os.write( fd, '%s' % util.xml_to_string( elem ) ) os.write( fd, '</tables>\n' ) os.close( fd ) diff -r dbff28bde968a05b787dd53cfca182c53a81eeb7 -r bc5fa254bafc981c70bdffd394e9c44c8dda4ab8 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -753,12 +753,15 @@ tool_dependencies_dict[ 'set_environment' ] = [ requirements_dict ] return tool_dependencies_dict def generate_metadata_for_changeset_revision( app, repository, repository_clone_url, shed_config_dict={}, relative_install_dir=None, repository_files_dir=None, - resetting_all_metadata_on_repository=False, updating_installed_repository=False ): + resetting_all_metadata_on_repository=False, updating_installed_repository=False, persist=False ): """ Generate metadata for a repository using it's files on disk. To generate metadata for changeset revisions older than the repository tip, the repository will have been cloned to a temporary location and updated to a specified changeset revision to access that changeset revision's disk files, so the value of repository_files_dir will not always be repository.repo_path (it could be an absolute path to a temporary directory containing a clone). If it is an absolute path, the value of relative_install_dir must contain repository.repo_path. + + The value of persist will be True when the installed repository contains a valid tool_data_table_conf.xml.sample file, in which case the entries + should ultimately be persisted to the file referred to by app.config.shed_tool_data_table_config. """ if updating_installed_repository: # Keep the original tool shed repository metadata if setting metadata on a repository installed into a local Galaxy instance for which @@ -810,9 +813,9 @@ relative_path, filename = os.path.split( sample_file ) if filename == 'tool_data_table_conf.xml.sample': new_table_elems = app.tool_data_tables.add_new_entries_from_config_file( config_filename=sample_file, - tool_data_path=app.config.tool_data_path, - tool_data_table_config_path=app.config.tool_data_table_config_path, - persist=False ) + tool_data_path=original_tool_data_path, + shed_tool_data_table_config=app.config.shed_tool_data_table_config, + persist=persist ) for root, dirs, files in os.walk( files_dir ): if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0: if '.hg' in dirs: @@ -1768,15 +1771,15 @@ return tool, message, sample_files def handle_sample_tool_data_table_conf_file( app, filename, persist=False ): """ - Parse the incoming filename and add new entries to the in-memory app.tool_data_tables dictionary. If persist is True (should only occur) - if call is from the Galaxy side (not the tool shed), the new entries will be appended to Galaxy's tool_data_table_conf.xml file on disk. + Parse the incoming filename and add new entries to the in-memory app.tool_data_tables dictionary. If persist is True (should only occur + if call is from the Galaxy side, not the tool shed), the new entries will be appended to Galaxy's shed_tool_data_table_conf.xml file on disk. """ error = False message = '' try: new_table_elems = app.tool_data_tables.add_new_entries_from_config_file( config_filename=filename, tool_data_path=app.config.tool_data_path, - tool_data_table_config_path=app.config.tool_data_table_config_path, + shed_tool_data_table_config=app.config.shed_tool_data_table_config, persist=persist ) except Exception, e: message = str( e ) @@ -2143,7 +2146,8 @@ relative_install_dir=relative_install_dir, repository_files_dir=None, resetting_all_metadata_on_repository=False, - updating_installed_repository=False ) + updating_installed_repository=False, + persist=False ) repository.metadata = metadata_dict if metadata_dict != original_metadata_dict: update_in_shed_tool_config( trans.app, repository ) @@ -2221,7 +2225,8 @@ relative_install_dir=repo_dir, repository_files_dir=work_dir, resetting_all_metadata_on_repository=True, - updating_installed_repository=False ) + updating_installed_repository=False, + persist=False ) if current_metadata_dict: if not metadata_changeset_revision and not metadata_dict: # We're at the first change set in the change log. diff -r dbff28bde968a05b787dd53cfca182c53a81eeb7 -r bc5fa254bafc981c70bdffd394e9c44c8dda4ab8 lib/galaxy/webapps/community/controllers/common.py --- a/lib/galaxy/webapps/community/controllers/common.py +++ b/lib/galaxy/webapps/community/controllers/common.py @@ -10,9 +10,8 @@ from galaxy.util.shed_util import generate_clone_url_for_repository_in_tool_shed, generate_message_for_invalid_tools, generate_metadata_for_changeset_revision from galaxy.util.shed_util import get_changectx_for_changeset, get_config_from_disk, get_configured_ui, get_file_context_from_ctx, get_named_tmpfile_from_ctx from galaxy.util.shed_util import get_parent_id, get_repository_in_tool_shed, get_repository_metadata_by_changeset_revision -from galaxy.util.shed_util import handle_sample_files_and_load_tool_from_disk, handle_sample_files_and_load_tool_from_tmp_config -from galaxy.util.shed_util import handle_sample_tool_data_table_conf_file, INITIAL_CHANGELOG_HASH, is_downloadable, load_tool_from_config, remove_dir -from galaxy.util.shed_util import reset_tool_data_tables, reversed_upper_bounded_changelog, strip_path +from galaxy.util.shed_util import handle_sample_files_and_load_tool_from_disk, handle_sample_files_and_load_tool_from_tmp_config, INITIAL_CHANGELOG_HASH +from galaxy.util.shed_util import is_downloadable, load_tool_from_config, remove_dir, reset_tool_data_tables, reversed_upper_bounded_changelog, strip_path from galaxy.web.base.controller import * from galaxy.web.base.controllers.admin import * from galaxy.webapps.community import model @@ -640,7 +639,8 @@ relative_install_dir=repo_dir, repository_files_dir=None, resetting_all_metadata_on_repository=False, - updating_installed_repository=False ) + updating_installed_repository=False, + persist=False ) if metadata_dict: downloadable = is_downloadable( metadata_dict ) repository_metadata = None diff -r dbff28bde968a05b787dd53cfca182c53a81eeb7 -r bc5fa254bafc981c70bdffd394e9c44c8dda4ab8 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -447,6 +447,12 @@ @web.expose @web.require_admin def deactivate_or_uninstall_repository( self, trans, **kwd ): + """ + Handle all changes when a tool shed repository is being deactivated or uninstalled. Notice that if the repository contents include + a file named tool_data_table_conf.xml.sample, it's entries are not removed from the defined config.shed_tool_data_table_config. This + is because it becomes a bit complex to determine if other installed repositories include tools that require the same entry. For now + we'll never delete entries from config.shed_tool_data_table_config, but we may choose to do so in the future if it becomes necessary. + """ params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) @@ -753,7 +759,8 @@ relative_install_dir=relative_install_dir, repository_files_dir=None, resetting_all_metadata_on_repository=False, - updating_installed_repository=False ) + updating_installed_repository=False, + persist=True ) tool_shed_repository.metadata = metadata_dict trans.sa_session.add( tool_shed_repository ) trans.sa_session.flush() @@ -1491,7 +1498,8 @@ relative_install_dir=relative_install_dir, repository_files_dir=None, resetting_all_metadata_on_repository=False, - updating_installed_repository=False ) + updating_installed_repository=False, + persist=False ) repository.metadata = metadata_dict if metadata_dict != original_metadata_dict: update_in_shed_tool_config( trans.app, repository ) @@ -1675,7 +1683,8 @@ relative_install_dir=relative_install_dir, repository_files_dir=None, resetting_all_metadata_on_repository=False, - updating_installed_repository=True ) + updating_installed_repository=True, + persist=True ) repository.metadata = metadata_dict # Update the repository changeset_revision in the database. repository.changeset_revision = latest_changeset_revision diff -r dbff28bde968a05b787dd53cfca182c53a81eeb7 -r bc5fa254bafc981c70bdffd394e9c44c8dda4ab8 run.sh --- a/run.sh +++ b/run.sh @@ -13,6 +13,7 @@ reports_wsgi.ini.sample shed_tool_conf.xml.sample tool_conf.xml.sample + shed_tool_data_table_conf.xml.sample tool_data_table_conf.xml.sample tool_sheds_conf.xml.sample openid_conf.xml.sample diff -r dbff28bde968a05b787dd53cfca182c53a81eeb7 -r bc5fa254bafc981c70bdffd394e9c44c8dda4ab8 shed_tool_data_table_conf.xml.sample --- /dev/null +++ b/shed_tool_data_table_conf.xml.sample @@ -0,0 +1,3 @@ +<?xml version="1.0"?> +<tables> +</tables> diff -r dbff28bde968a05b787dd53cfca182c53a81eeb7 -r bc5fa254bafc981c70bdffd394e9c44c8dda4ab8 universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -149,6 +149,16 @@ #enable_tool_shed_check = False #hours_between_check = 12 +# XML config file that contains data table entries for the ToolDataTableManager. This file is manually +# maintained by the Galaxy administrator. +#tool_data_table_config_path = tool_data_table_conf.xml + +# XML config file that contains additional data table entries for the ToolDataTableManager. This file +# is automatically generated based on the current installed tool shed repositories that contain valid +# tool_data_table_conf.xml.sample files. At the time of installation, these entries are automatically +# added to the following file, which is parsed and applied to the ToolDataTableManager at server start up. +#shed_tool_data_table_config = shed_tool_data_table_conf.xml + # Directory where data used by tools is located, see the samples in that # directory and the wiki for help: # http://wiki.g2.bx.psu.edu/Admin/Data%20Integration Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.