1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/ec4584a0e500/ changeset: ec4584a0e500 user: greg date: 2012-05-30 22:00:18 summary: Fixes for handling tool shed repositories that include tool_data_table_conf.xml.sample files. The fixes apply to determining if the tool is valid in the tool shed and also to the Galaxy environment when the tool shed repository is installed. affected #: 6 files diff -r 16a93eb6eaf6b2f9b73bbe10d0e4fab1d1142b9f -r ec4584a0e500e755aa5436691318d442831d8787 lib/galaxy/tool_shed/install_manager.py --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -152,7 +152,12 @@ if repository_tools_tups: sample_files = metadata_dict.get( 'sample_files', [] ) # Handle missing data table entries for tool parameters that are dynamically generated select lists. - repository_tools_tups = handle_missing_data_table_entry( self.app, self.tool_path, sample_files, repository_tools_tups ) + repository_tools_tups = handle_missing_data_table_entry( self.app, + tool_shed_repository, + changeset_revision, + self.tool_path, + repository_tools_tups, + work_dir ) # Handle missing index files for tool parameters that are dynamically generated select lists. repository_tools_tups, sample_files_copied = handle_missing_index_file( self.app, self.tool_path, sample_files, repository_tools_tups ) # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. diff -r 16a93eb6eaf6b2f9b73bbe10d0e4fab1d1142b9f -r ec4584a0e500e755aa5436691318d442831d8787 lib/galaxy/tools/data/__init__.py --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -6,7 +6,7 @@ to modify the tool configurations. """ -import logging, sys, os.path +import logging, sys, os, os.path, tempfile, shutil from galaxy import util log = logging.getLogger( __name__ ) @@ -15,6 +15,9 @@ """Manages a collection of tool data tables""" def __init__( self, config_filename=None ): self.data_tables = {} + # Store config elements for on-the-fly persistence. + self.data_table_elems = [] + self.data_table_elem_names = [] if config_filename: self.load_from_config_file( config_filename ) def __getitem__( self, key ): @@ -29,14 +32,19 @@ type = table_elem.get( 'type', 'tabular' ) assert type in tool_data_table_types, "Unknown data table type '%s'" % type table_elems.append( table_elem ) + table_elem_name = table_elem.get( 'name', None ) + if table_elem_name and table_elem_name not in self.data_table_elem_names: + self.data_table_elem_names.append( table_elem_name ) + self.data_table_elems.append( table_elem ) table = tool_data_table_types[ type ]( table_elem ) if table.name not in self.data_tables: self.data_tables[ table.name ] = table log.debug( "Loaded tool data table '%s'", table.name ) return table_elems - def add_new_entries_from_config_file( self, config_filename ): + def add_new_entries_from_config_file( self, config_filename, tool_data_table_config_path ): """ - We have 2 cases to handle, files whose root tag is <tables>, for example: + This method is called when a tool shed repository that includes a tool_data_table_conf.xml.sample file is being + installed into a local galaxy instance. We have 2 cases to handle, files whose root tag is <tables>, for example: <tables><!-- Location of Tmap files --><table name="tmap_indexes" comment_char="#"> @@ -53,6 +61,8 @@ """ tree = util.parse_xml( config_filename ) root = tree.getroot() + # Make a copy of the current list of data_table_elem_names so we can persist later if changes to the config file are necessary. + original_data_table_elem_names = [ name for name in self.data_table_elem_names ] if root.tag == 'tables': table_elems = self.load_from_config_file( config_filename ) else: @@ -60,11 +70,31 @@ type = root.get( 'type', 'tabular' ) assert type in tool_data_table_types, "Unknown data table type '%s'" % type table_elems.append( root ) + table_elem_name = root.get( 'name', None ) + if table_elem_name and table_elem_name not in self.data_table_elem_names: + self.data_table_elem_names.append( table_elem_name ) + self.data_table_elems.append( root ) table = tool_data_table_types[ type ]( root ) if table.name not in self.data_tables: self.data_tables[ table.name ] = table - log.debug( "Loaded tool data table '%s", table.name ) + log.debug( "Added new tool data table '%s'", table.name ) + if self.data_table_elem_names != original_data_table_elem_names: + # Persist Galaxy's version of the changed tool_data_table_conf.xml file. + self.to_xml_file( tool_data_table_config_path ) return table_elems + def to_xml_file( self, tool_data_table_config_path ): + """Write the current in-memory version of the tool_data-table_conf.xml file to disk.""" + full_path = os.path.abspath( tool_data_table_config_path ) + fd, filename = tempfile.mkstemp() + os.write( fd, '<?xml version="1.0"?>\n' ) + os.write( fd, "<!-- Use the file tool_data_table_conf.xml.oldlocstyle if you don't want to update your loc files as changed in revision 4550:535d276c92bc-->\n" ) + os.write( fd, '<tables>\n' ) + for elem in self.data_table_elems: + os.write( fd, '%s' % util.xml_to_string( elem ) ) + os.write( fd, '</tables>\n' ) + os.close( fd ) + shutil.move( filename, full_path ) + os.chmod( full_path, 0644 ) class ToolDataTable( object ): def __init__( self, config_element ): diff -r 16a93eb6eaf6b2f9b73bbe10d0e4fab1d1142b9f -r ec4584a0e500e755aa5436691318d442831d8787 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -1003,10 +1003,10 @@ return shed_url # The tool shed from which the repository was originally installed must no longer be configured in tool_sheds_conf.xml. return None -def handle_missing_data_table_entry( app, tool_path, sample_files, repository_tools_tups ): +def handle_missing_data_table_entry( app, repository, changeset_revision, tool_path, repository_tools_tups, dir ): """ Inspect each tool to see if any have input parameters that are dynamically generated select lists that require entries in the - tool_data_table_conf.xml file. This method is not called only from Galaxy (not the tool shed) when a repository is being installed. + tool_data_table_conf.xml file. This method is called only from Galaxy (not the tool shed) when a repository is being installed. """ missing_data_table_entry = False for index, repository_tools_tup in enumerate( repository_tools_tups ): @@ -1015,13 +1015,11 @@ missing_data_table_entry = True break if missing_data_table_entry: - sample_file = None # The repository must contain a tool_data_table_conf.xml.sample file that includes all required entries for all tools in the repository. - for sample_file in sample_files: - sample_file_name = strip_path( sample_file ) - if sample_file_name == 'tool_data_table_conf.xml.sample': - break - error, correction_msg = handle_sample_tool_data_table_conf_file( app, sample_file ) + sample_tool_data_table_conf = get_config_from_repository( app, 'tool_data_table_conf.xml.sample', repository, changeset_revision, dir ) + # Add entries to the ToolDataTableManager's in-memory data_tables dictionary as well as the list of data_table_elems and the list of + # data_table_elem_names. + error, correction_msg = handle_sample_tool_data_table_conf_file( app, sample_tool_data_table_conf ) if error: # TODO: Do more here than logging an exception. log.debug( correction_msg ) @@ -1059,47 +1057,16 @@ return repository_tools_tups, sample_files_copied def handle_sample_tool_data_table_conf_file( app, filename ): """ - Parse the incoming filename and add new entries to the in-memory app.tool_data_tables dictionary as well as appending them to the - shed's tool_data_table_conf.xml file on disk. + Parse the incoming filename and add new entries to the in-memory app.tool_data_tables dictionary as well as appending them to + Galaxy's tool_data_table_conf.xml file on disk. """ - # TODO: Load an in-memory version of the tool_data_table_conf.xml file, and write it to disk - # from the in-memory version only when changes are made. error = False message = '' try: - new_table_elems = app.tool_data_tables.add_new_entries_from_config_file( filename ) + new_table_elems = app.tool_data_tables.add_new_entries_from_config_file( filename, app.config.tool_data_table_config_path ) except Exception, e: message = str( e ) error = True - """ - # TODO: eliminate this - the shed should not need to write this to disk... - if not error: - # Add an entry to the end of the tool_data_table_conf.xml file. - tdt_config = "%s/tool_data_table_conf.xml" % app.config.root - if os.path.exists( tdt_config ): - # Make a backup of the file since we're going to be changing it. - today = date.today() - backup_date = today.strftime( "%Y_%m_%d" ) - tdt_config_copy = '%s/tool_data_table_conf.xml_%s_backup' % ( app.config.root, backup_date ) - shutil.copy( os.path.abspath( tdt_config ), os.path.abspath( tdt_config_copy ) ) - # Write each line of the tool_data_table_conf.xml file, except the last line to a temp file. - fh = tempfile.NamedTemporaryFile( 'wb' ) - tmp_filename = fh.name - fh.close() - new_tdt_config = open( tmp_filename, 'wb' ) - for i, line in enumerate( open( tdt_config, 'rb' ) ): - if line.find( '</tables>' ) >= 0: - for new_table_elem in new_table_elems: - new_tdt_config.write( ' %s\n' % util.xml_to_string( new_table_elem ).rstrip( '\n' ) ) - new_tdt_config.write( '</tables>\n' ) - else: - new_tdt_config.write( line ) - new_tdt_config.close() - shutil.move( tmp_filename, os.path.abspath( tdt_config ) ) - else: - message = "The required file named tool_data_table_conf.xml does not exist in the Galaxy install directory." - error = True - """ return error, message def handle_tool_dependencies( app, tool_shed_repository, installed_changeset_revision, tool_dependencies_config ): """ @@ -1208,11 +1175,20 @@ tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) repository_tools_tups = get_repository_tools_tups( trans.app, metadata_dict ) if repository_tools_tups: + # Handle missing data table entries for tool parameters that are dynamically generated select lists. work_dir = make_tmp_directory() + repository_tools_tups = handle_missing_data_table_entry( trans.app, + tool_shed_repository, + changeset_revision, + tool_path, + repository_tools_tups, + work_dir ) + try: + shutil.rmtree( work_dir ) + except: + pass + # Handle missing index files for tool parameters that are dynamically generated select lists. sample_files = metadata_dict.get( 'sample_files', [] ) - # Handle missing data table entries for tool parameters that are dynamically generated select lists. - repository_tools_tups = handle_missing_data_table_entry( trans.app, tool_path, sample_files, repository_tools_tups ) - # Handle missing index files for tool parameters that are dynamically generated select lists. repository_tools_tups, sample_files_copied = handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups ) # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. copy_sample_files( trans.app, sample_files, sample_files_copied=sample_files_copied ) @@ -1406,7 +1382,7 @@ trans.app.toolbox.write_integrated_tool_panel_config_file() def reset_tool_data_tables( app ): # Reset the tool_data_tables to an empty dictionary. - app.tool_data_tables = galaxy.tools.data.ToolDataTableManager() + app.tool_data_tables.data_tables = {} def strip_path( fpath ): file_path, file_name = os.path.split( fpath ) return file_name diff -r 16a93eb6eaf6b2f9b73bbe10d0e4fab1d1142b9f -r ec4584a0e500e755aa5436691318d442831d8787 lib/galaxy/web/controllers/admin_toolshed.py --- a/lib/galaxy/web/controllers/admin_toolshed.py +++ b/lib/galaxy/web/controllers/admin_toolshed.py @@ -613,6 +613,7 @@ else: ctx_rev = repository.ctx_rev clone_repository( repository_clone_url, os.path.abspath( relative_install_dir ), ctx_rev ) + tool_section = None if repository.includes_tools: # Get the location in the tool panel in which each tool was originally loaded. metadata = repository.metadata @@ -668,19 +669,19 @@ tool_section = trans.app.toolbox.tool_panel[ section_key ] else: tool_section = None - tool_shed_repository, metadata_dict, error_message = load_repository_contents( trans, - repository_name=repository.name, - description=repository.description, - owner=repository.owner, - changeset_revision=repository.installed_changeset_revision, - ctx_rev=ctx_rev, - tool_path=tool_path, - repository_clone_url=repository_clone_url, - relative_install_dir=relative_install_dir, - tool_shed=repository.tool_shed, - tool_section=tool_section, - shed_tool_conf=shed_tool_conf, - install_tool_dependencies=install_tool_dependencies ) + tool_shed_repository, metadata_dict, error_message = load_repository_contents( trans, + repository_name=repository.name, + description=repository.description, + owner=repository.owner, + changeset_revision=repository.installed_changeset_revision, + ctx_rev=ctx_rev, + tool_path=tool_path, + repository_clone_url=repository_clone_url, + relative_install_dir=relative_install_dir, + tool_shed=repository.tool_shed, + tool_section=tool_section, + shed_tool_conf=shed_tool_conf, + install_tool_dependencies=install_tool_dependencies ) if error_message: message += error_message status = 'error' diff -r 16a93eb6eaf6b2f9b73bbe10d0e4fab1d1142b9f -r ec4584a0e500e755aa5436691318d442831d8787 lib/galaxy/webapps/community/app.py --- a/lib/galaxy/webapps/community/app.py +++ b/lib/galaxy/webapps/community/app.py @@ -36,8 +36,8 @@ self.security = security.SecurityHelper( id_secret=self.config.id_secret ) # Tag handler self.tag_handler = CommunityTagHandler() - # Tool data tables - self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_table_config_path ) + # Tool data tables - never pass a config file here because the tool shed should always have an empty dictionary! + self.tool_data_tables = galaxy.tools.data.ToolDataTableManager() # The tool shed has no toolbox, but this attribute is still required. self.toolbox = None # Load security policy diff -r 16a93eb6eaf6b2f9b73bbe10d0e4fab1d1142b9f -r ec4584a0e500e755aa5436691318d442831d8787 lib/galaxy/webapps/community/controllers/common.py --- a/lib/galaxy/webapps/community/controllers/common.py +++ b/lib/galaxy/webapps/community/controllers/common.py @@ -201,23 +201,18 @@ if options: if options.tool_data_table or options.missing_tool_data_table_name: # Make sure the repository contains a tool_data_table_conf.xml.sample file. - sample_found = False - for sample_file in sample_files: - sample_file_name = strip_path( sample_file ) - if sample_file_name == 'tool_data_table_conf.xml.sample': - sample_found = True - error, correction_msg = handle_sample_tool_data_table_conf_file( trans.app, sample_file ) - if error: - can_set_metadata = False - invalid_files.append( ( sample_file_name, correction_msg ) ) - else: - options.missing_tool_data_table_name = None - break - if not sample_found: + sample_tool_data_table_conf = get_config( 'tool_data_table_conf.xml.sample', repo, repo_dir, ctx, dir ) + if sample_tool_data_table_conf: + error, correction_msg = handle_sample_tool_data_table_conf_file( trans.app, sample_tool_data_table_conf ) + if error: + can_set_metadata = False + invalid_files.append( ( 'tool_data_table_conf.xml.sample', correction_msg ) ) + else: + options.missing_tool_data_table_name = None + else: can_set_metadata = False - correction_msg = "This file requires an entry in the tool_data_table_conf.xml file. " - correction_msg += "Upload a file named tool_data_table_conf.xml.sample to the repository " - correction_msg += "that includes the required entry to correct this error.<br/>" + correction_msg = "This file requires an entry in the tool_data_table_conf.xml file. Upload a file named tool_data_table_conf.xml.sample " + correction_msg += "to the repository that includes the required entry to correct this error.<br/>" invalid_files.append( ( xml_file_in_ctx, correction_msg ) ) if options.index_file or options.missing_index_file: # Make sure the repository contains the required xxx.loc.sample file. @@ -245,8 +240,8 @@ correction_msg = "This file refers to a file named <b>%s</b>. " % str( index_file ) correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % str( index_file_name ) invalid_files.append( ( xml_file_in_ctx, correction_msg ) ) - # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. - reset_tool_data_tables( trans.app ) + # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. + reset_tool_data_tables( trans.app ) return sample_files_copied, can_set_metadata, invalid_files def clean_repository_metadata( trans, id, changeset_revisions ): # Delete all repository_metadata reecords associated with the repository that have a changeset_revision that is not in changeset_revisions. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.