1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/98ecd305e3d3/ Changeset: 98ecd305e3d3 Branch: release_15.03 User: dan Date: 2015-02-18 19:39:07+00:00 Summary: Fixes for handling Data Managers that use undeclared data tables. Affected #: 3 files diff -r b3a0812f9129f8e1e8bdad3a273c3410edba4e7a -r 98ecd305e3d3f02af78c9f61297f8299966a5187 lib/galaxy/tools/data/__init__.py --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -154,7 +154,7 @@ with open( full_path, 'wb' ) as out: out.write( '<?xml version="1.0"?>\n<tables>\n' ) for elem in out_elems: - out.write( util.xml_to_string( elem ) ) + out.write( util.xml_to_string( elem, pretty=True ) ) out.write( '</tables>\n' ) os.chmod( full_path, 0644 ) @@ -452,19 +452,53 @@ """ Returns table entry associated with a col/val pair. """ + rval = self.get_entries( query_attr, query_val, return_attr, default=default, limit=1 ) + if rval: + return rval[0] + return default + + def get_entries( self, query_attr, query_val, return_attr, default=None, limit=None ): + """ + Returns table entry associated with a col/val pair. + """ query_col = self.columns.get( query_attr, None ) if query_col is None: return default - return_col = self.columns.get( return_attr, None ) - if return_col is None: - return default - rval = default + if return_attr is not None: + return_col = self.columns.get( return_attr, None ) + if return_col is None: + return default + rval = [] # Look for table entry. for fields in self.get_fields(): if fields[ query_col ] == query_val: - rval = fields[ return_col ] + if return_attr is None: + field_dict = {} + for i, col_name in enumerate( self.get_column_name_list() ): + field_dict[ col_name or i ] = fields[i] + rval.append( field_dict ) + else: + rval.append( fields[ return_col ] ) + if limit is not None and len( rval ) == limit: + break + return rval or default + + def get_filename_for_source( self, source, default=None ): + if source: + #if dict, assume is compatible info dict, otherwise call method + if isinstance( source, dict ): + source_repo_info = source + else: + source_repo_info = source.get_tool_shed_repository_info_dict() + else: + source_repo_info = None + filename = default + for name, value in self.filenames.iteritems(): + repo_info = value.get( 'tool_shed_repository', None ) + if ( not source_repo_info and not repo_info ) or ( source_repo_info and repo_info and source_repo_info == repo_info ): + filename = name break - return rval + return filename def _add_entry( self, entry, allow_duplicates=True, persist=False, persist_on_error=False, entry_source=None, **kwd ): #accepts dict or list of columns @@ -493,19 +527,7 @@ filename = None if persist and ( not is_error or persist_on_error ): - if entry_source: - #if dict, assume is compatible info dict, otherwise call method - if isinstance( entry_source, dict ): - source_repo_info = entry_source - else: - source_repo_info = entry_source.get_tool_shed_repository_info_dict() - else: - source_repo_info = None - for name, value in self.filenames.iteritems(): - repo_info = value.get( 'tool_shed_repository', None ) - if ( not source_repo_info and not repo_info ) or ( source_repo_info and repo_info and source_repo_info == repo_info ): - filename = name - break + filename = self.get_filename_for_source( entry_source ) if filename is None: #should we default to using any filename here instead? log.error( "Unable to determine filename for persisting data table '%s' values: '%s'.", self.name, fields ) diff -r b3a0812f9129f8e1e8bdad3a273c3410edba4e7a -r 98ecd305e3d3f02af78c9f61297f8299966a5187 lib/galaxy/tools/data_manager/manager.py --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -117,6 +117,7 @@ self.move_by_data_table_column = {} self.value_translation_by_data_table_column = {} self.tool_shed_repository_info_dict = None + self.undeclared_tables = False if elem is not None: self.load_from_element( elem, tool_path or self.data_managers.tool_path ) @@ -175,6 +176,7 @@ tool_shed_repository_id=tool_shed_repository_id ) self.name = elem.get( 'name', self.tool.name ) self.description = elem.get( 'description', self.tool.description ) + self.undeclared_tables = util.asbool( elem.get( 'undeclared_tables', self.undeclared_tables ) ) for data_table_elem in elem.findall( 'data_table' ): data_table_name = data_table_elem.get( "name" ) @@ -299,11 +301,28 @@ moved = self.process_move( data_table_name, name, output_ref_values[ name ].extra_files_path, **data_table_value ) data_table_value[ name ] = self.process_value_translation( data_table_name, name, **data_table_value ) data_table.add_entry( data_table_value, persist=True, entry_source=self ) - - for data_table_name, data_table_values in data_tables_dict.iteritems(): - #tool returned extra data table entries, but data table was not declared in data manager - #do not add these values, but do provide messages - log.warning( 'The data manager "%s" returned an undeclared data table "%s" with new entries "%s". These entries will not be created. Please confirm that an entry for "%s" exists in your "%s" file.' % ( self.id, data_table_name, data_table_values, data_table_name, self.data_managers.filename ) ) + if self.undeclared_tables and data_tables_dict: + # We handle the data move, by just moving all the data out of the extra files path + # moving a directory and the target already exists, we move the contents instead + log.debug( 'Attempting to add entries for undeclared tables: %s.', ', '.join( data_tables_dict.keys() ) ) + for ref_file in out_data.values(): + util.move_merge( ref_file.extra_files_path, self.data_managers.app.config.galaxy_data_manager_data_path ) + path_column_names = [ 'path' ] + for data_table_name, data_table_values in data_tables_dict.iteritems(): + data_table = self.data_managers.app.tool_data_tables.get( data_table_name, None ) + if not isinstance( data_table_values, list ): + data_table_values = [ data_table_values ] + for data_table_row in data_table_values: + data_table_value = dict( **data_table_row ) #keep original values here + for name, value in data_table_row.iteritems(): + if name in path_column_names: + data_table_value[ name ] = os.path.abspath( os.path.join( self.data_managers.app.config.galaxy_data_manager_data_path, value ) ) + data_table.add_entry( data_table_value, persist=True, entry_source=self ) + else: + for data_table_name, data_table_values in data_tables_dict.iteritems(): + # tool returned extra data table entries, but data table was not declared in data manager + # do not add these values, but do provide messages + log.warning( 'The data manager "%s" returned an undeclared data table "%s" with new entries "%s". These entries will not be created. Please confirm that an entry for "%s" exists in your "%s" file.' % ( self.id, data_table_name, data_table_values, data_table_name, self.data_managers.filename ) ) def process_move( self, data_table_name, column_name, source_base_path, relative_symlinks=False, **kwd ): if data_table_name in self.move_by_data_table_column and column_name in self.move_by_data_table_column[ data_table_name ]: diff -r b3a0812f9129f8e1e8bdad3a273c3410edba4e7a -r 98ecd305e3d3f02af78c9f61297f8299966a5187 lib/tool_shed/tools/data_table_manager.py --- a/lib/tool_shed/tools/data_table_manager.py +++ b/lib/tool_shed/tools/data_table_manager.py @@ -113,16 +113,20 @@ error = True return error, message + def get_target_install_dir( self, tool_shed_repository ): + tool_path, relative_target_dir = tool_shed_repository.get_tool_relative_path( self.app ) + # This is where index files will reside on a per repo/installed version basis. + target_dir = os.path.join( self.app.config.shed_tool_data_path, relative_target_dir ) + if not os.path.exists( target_dir ): + os.makedirs( target_dir ) + return target_dir, tool_path, relative_target_dir + def install_tool_data_tables( self, tool_shed_repository, tool_index_sample_files ): TOOL_DATA_TABLE_FILE_NAME = 'tool_data_table_conf.xml' TOOL_DATA_TABLE_FILE_SAMPLE_NAME = '%s.sample' % ( TOOL_DATA_TABLE_FILE_NAME ) SAMPLE_SUFFIX = '.sample' SAMPLE_SUFFIX_OFFSET = -len( SAMPLE_SUFFIX ) - tool_path, relative_target_dir = tool_shed_repository.get_tool_relative_path( self.app ) - # This is where index files will reside on a per repo/installed version basis. - target_dir = os.path.join( self.app.config.shed_tool_data_path, relative_target_dir ) - if not os.path.exists( target_dir ): - os.makedirs( target_dir ) + target_dir, tool_path, relative_target_dir = self.get_target_install_dir( tool_shed_repository ) for sample_file in tool_index_sample_files: path, filename = os.path.split ( sample_file ) target_filename = filename Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.