From fec6e0cf72e90b3516fed5b1f3969482811e11ba Mon Sep 17 00:00:00 2001 From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com> Date: Thu, 24 Oct 2024 16:03:42 +0800 Subject: [PATCH] [fix](statistics)Skip shadow index while analyzing a table. (#42201) Skip shadow index while analyzing a table. --- .../org/apache/doris/catalog/OlapTable.java | 17 +++- .../apache/doris/catalog/OlapTableTest.java | 77 +++++++++++++++++++ 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 2d97bdc6fbcc5b..999e0c43995f00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -83,6 +83,7 @@ import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -954,7 +955,7 @@ public List getSchemaByIndexId(Long indexId, boolean full) { if (full) { return indexIdToMeta.get(indexId).getSchema(); } else { - return indexIdToMeta.get(indexId).getSchema().stream().filter(column -> column.isVisible()) + return indexIdToMeta.get(indexId).getSchema().stream().filter(Column::isVisible) .collect(Collectors.toList()); } } @@ -962,8 +963,8 @@ public List getSchemaByIndexId(Long indexId, boolean full) { @Override public Set getSchemaAllIndexes(boolean full) { Set columns = Sets.newHashSet(); - for (Long indexId : indexIdToMeta.keySet()) { - columns.addAll(getSchemaByIndexId(indexId, full)); + for (MaterializedIndex index : getVisibleIndex()) { + columns.addAll(getSchemaByIndexId(index.getId(), full)); } return columns; } @@ -3405,4 +3406,14 @@ public boolean autoAnalyzeEnabled() { return properties.get(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY) .equalsIgnoreCase(PropertyAnalyzer.ENABLE_AUTO_ANALYZE_POLICY); } + + @VisibleForTesting + protected void addIndexIdToMetaForUnitTest(long id, MaterializedIndexMeta meta) { + indexIdToMeta.put(id, meta); + } + + @VisibleForTesting + protected void addIndexNameToIdForUnitTest(String name, long id) { + indexNameToId.put(name, id); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java index fd394353cd21d5..950371de3034bd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java @@ -22,6 +22,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.io.FastByteArrayOutputStream; import org.apache.doris.common.util.UnitTestUtil; +import org.apache.doris.thrift.TStorageType; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -35,6 +36,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Set; public class OlapTableTest { @@ -168,4 +170,79 @@ public void testGetPartitionRowCount() { olapTable.getRowCountForPartitionIndex(11, 10, true); } + + @Test + public void testGetSchemaAllIndexes() { + OlapTable table = new OlapTable(); + List schema1 = Lists.newArrayList(); + Column col1 = new Column("col1", PrimitiveType.INT); + Column col2 = new Column("col2", PrimitiveType.INT); + Column col3 = new Column("col3", PrimitiveType.INT); + Column col4 = new Column("col4", PrimitiveType.INT); + schema1.add(col1); + schema1.add(col2); + MaterializedIndexMeta meta1 = new MaterializedIndexMeta(1L, schema1, 1, 1, (short) 1, + TStorageType.COLUMN, KeysType.DUP_KEYS, null); + table.addIndexIdToMetaForUnitTest(1, meta1); + table.addIndexNameToIdForUnitTest("index1", 1L); + + List schema2 = Lists.newArrayList(); + schema2.add(col3); + schema2.add(col4); + MaterializedIndexMeta meta2 = new MaterializedIndexMeta(2L, schema2, 1, 1, (short) 1, + TStorageType.COLUMN, KeysType.DUP_KEYS, null); + table.addIndexIdToMetaForUnitTest(1, meta1); + table.addIndexIdToMetaForUnitTest(2, meta2); + table.addIndexNameToIdForUnitTest("index2", 2L); + + MaterializedIndex index1 = new MaterializedIndex(1, MaterializedIndex.IndexState.NORMAL); + new MockUp() { + @Mock + public List getVisibleIndex() { + return Lists.newArrayList(index1); + } + }; + + Set schemaAllIndexes = table.getSchemaAllIndexes(false); + Assert.assertEquals(2, schemaAllIndexes.size()); + Assert.assertFalse(schemaAllIndexes.contains(col3)); + Assert.assertFalse(schemaAllIndexes.contains(col4)); + Assert.assertTrue(schemaAllIndexes.contains(col1)); + Assert.assertTrue(schemaAllIndexes.contains(col2)); + + MaterializedIndex index2 = new MaterializedIndex(2, MaterializedIndex.IndexState.NORMAL); + new MockUp() { + @Mock + public List getVisibleIndex() { + return Lists.newArrayList(index2); + } + }; + schemaAllIndexes = table.getSchemaAllIndexes(false); + Assert.assertEquals(2, schemaAllIndexes.size()); + Assert.assertTrue(schemaAllIndexes.contains(col3)); + Assert.assertTrue(schemaAllIndexes.contains(col4)); + Assert.assertFalse(schemaAllIndexes.contains(col1)); + Assert.assertFalse(schemaAllIndexes.contains(col2)); + + new MockUp() { + @Mock + public List getVisibleIndex() { + return Lists.newArrayList(index1, index2); + } + }; + schemaAllIndexes = table.getSchemaAllIndexes(false); + Assert.assertEquals(4, schemaAllIndexes.size()); + Assert.assertTrue(schemaAllIndexes.contains(col3)); + Assert.assertTrue(schemaAllIndexes.contains(col4)); + Assert.assertTrue(schemaAllIndexes.contains(col1)); + Assert.assertTrue(schemaAllIndexes.contains(col2)); + + col1.setIsVisible(false); + schemaAllIndexes = table.getSchemaAllIndexes(false); + Assert.assertEquals(3, schemaAllIndexes.size()); + Assert.assertTrue(schemaAllIndexes.contains(col3)); + Assert.assertTrue(schemaAllIndexes.contains(col4)); + Assert.assertFalse(schemaAllIndexes.contains(col1)); + Assert.assertTrue(schemaAllIndexes.contains(col2)); + } }