[fix](statistics)Skip shadow index while analyzing a table. (#42201)

Skip shadow index while analyzing a table.
apache · Oct 24, 2024 · fec6e0c · fec6e0c
1 parent 6a429fb
commit fec6e0c
Show file tree

Hide file tree

Showing 2 changed files with 91 additions and 3 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -83,6 +83,7 @@
 import org.apache.doris.thrift.TTableDescriptor;
 import org.apache.doris.thrift.TTableType;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
@@ -954,16 +955,16 @@ public List<Column> getSchemaByIndexId(Long indexId, boolean full) {
         if (full) {
             return indexIdToMeta.get(indexId).getSchema();
         } else {
-            return indexIdToMeta.get(indexId).getSchema().stream().filter(column -> column.isVisible())
+            return indexIdToMeta.get(indexId).getSchema().stream().filter(Column::isVisible)
                     .collect(Collectors.toList());
         }
     }
 
     @Override
     public Set<Column> getSchemaAllIndexes(boolean full) {
         Set<Column> columns = Sets.newHashSet();
-        for (Long indexId : indexIdToMeta.keySet()) {
-            columns.addAll(getSchemaByIndexId(indexId, full));
+        for (MaterializedIndex index : getVisibleIndex()) {
+            columns.addAll(getSchemaByIndexId(index.getId(), full));
         }
         return columns;
     }
@@ -3405,4 +3406,14 @@ public boolean autoAnalyzeEnabled() {
         return properties.get(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY)
                 .equalsIgnoreCase(PropertyAnalyzer.ENABLE_AUTO_ANALYZE_POLICY);
     }
+
+    @VisibleForTesting
+    protected void addIndexIdToMetaForUnitTest(long id, MaterializedIndexMeta meta) {
+        indexIdToMeta.put(id, meta);
+    }
+
+    @VisibleForTesting
+    protected void addIndexNameToIdForUnitTest(String name, long id) {
+        indexNameToId.put(name, id);
+    }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
@@ -22,6 +22,7 @@
 import org.apache.doris.common.FeConstants;
 import org.apache.doris.common.io.FastByteArrayOutputStream;
 import org.apache.doris.common.util.UnitTestUtil;
+import org.apache.doris.thrift.TStorageType;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
@@ -35,6 +36,7 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 public class OlapTableTest {
 
@@ -168,4 +170,79 @@ public void testGetPartitionRowCount() {
 
         olapTable.getRowCountForPartitionIndex(11, 10, true);
     }
+
+    @Test
+    public void testGetSchemaAllIndexes() {
+        OlapTable table = new OlapTable();
+        List<Column> schema1 = Lists.newArrayList();
+        Column col1 = new Column("col1", PrimitiveType.INT);
+        Column col2 = new Column("col2", PrimitiveType.INT);
+        Column col3 = new Column("col3", PrimitiveType.INT);
+        Column col4 = new Column("col4", PrimitiveType.INT);
+        schema1.add(col1);
+        schema1.add(col2);
+        MaterializedIndexMeta meta1 = new MaterializedIndexMeta(1L, schema1, 1, 1, (short) 1,
+                TStorageType.COLUMN, KeysType.DUP_KEYS, null);
+        table.addIndexIdToMetaForUnitTest(1, meta1);
+        table.addIndexNameToIdForUnitTest("index1", 1L);
+
+        List<Column> schema2 = Lists.newArrayList();
+        schema2.add(col3);
+        schema2.add(col4);
+        MaterializedIndexMeta meta2 = new MaterializedIndexMeta(2L, schema2, 1, 1, (short) 1,
+                TStorageType.COLUMN, KeysType.DUP_KEYS, null);
+        table.addIndexIdToMetaForUnitTest(1, meta1);
+        table.addIndexIdToMetaForUnitTest(2, meta2);
+        table.addIndexNameToIdForUnitTest("index2", 2L);
+
+        MaterializedIndex index1 = new MaterializedIndex(1, MaterializedIndex.IndexState.NORMAL);
+        new MockUp<OlapTable>() {
+            @Mock
+            public List<MaterializedIndex> getVisibleIndex() {
+                return Lists.newArrayList(index1);
+            }
+        };
+
+        Set<Column> schemaAllIndexes = table.getSchemaAllIndexes(false);
+        Assert.assertEquals(2, schemaAllIndexes.size());
+        Assert.assertFalse(schemaAllIndexes.contains(col3));
+        Assert.assertFalse(schemaAllIndexes.contains(col4));
+        Assert.assertTrue(schemaAllIndexes.contains(col1));
+        Assert.assertTrue(schemaAllIndexes.contains(col2));
+
+        MaterializedIndex index2 = new MaterializedIndex(2, MaterializedIndex.IndexState.NORMAL);
+        new MockUp<OlapTable>() {
+            @Mock
+            public List<MaterializedIndex> getVisibleIndex() {
+                return Lists.newArrayList(index2);
+            }
+        };
+        schemaAllIndexes = table.getSchemaAllIndexes(false);
+        Assert.assertEquals(2, schemaAllIndexes.size());
+        Assert.assertTrue(schemaAllIndexes.contains(col3));
+        Assert.assertTrue(schemaAllIndexes.contains(col4));
+        Assert.assertFalse(schemaAllIndexes.contains(col1));
+        Assert.assertFalse(schemaAllIndexes.contains(col2));
+
+        new MockUp<OlapTable>() {
+            @Mock
+            public List<MaterializedIndex> getVisibleIndex() {
+                return Lists.newArrayList(index1, index2);
+            }
+        };
+        schemaAllIndexes = table.getSchemaAllIndexes(false);
+        Assert.assertEquals(4, schemaAllIndexes.size());
+        Assert.assertTrue(schemaAllIndexes.contains(col3));
+        Assert.assertTrue(schemaAllIndexes.contains(col4));
+        Assert.assertTrue(schemaAllIndexes.contains(col1));
+        Assert.assertTrue(schemaAllIndexes.contains(col2));
+
+        col1.setIsVisible(false);
+        schemaAllIndexes = table.getSchemaAllIndexes(false);
+        Assert.assertEquals(3, schemaAllIndexes.size());
+        Assert.assertTrue(schemaAllIndexes.contains(col3));
+        Assert.assertTrue(schemaAllIndexes.contains(col4));
+        Assert.assertFalse(schemaAllIndexes.contains(col1));
+        Assert.assertTrue(schemaAllIndexes.contains(col2));
+    }
 }