Subscription IT: intro extensive regression tests from test group

.github/workflows/pipe-it-2cluster.yml

@@ -58,7 +58,8 @@ jobs:
           -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
           -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \
           -pl integration-test \
-          -am -PMultiClusterIT2AutoCreateSchema
+          -am -PMultiClusterIT2AutoCreateSchema \
+          -ntp
       - name: Upload Artifact
         if: failure()
         uses: actions/upload-artifact@v4
@@ -95,15 +96,16 @@ jobs:
           -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
           -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
           -pl integration-test \
-          -am -PMultiClusterIT2ManualCreateSchema
+          -am -PMultiClusterIT2ManualCreateSchema \
+          -ntp
       - name: Upload Artifact
         if: failure()
         uses: actions/upload-artifact@v4
         with:
           name: cluster-log-manual-create-schema-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
           path: integration-test/target/cluster-logs
           retention-days: 30
-  subscription:
+  subscription-arch-verification:
     strategy:
       fail-fast: false
       max-parallel: 15
@@ -132,11 +134,50 @@ jobs:
           -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
           -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
           -pl integration-test \
-          -am -PMultiClusterIT2Subscription
+          -am -PMultiClusterIT2SubscriptionArchVerification \
+          -ntp
       - name: Upload Artifact
         if: failure()
         uses: actions/upload-artifact@v4
         with:
           name: cluster-log-subscription-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
           path: integration-test/target/cluster-logs
           retention-days: 30
+  subscription-regression:
+    strategy:
+      fail-fast: false
+      max-parallel: 15
+      matrix:
+        java: [ 17 ]
+        # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal
+        cluster1: [ ScalableSingleNodeMode ]
+        cluster2: [ ScalableSingleNodeMode ]
+        os: [ ubuntu-latest ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: liberica
+          java-version: ${{ matrix.java }}
+      - name: IT Test
+        shell: bash
+        # we do not compile client-cpp for saving time, it is tested in client.yml
+        # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
+        run: |
+          mvn clean verify \
+          -P with-integration-tests \
+          -DskipUTs \
+          -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
+          -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
+          -pl integration-test \
+          -am -PMultiClusterIT2SubscriptionRegression \
+          -ntp
+      - name: Upload Artifact
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cluster-log-subscription-regression-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
+          path: integration-test/target/cluster-logs
+          retention-days: 30

integration-test/pom.xml

@@ -415,7 +415,7 @@
             </activation>
             <properties>
                 <integrationTest.excludedGroups/>
-                <integrationTest.includedGroups>org.apache.iotdb.itbase.category.MultiClusterIT1,org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema,org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema,org.apache.iotdb.itbase.category.MultiClusterIT2Subscription,org.apache.iotdb.itbase.category.MultiClusterIT3</integrationTest.includedGroups>
+                <integrationTest.includedGroups>org.apache.iotdb.itbase.category.MultiClusterIT1,org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema,org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionArchVerification,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression,org.apache.iotdb.itbase.category.MultiClusterIT3</integrationTest.includedGroups>
                 <integrationTest.launchNodeInSameJVM>false</integrationTest.launchNodeInSameJVM>
                 <integrationTest.randomSelectWriteNode>true</integrationTest.randomSelectWriteNode>
                 <integrationTest.readAndVerifyWithMultiNode>true</integrationTest.readAndVerifyWithMultiNode>
@@ -465,13 +465,27 @@
             </properties>
         </profile>
         <profile>
-            <id>MultiClusterIT2Subscription</id>
+            <id>MultiClusterIT2SubscriptionArchVerification</id>
             <activation>
                 <activeByDefault>false</activeByDefault>
             </activation>
             <properties>
                 <integrationTest.excludedGroups/>
-                <integrationTest.includedGroups>org.apache.iotdb.itbase.category.MultiClusterIT2Subscription</integrationTest.includedGroups>
+                <integrationTest.includedGroups>org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionArchVerification</integrationTest.includedGroups>
+                <integrationTest.launchNodeInSameJVM>false</integrationTest.launchNodeInSameJVM>
+                <integrationTest.randomSelectWriteNode>true</integrationTest.randomSelectWriteNode>
+                <integrationTest.readAndVerifyWithMultiNode>true</integrationTest.readAndVerifyWithMultiNode>
+                <integrationTest.testEnv>MultiCluster</integrationTest.testEnv>
+            </properties>
+        </profile>
+        <profile>
+            <id>MultiClusterIT2SubscriptionRegression</id>
+            <activation>
+                <activeByDefault>false</activeByDefault>
+            </activation>
+            <properties>
+                <integrationTest.excludedGroups/>
+                <integrationTest.includedGroups>org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression</integrationTest.includedGroups>
                 <integrationTest.launchNodeInSameJVM>false</integrationTest.launchNodeInSameJVM>
                 <integrationTest.randomSelectWriteNode>true</integrationTest.randomSelectWriteNode>
                 <integrationTest.readAndVerifyWithMultiNode>true</integrationTest.readAndVerifyWithMultiNode>

integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionArchVerification.java

@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.itbase.category;
+
+public interface MultiClusterIT2SubscriptionArchVerification {}

integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2Subscription.java → integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionRegression.java

@@ -19,4 +19,4 @@
 
 package org.apache.iotdb.itbase.category;
 
-public interface MultiClusterIT2Subscription {}
+public interface MultiClusterIT2SubscriptionRegression {}

integration-test/src/test/java/org/apache/iotdb/subscription/it/AbstractSubscriptionIT.java

@@ -24,19 +24,20 @@
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
-import org.junit.rules.TestName;
 import org.junit.rules.TestRule;
 
 public abstract class AbstractSubscriptionIT {
 
-  @Rule public TestName testName = new TestName();
+  @Rule public DisplayName testName = new DisplayName();
 
-  @Rule public final TestRule skipOnSetUpFailure = new SkipOnSetUpFailure("setUp");
+  @Rule
+  public final TestRule skipOnSetUpAndTearDownFailure =
+      new SkipOnSetUpAndTearDownFailure("setUp", "tearDown");
 
   @Before
-  public void setUp() {
+  public void setUp() throws Exception {
     // set thread name
-    Thread.currentThread().setName(String.format("%s - main", testName.getMethodName()));
+    Thread.currentThread().setName(String.format("%s - main", testName.getDisplayName()));
 
     // set thread pools core size
     SubscriptionExecutorServiceManager.setControlFlowExecutorCorePoolSize(1);
@@ -45,5 +46,5 @@ public void setUp() {
   }
 
   @After
-  public void tearDown() {}
+  public void tearDown() throws Exception {}
 }

integration-test/src/test/java/org/apache/iotdb/subscription/it/DisplayName.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.subscription.it;
+
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+
+public class DisplayName extends TestWatcher {
+
+  private volatile String displayName;
+
+  public DisplayName() {}
+
+  protected void starting(final Description d) {
+    this.displayName = d.getDisplayName();
+  }
+
+  public String getDisplayName() {
+    return displayName;
+  }
+}

integration-test/src/test/java/org/apache/iotdb/subscription/it/SkipOnSetUpAndTearDownFailure.java

@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.subscription.it;
+
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.junit.Assume;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.MultipleFailureException;
+import org.junit.runners.model.Statement;
+
+import java.lang.reflect.Method;
+
+public class SkipOnSetUpAndTearDownFailure implements TestRule {
+
+  private final String setUpMethodName;
+  private final String tearDownMethodName;
+
+  /**
+   * @param setUpMethodName Should be exactly the same as the method name decorated with @Before.
+   * @param tearDownMethodName Should be exactly the same as the method name decorated with @After.
+   */
+  public SkipOnSetUpAndTearDownFailure(
+      @NonNull final String setUpMethodName, @NonNull final String tearDownMethodName) {
+    this.setUpMethodName = setUpMethodName;
+    this.tearDownMethodName = tearDownMethodName;
+  }
+
+  @Override
+  public Statement apply(final Statement base, final Description description) {
+    return new Statement() {
+      @Override
+      public void evaluate() throws Throwable {
+        try {
+          base.evaluate();
+        } catch (final Throwable e) {
+          // Pay attention to the situation of MultipleFailureException...
+          if ((e instanceof MultipleFailureException
+                  && ((MultipleFailureException) e)
+                      .getFailures().stream().allMatch(this::isExceptionInSetUpOrTearDown))
+              || isExceptionInSetUpOrTearDown(e)) {
+            Assume.assumeTrue(
+                String.format(
+                    "Skipping test due to setup or tearDown failure for %s#%s",
+                    description.getClassName(), description.getMethodName()),
+                false);
+          }
+
+          // Re-throw the exception (which means the test has failed).
+          throw e;
+
+          // Regardless of the circumstances, the method decorated with @After will always be
+          // executed.
+        }
+      }
+
+      private boolean isExceptionInSetUpOrTearDown(final Throwable e) {
+        // Trace back the exception stack to determine whether the exception was thrown during the
+        // setUp or tearDown phase.
+        for (final StackTraceElement stackTraceElement : e.getStackTrace()) {
+          if (setUpMethodName.equals(stackTraceElement.getMethodName())
+              && description.getClassName().equals(stackTraceElement.getClassName())
+              && isMethodAnnotationWithBefore(stackTraceElement.getMethodName())) {
+            e.printStackTrace();
+            return true;
+          }
+
+          if (tearDownMethodName.equals(stackTraceElement.getMethodName())
+              && description.getClassName().equals(stackTraceElement.getClassName())
+              && isMethodAnnotationWithAfter(stackTraceElement.getMethodName())) {
+            e.printStackTrace();
+            return true;
+          }
+        }
+        return false;
+      }
+
+      private boolean isMethodAnnotationWithBefore(final String methodName) {
+        try {
+          final Method method = description.getTestClass().getDeclaredMethod(methodName);
+          return method.isAnnotationPresent(org.junit.Before.class);
+        } catch (final Throwable ignored) {
+          return false;
+        }
+      }
+
+      private boolean isMethodAnnotationWithAfter(final String methodName) {
+        try {
+          final Method method = description.getTestClass().getDeclaredMethod(methodName);
+          return method.isAnnotationPresent(org.junit.After.class);
+        } catch (final Throwable ignored) {
+          return false;
+        }
+      }
+    };
+  }
+}