diff --git a/.github/workflows/pipe-it-2cluster.yml b/.github/workflows/pipe-it-2cluster.yml index e19a4b2f9efb..4c638bf7701b 100644 --- a/.github/workflows/pipe-it-2cluster.yml +++ b/.github/workflows/pipe-it-2cluster.yml @@ -58,7 +58,8 @@ jobs: -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ -pl integration-test \ - -am -PMultiClusterIT2AutoCreateSchema + -am -PMultiClusterIT2AutoCreateSchema \ + -ntp - name: Upload Artifact if: failure() uses: actions/upload-artifact@v4 @@ -95,7 +96,8 @@ jobs: -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ -pl integration-test \ - -am -PMultiClusterIT2ManualCreateSchema + -am -PMultiClusterIT2ManualCreateSchema \ + -ntp - name: Upload Artifact if: failure() uses: actions/upload-artifact@v4 @@ -103,7 +105,7 @@ jobs: name: cluster-log-manual-create-schema-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} path: integration-test/target/cluster-logs retention-days: 30 - subscription: + subscription-arch-verification: strategy: fail-fast: false max-parallel: 15 @@ -132,7 +134,8 @@ jobs: -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ -pl integration-test \ - -am -PMultiClusterIT2Subscription + -am -PMultiClusterIT2SubscriptionArchVerification \ + -ntp - name: Upload Artifact if: failure() uses: actions/upload-artifact@v4 @@ -140,3 +143,41 @@ jobs: name: cluster-log-subscription-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} path: integration-test/target/cluster-logs retention-days: 30 + subscription-regression: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [ 17 ] + # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal + cluster1: [ ScalableSingleNodeMode ] + cluster2: [ ScalableSingleNodeMode ] + os: [ ubuntu-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: ${{ matrix.java }} + - name: IT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ + -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ + -pl integration-test \ + -am -PMultiClusterIT2SubscriptionRegression \ + -ntp + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cluster-log-subscription-regression-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} + path: integration-test/target/cluster-logs + retention-days: 30 diff --git a/integration-test/pom.xml b/integration-test/pom.xml index 896a9bfa9408..aefd0027f26c 100644 --- a/integration-test/pom.xml +++ b/integration-test/pom.xml @@ -415,7 +415,7 @@ - org.apache.iotdb.itbase.category.MultiClusterIT1,org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema,org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema,org.apache.iotdb.itbase.category.MultiClusterIT2Subscription,org.apache.iotdb.itbase.category.MultiClusterIT3 + org.apache.iotdb.itbase.category.MultiClusterIT1,org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema,org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionArchVerification,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression,org.apache.iotdb.itbase.category.MultiClusterIT3 false true true @@ -465,13 +465,27 @@ - MultiClusterIT2Subscription + MultiClusterIT2SubscriptionArchVerification false - org.apache.iotdb.itbase.category.MultiClusterIT2Subscription + org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionArchVerification + false + true + true + MultiCluster + + + + MultiClusterIT2SubscriptionRegression + + false + + + + org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression false true true diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionArchVerification.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionArchVerification.java new file mode 100644 index 000000000000..3f0c5afa50da --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionArchVerification.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.itbase.category; + +public interface MultiClusterIT2SubscriptionArchVerification {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2Subscription.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionRegression.java similarity index 93% rename from integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2Subscription.java rename to integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionRegression.java index 34d870e4d6d1..36d9dddb9751 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2Subscription.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2SubscriptionRegression.java @@ -19,4 +19,4 @@ package org.apache.iotdb.itbase.category; -public interface MultiClusterIT2Subscription {} +public interface MultiClusterIT2SubscriptionRegression {} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/AbstractSubscriptionIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/AbstractSubscriptionIT.java index a168dbc78ba7..bae161ba24ff 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/AbstractSubscriptionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/AbstractSubscriptionIT.java @@ -24,19 +24,20 @@ import org.junit.After; import org.junit.Before; import org.junit.Rule; -import org.junit.rules.TestName; import org.junit.rules.TestRule; public abstract class AbstractSubscriptionIT { - @Rule public TestName testName = new TestName(); + @Rule public DisplayName testName = new DisplayName(); - @Rule public final TestRule skipOnSetUpFailure = new SkipOnSetUpFailure("setUp"); + @Rule + public final TestRule skipOnSetUpAndTearDownFailure = + new SkipOnSetUpAndTearDownFailure("setUp", "tearDown"); @Before - public void setUp() { + public void setUp() throws Exception { // set thread name - Thread.currentThread().setName(String.format("%s - main", testName.getMethodName())); + Thread.currentThread().setName(String.format("%s - main", testName.getDisplayName())); // set thread pools core size SubscriptionExecutorServiceManager.setControlFlowExecutorCorePoolSize(1); @@ -45,5 +46,5 @@ public void setUp() { } @After - public void tearDown() {} + public void tearDown() throws Exception {} } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/DisplayName.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/DisplayName.java new file mode 100644 index 000000000000..b1faae54287b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/DisplayName.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it; + +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; + +public class DisplayName extends TestWatcher { + + private volatile String displayName; + + public DisplayName() {} + + protected void starting(final Description d) { + this.displayName = d.getDisplayName(); + } + + public String getDisplayName() { + return displayName; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/SkipOnSetUpAndTearDownFailure.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/SkipOnSetUpAndTearDownFailure.java new file mode 100644 index 000000000000..6c3891612198 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/SkipOnSetUpAndTearDownFailure.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it; + +import org.checkerframework.checker.nullness.qual.NonNull; +import org.junit.Assume; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.MultipleFailureException; +import org.junit.runners.model.Statement; + +import java.lang.reflect.Method; + +public class SkipOnSetUpAndTearDownFailure implements TestRule { + + private final String setUpMethodName; + private final String tearDownMethodName; + + /** + * @param setUpMethodName Should be exactly the same as the method name decorated with @Before. + * @param tearDownMethodName Should be exactly the same as the method name decorated with @After. + */ + public SkipOnSetUpAndTearDownFailure( + @NonNull final String setUpMethodName, @NonNull final String tearDownMethodName) { + this.setUpMethodName = setUpMethodName; + this.tearDownMethodName = tearDownMethodName; + } + + @Override + public Statement apply(final Statement base, final Description description) { + return new Statement() { + @Override + public void evaluate() throws Throwable { + try { + base.evaluate(); + } catch (final Throwable e) { + // Pay attention to the situation of MultipleFailureException... + if ((e instanceof MultipleFailureException + && ((MultipleFailureException) e) + .getFailures().stream().allMatch(this::isExceptionInSetUpOrTearDown)) + || isExceptionInSetUpOrTearDown(e)) { + Assume.assumeTrue( + String.format( + "Skipping test due to setup or tearDown failure for %s#%s", + description.getClassName(), description.getMethodName()), + false); + } + + // Re-throw the exception (which means the test has failed). + throw e; + + // Regardless of the circumstances, the method decorated with @After will always be + // executed. + } + } + + private boolean isExceptionInSetUpOrTearDown(final Throwable e) { + // Trace back the exception stack to determine whether the exception was thrown during the + // setUp or tearDown phase. + for (final StackTraceElement stackTraceElement : e.getStackTrace()) { + if (setUpMethodName.equals(stackTraceElement.getMethodName()) + && description.getClassName().equals(stackTraceElement.getClassName()) + && isMethodAnnotationWithBefore(stackTraceElement.getMethodName())) { + e.printStackTrace(); + return true; + } + + if (tearDownMethodName.equals(stackTraceElement.getMethodName()) + && description.getClassName().equals(stackTraceElement.getClassName()) + && isMethodAnnotationWithAfter(stackTraceElement.getMethodName())) { + e.printStackTrace(); + return true; + } + } + return false; + } + + private boolean isMethodAnnotationWithBefore(final String methodName) { + try { + final Method method = description.getTestClass().getDeclaredMethod(methodName); + return method.isAnnotationPresent(org.junit.Before.class); + } catch (final Throwable ignored) { + return false; + } + } + + private boolean isMethodAnnotationWithAfter(final String methodName) { + try { + final Method method = description.getTestClass().getDeclaredMethod(methodName); + return method.isAnnotationPresent(org.junit.After.class); + } catch (final Throwable ignored) { + return false; + } + } + }; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/SkipOnSetUpFailure.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/SkipOnSetUpFailure.java deleted file mode 100644 index add7b7c1e2b8..000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/SkipOnSetUpFailure.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.subscription.it; - -import org.checkerframework.checker.nullness.qual.NonNull; -import org.junit.AssumptionViolatedException; -import org.junit.rules.TestRule; -import org.junit.runner.Description; -import org.junit.runners.model.Statement; - -import java.lang.reflect.Method; - -public class SkipOnSetUpFailure implements TestRule { - - private final String setUpMethodName; - - /** - * @param setUpMethodName Should be exactly the same as the method name decorated with @Before. - */ - public SkipOnSetUpFailure(@NonNull final String setUpMethodName) { - this.setUpMethodName = setUpMethodName; - } - - @Override - public Statement apply(final Statement base, final Description description) { - return new Statement() { - @Override - public void evaluate() throws Throwable { - try { - base.evaluate(); - } catch (final Throwable e) { - // Trace back the exception stack to determine whether the exception was thrown during the - // setUp phase. - for (final StackTraceElement stackTraceElement : e.getStackTrace()) { - if (setUpMethodName.equals(stackTraceElement.getMethodName()) - && description.getClassName().equals(stackTraceElement.getClassName()) - && isMethodAnnotationWithBefore(stackTraceElement.getMethodName())) { - e.printStackTrace(); - // Skip this test. - throw new AssumptionViolatedException( - String.format( - "Skipping test due to setup failure for %s#%s", - description.getClassName(), description.getMethodName())); - } - } - - // Re-throw the exception (which means the test has failed). - throw e; - - // Regardless of the circumstances, the method decorated with @After will always be - // executed. - } - } - - private boolean isMethodAnnotationWithBefore(final String methodName) { - try { - final Method method = description.getTestClass().getDeclaredMethod(methodName); - return method.isAnnotationPresent(org.junit.Before.class); - } catch (final Throwable ignored) { - return false; - } - } - }; - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java index 41dd654383d6..fee93bacdd5d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java @@ -39,6 +39,7 @@ import org.apache.iotdb.subscription.it.AbstractSubscriptionIT; import org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant; +import org.apache.tsfile.utils.Pair; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -48,9 +49,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; @@ -65,7 +68,7 @@ public class IoTDBSubscriptionRestartIT extends AbstractSubscriptionIT { @Override @Before - public void setUp() { + public void setUp() throws Exception { super.setUp(); EnvFactory.getEnv() @@ -82,10 +85,10 @@ public void setUp() { @Override @After - public void tearDown() { - super.tearDown(); - + public void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); + + super.tearDown(); } @Test @@ -104,9 +107,10 @@ public void testSubscriptionAfterRestartCluster() throws Exception { } // Subscription - final SubscriptionPullConsumer consumer; + final SubscriptionPullConsumer consumer1; + final SubscriptionPullConsumer consumer2; try { - consumer = + consumer1 = new SubscriptionPullConsumer.Builder() .host(host) .port(port) @@ -116,8 +120,21 @@ public void testSubscriptionAfterRestartCluster() throws Exception { .heartbeatIntervalMs(1000) // narrow heartbeat interval .endpointsSyncIntervalMs(5000) // narrow endpoints sync interval .buildPullConsumer(); - consumer.open(); - consumer.subscribe(topicName); + consumer1.open(); + consumer1.subscribe(topicName); + + consumer2 = + new SubscriptionPullConsumer.Builder() + .host(host) + .port(port) + .consumerId("c2") + .consumerGroupId("cg2") + .autoCommit(true) + .heartbeatIntervalMs(1000) // narrow heartbeat interval + .endpointsSyncIntervalMs(5000) // narrow endpoints sync interval + .buildPullConsumer(); + consumer2.open(); + consumer2.subscribe(topicName); } catch (final Exception e) { e.printStackTrace(); fail(e.getMessage()); @@ -145,7 +162,7 @@ public void testSubscriptionAfterRestartCluster() throws Exception { client.showSubscription(new TShowSubscriptionReq()); Assert.assertEquals(RpcUtils.SUCCESS_STATUS.getCode(), showSubscriptionResp.status.getCode()); Assert.assertNotNull(showSubscriptionResp.subscriptionInfoList); - Assert.assertEquals(1, showSubscriptionResp.subscriptionInfoList.size()); + Assert.assertEquals(2, showSubscriptionResp.subscriptionInfoList.size()); } // Insert some historical data @@ -162,17 +179,18 @@ public void testSubscriptionAfterRestartCluster() throws Exception { } // Subscription again - final Map timestamps = new HashMap<>(); + final Map, Long> timestamps = new ConcurrentHashMap<>(); final AtomicBoolean isClosed = new AtomicBoolean(false); - final Thread thread = + final List threads = new ArrayList<>(); + threads.add( new Thread( () -> { - try (final SubscriptionPullConsumer consumerRef = consumer) { + try (final SubscriptionPullConsumer consumerRef1 = consumer1) { while (!isClosed.get()) { LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time final List messages; try { - messages = consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + messages = consumerRef1.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); } catch (final Exception e) { e.printStackTrace(); // Avoid failure @@ -183,13 +201,13 @@ public void testSubscriptionAfterRestartCluster() throws Exception { message.getSessionDataSetsHandler()) { while (dataSet.hasNext()) { final long timestamp = dataSet.next().getTimestamp(); - timestamps.put(timestamp, timestamp); + timestamps.put(new Pair<>(timestamp, consumerRef1.toString()), timestamp); } } } // Auto commit } - consumerRef.unsubscribe(topicName); + consumerRef1.unsubscribe(topicName); } catch (final Exception e) { e.printStackTrace(); // Avoid failure @@ -197,19 +215,57 @@ public void testSubscriptionAfterRestartCluster() throws Exception { LOGGER.info("consumer exiting..."); } }, - String.format("%s - %s", testName.getMethodName(), consumer)); - thread.start(); + String.format("%s - %s", testName.getDisplayName(), consumer1))); + threads.add( + new Thread( + () -> { + try (final SubscriptionPullConsumer consumerRef2 = consumer2) { + while (!isClosed.get()) { + LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time + final List messages; + try { + messages = consumerRef2.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS); + } catch (final Exception e) { + e.printStackTrace(); + // Avoid failure + continue; + } + for (final SubscriptionMessage message : messages) { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + while (dataSet.hasNext()) { + final long timestamp = dataSet.next().getTimestamp(); + timestamps.put(new Pair<>(timestamp, consumerRef2.toString()), timestamp); + } + } + } + // Auto commit + } + consumerRef2.unsubscribe(topicName); + } catch (final Exception e) { + e.printStackTrace(); + // Avoid failure + } finally { + LOGGER.info("consumer exiting..."); + } + }, + String.format("%s - %s", testName.getDisplayName(), consumer2))); + for (final Thread thread : threads) { + thread.start(); + } // Check timestamps size try { // Keep retrying if there are execution failures - AWAIT.untilAsserted(() -> Assert.assertEquals(100, timestamps.size())); + AWAIT.untilAsserted(() -> Assert.assertEquals(200, timestamps.size())); } catch (final Exception e) { e.printStackTrace(); fail(e.getMessage()); } finally { isClosed.set(true); - thread.join(); + for (final Thread thread : threads) { + thread.join(); + } } } @@ -310,7 +366,7 @@ public void testSubscriptionAfterRestartDataNode() throws Exception { LOGGER.info("consumer exiting..."); } }, - String.format("%s - %s", testName.getMethodName(), consumer)); + String.format("%s - %s", testName.getDisplayName(), consumer)); thread.start(); // Start DN 1 & DN 2 @@ -437,7 +493,7 @@ public void testSubscriptionWhenConfigNodeLeaderChange() throws Exception { LOGGER.info("consumer exiting..."); } }, - String.format("%s - %s", testName.getMethodName(), consumer)); + String.format("%s - %s", testName.getDisplayName(), consumer)); thread.start(); // Shutdown leader CN diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java index fb25cdc3a6c0..5c4fc94a7c3e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java @@ -26,14 +26,14 @@ import org.junit.After; import org.junit.Before; -abstract class AbstractSubscriptionDualIT extends AbstractSubscriptionIT { +public abstract class AbstractSubscriptionDualIT extends AbstractSubscriptionIT { protected BaseEnv senderEnv; protected BaseEnv receiverEnv; @Override @Before - public void setUp() { + public void setUp() throws Exception { super.setUp(); MultiEnvFactory.createEnv(2); @@ -58,10 +58,10 @@ protected void setUpConfig() { @Override @After - public void tearDown() { - super.tearDown(); - + public void tearDown() throws Exception { senderEnv.cleanClusterEnvironment(); receiverEnv.cleanClusterEnvironment(); + + super.tearDown(); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java index eccfc634e6b9..6f8cba43e792 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java @@ -26,7 +26,7 @@ import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2Subscription; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionArchVerification; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.iotdb.session.subscription.SubscriptionSession; @@ -69,7 +69,7 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2Subscription.class}) +@Category({MultiClusterIT2SubscriptionArchVerification.class}) public class IoTDBSubscriptionConsumerGroupIT extends AbstractSubscriptionDualIT { // Test dimensions: @@ -123,7 +123,7 @@ protected void setUpConfig() { @Override @Before - public void setUp() { + public void setUp() throws Exception { super.setUp(); // Setup connector attributes @@ -1036,7 +1036,7 @@ private void pollMessagesAndCheck( LOGGER.info("consumer {} exiting...", consumers.get(index)); } }, - String.format("%s - %s", testName.getMethodName(), consumers.get(index).toString())); + String.format("%s - %s", testName.getDisplayName(), consumers.get(index).toString())); t.start(); threads.add(t); } @@ -1058,7 +1058,7 @@ private void pollMessagesAndCheck( for (final DataNodeWrapper wrapper : senderEnv.getDataNodeWrapperList()) { // wrapper.executeJstack(); wrapper.executeJstack( - String.format("%s_%s", testName.getMethodName(), currentTime[0])); + String.format("%s_%s", testName.getDisplayName(), currentTime[0])); } currentTime[0] = System.currentTimeMillis(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java index 39a9f2225f6b..a9eef996012c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java @@ -22,7 +22,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2Subscription; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionArchVerification; import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.iotdb.session.subscription.SubscriptionSession; import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; @@ -49,7 +49,7 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2Subscription.class}) +@Category({MultiClusterIT2SubscriptionArchVerification.class}) public class IoTDBSubscriptionTimePrecisionIT extends AbstractSubscriptionDualIT { private static final Logger LOGGER = @@ -160,7 +160,7 @@ public void testTopicTimePrecision() throws Exception { LOGGER.info("consumer exiting..."); } }, - String.format("%s - consumer", testName.getMethodName())); + String.format("%s - consumer", testName.getDisplayName())); thread.start(); // Check data on receiver diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java index ce8b46f2aead..1317dd2dbf02 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java @@ -27,7 +27,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2Subscription; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionArchVerification; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.iotdb.session.subscription.SubscriptionSession; @@ -68,7 +68,7 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2Subscription.class}) +@Category({MultiClusterIT2SubscriptionArchVerification.class}) public class IoTDBSubscriptionTopicIT extends AbstractSubscriptionDualIT { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSubscriptionTopicIT.class); @@ -162,7 +162,7 @@ private void testTopicWithPathTemplate(final String topicFormat) throws Exceptio LOGGER.info("consumer exiting..."); } }, - String.format("%s - consumer", testName.getMethodName())); + String.format("%s - consumer", testName.getDisplayName())); thread.start(); // Check data on receiver @@ -263,7 +263,7 @@ private void testTopicWithTimeTemplate(final String topicFormat) throws Exceptio LOGGER.info("consumer exiting..."); } }, - String.format("%s - consumer", testName.getMethodName())); + String.format("%s - consumer", testName.getDisplayName())); thread.start(); // Check data on receiver @@ -360,7 +360,7 @@ private void testTopicWithProcessorTemplate(final String topicFormat) throws Exc LOGGER.info("consumer exiting..."); } }, - String.format("%s - consumer", testName.getMethodName())); + String.format("%s - consumer", testName.getDisplayName())); thread.start(); // Check data on receiver @@ -484,7 +484,7 @@ public void testTopicNameWithBackQuote() throws Exception { LOGGER.info("consumer exiting..."); } }, - String.format("%s - consumer", testName.getMethodName())); + String.format("%s - consumer", testName.getDisplayName())); thread.start(); // Check data on receiver @@ -658,7 +658,7 @@ private void testTopicWithSnapshotModeTemplate(final String topicFormat) throws LOGGER.info("consumer exiting..."); } }, - String.format("%s - consumer", testName.getMethodName())); + String.format("%s - consumer", testName.getDisplayName())); thread.start(); try { @@ -769,7 +769,7 @@ private void testTopicWithLooseRangeTemplate(final String topicFormat) throws Ex LOGGER.info("consumer exiting..."); } }, - String.format("%s - consumer", testName.getMethodName()))); + String.format("%s - consumer", testName.getDisplayName()))); // Insert some realtime data on sender threads.add( @@ -792,7 +792,7 @@ private void testTopicWithLooseRangeTemplate(final String topicFormat) throws Ex } dataPrepared.set(true); }, - String.format("%s - data inserter", testName.getMethodName()))); + String.format("%s - data inserter", testName.getDisplayName()))); for (final Thread thread : threads) { thread.start(); diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java index bb248f745d4e..5fa7c5808fa8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java @@ -25,11 +25,11 @@ import org.junit.After; import org.junit.Before; -abstract class AbstractSubscriptionLocalIT extends AbstractSubscriptionIT { +public abstract class AbstractSubscriptionLocalIT extends AbstractSubscriptionIT { @Override @Before - public void setUp() { + public void setUp() throws Exception { super.setUp(); EnvFactory.getEnv().initClusterEnvironment(); @@ -37,9 +37,9 @@ public void setUp() { @Override @After - public void tearDown() { - super.tearDown(); - + public void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); + + super.tearDown(); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java index 68287ab66193..12e500501967 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java @@ -162,7 +162,7 @@ public void onFailure(final Throwable e) { LOGGER.info("consumer exiting..."); } }, - String.format("%s - consumer", testName.getMethodName())); + String.format("%s - consumer", testName.getDisplayName())); thread.start(); // Check row count @@ -391,7 +391,7 @@ public void testPollUnsubscribedTopics() throws Exception { LOGGER.info("consumer exiting..."); } }, - String.format("%s - consumer", testName.getMethodName())); + String.format("%s - consumer", testName.getDisplayName())); thread.start(); // Check row count diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionDataTypeIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionDataTypeIT.java index 413540a89c1f..723b088aac3e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionDataTypeIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionDataTypeIT.java @@ -326,7 +326,7 @@ private void testPullConsumerSubscribeDataTemplate( LOGGER.info("consumer exiting..."); } }, - String.format("%s - consumer", testName.getMethodName())); + String.format("%s - consumer", testName.getDisplayName())); thread.start(); // Check row count diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java index d0d23a0407f0..5f607a259683 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java @@ -27,7 +27,7 @@ import org.junit.After; import org.junit.Before; -abstract class AbstractSubscriptionTripleIT extends AbstractSubscriptionIT { +public abstract class AbstractSubscriptionTripleIT extends AbstractSubscriptionIT { protected BaseEnv sender; protected BaseEnv receiver1; @@ -35,7 +35,7 @@ abstract class AbstractSubscriptionTripleIT extends AbstractSubscriptionIT { @Override @Before - public void setUp() { + public void setUp() throws Exception { super.setUp(); // increase the number of threads to speed up testing @@ -68,11 +68,11 @@ protected void setUpConfig() { @Override @After - public void tearDown() { - super.tearDown(); - + public void tearDown() throws Exception { sender.cleanClusterEnvironment(); receiver1.cleanClusterEnvironment(); receiver2.cleanClusterEnvironment(); + + super.tearDown(); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/IoTDBSubscriptionSharingIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/IoTDBSubscriptionSharingIT.java index 0dc6edf1548c..63f560a665f5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/IoTDBSubscriptionSharingIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/IoTDBSubscriptionSharingIT.java @@ -22,7 +22,7 @@ import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2Subscription; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionArchVerification; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; @@ -59,8 +59,12 @@ import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; import static org.junit.Assert.fail; +/** + * refer to {@link + * org.apache.iotdb.subscription.it.triple.regression.pushconsumer.multi.IoTDBMultiGroupVsMultiConsumerIT} + */ @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2Subscription.class}) +@Category({MultiClusterIT2SubscriptionArchVerification.class}) public class IoTDBSubscriptionSharingIT extends AbstractSubscriptionTripleIT { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSubscriptionSharingIT.class); @@ -334,7 +338,6 @@ private void preparePushConsumers() { return ConsumeResult.SUCCESS; }) .buildPushConsumer()); - consumers.add( new SubscriptionPushConsumer.Builder() .host(sender.getIP()) @@ -390,7 +393,7 @@ private void preparePushConsumers() { reader.query( QueryExpression.create( Collections.singletonList( - new Path(databasePrefix + 6 + ".d_0", "s_0", true)), + new Path(databasePrefix + "6.d_0", "s_0", true)), null)); while (dataset.hasNext()) { rowCount6.addAndGet(1); @@ -465,7 +468,7 @@ private void preparePushConsumers() { @Override @Before - public void setUp() { + public void setUp() throws Exception { super.setUp(); // prepare schemaList @@ -475,7 +478,7 @@ public void setUp() { @Override @After - public void tearDown() { + public void tearDown() throws Exception { // log some info try { LOGGER.info("[src] {} = {}", sql1, getCount(sender, sql1)); @@ -531,7 +534,8 @@ public void testSubscriptionSharing() { getCount(sender, sql1), getCount(receiver1, sql1) + getCount(receiver2, sql1)); // "c4,c6|topic2" - Assert.assertEquals(105, getCount(receiver1, sql2) + getCount(receiver2, sql2)); + Assert.assertEquals( + getCount(sender, sql2) - 400, getCount(receiver1, sql2) + getCount(receiver2, sql2)); // "c4,c5|c7,c9|topic3" final long topic3Total = getCount(receiver1, sql3) + getCount(receiver2, sql3); diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/AbstractSubscriptionRegressionIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/AbstractSubscriptionRegressionIT.java new file mode 100644 index 000000000000..21ed309c0eb9 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/AbstractSubscriptionRegressionIT.java @@ -0,0 +1,467 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression; + +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.session.subscription.SubscriptionSession; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.session.subscription.payload.SubscriptionTsFileHandler; +import org.apache.iotdb.subscription.it.triple.AbstractSubscriptionTripleIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.junit.After; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Before; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.time.Duration; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS; + +public abstract class AbstractSubscriptionRegressionIT extends AbstractSubscriptionTripleIT { + + private static final Logger LOGGER = + LoggerFactory.getLogger(AbstractSubscriptionRegressionIT.class); + private static final String DROP_DATABASE_SQL = "drop database "; + + protected static final SimpleDateFormat FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + + public String SRC_HOST; + public String DEST_HOST; + public String DEST_HOST2; + + public int SRC_PORT; + public int DEST_PORT; + public int DEST_PORT2; + + public SubscriptionSession subs; + + public Session session_src; + public Session session_dest; + public Session session_dest2; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + beforeSuite(); + } + + @Override + @After + public void tearDown() throws Exception { + afterSuite(); + super.tearDown(); + } + + public void beforeSuite() throws IoTDBConnectionException { + SRC_HOST = sender.getIP(); + DEST_HOST = receiver1.getIP(); + DEST_HOST2 = receiver2.getIP(); + + SRC_PORT = Integer.parseInt(sender.getPort()); + DEST_PORT = Integer.parseInt(receiver1.getPort()); + DEST_PORT2 = Integer.parseInt(receiver2.getPort()); + + session_src = + new Session.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .username("root") + .password("root") + .zoneId(ZoneId.of("Asia/Shanghai")) + .build(); + session_dest = + new Session.Builder() + .host(DEST_HOST) + .port(DEST_PORT) + .username("root") + .password("root") + .zoneId(ZoneId.of("Asia/Shanghai")) + .build(); + session_dest2 = + new Session.Builder() + .host(DEST_HOST2) + .port(DEST_PORT2) + .username("root") + .password("root") + .zoneId(ZoneId.of("Asia/Shanghai")) + .build(); + session_src.open(false); + session_dest.open(false); + session_dest2.open(false); + + subs = new SubscriptionSession(SRC_HOST, SRC_PORT); + subs.open(); + System.out.println("TestConfig beforeClass"); + } + + public void afterSuite() throws IoTDBConnectionException { + System.out.println("TestConfig afterClass"); + session_src.close(); + session_dest.close(); + session_dest2.close(); + subs.close(); + } + + public void createDB(String database) throws IoTDBConnectionException { + try { + session_src.createDatabase(database); + } catch (StatementExecutionException e) { + } + try { + session_dest.createDatabase(database); + } catch (StatementExecutionException e) { + } + try { + session_dest2.createDatabase(database); + } catch (StatementExecutionException e) { + } + } + + public void dropDB(String pattern) throws IoTDBConnectionException { + try { + session_src.executeNonQueryStatement(DROP_DATABASE_SQL + pattern + "*"); + } catch (StatementExecutionException e) { + System.out.println("### src:" + e); + } + try { + session_dest.executeNonQueryStatement(DROP_DATABASE_SQL + pattern + "*"); + } catch (StatementExecutionException e) { + System.out.println("### dest:" + e); + } + try { + session_dest2.executeNonQueryStatement(DROP_DATABASE_SQL + pattern + "*"); + } catch (StatementExecutionException e) { + System.out.println("### dest2:" + e); + } + } + + public SubscriptionPullConsumer create_pull_consumer( + String groupId, String consumerId, Boolean autoCommit, Long interval) + throws TException, IoTDBConnectionException, IOException, StatementExecutionException { + SubscriptionPullConsumer pullConsumer; + Properties properties = new Properties(); + properties.put(ConsumerConstant.HOST_KEY, SRC_HOST); + properties.put(ConsumerConstant.PORT_KEY, SRC_PORT); + if (groupId != null) { + properties.put(ConsumerConstant.CONSUMER_GROUP_ID_KEY, groupId); + } + if (consumerId != null) { + properties.put(ConsumerConstant.CONSUMER_ID_KEY, consumerId); + } + if (autoCommit != null) { + properties.put(ConsumerConstant.AUTO_COMMIT_KEY, autoCommit); + } + if (interval != null) { + properties.put(ConsumerConstant.AUTO_COMMIT_INTERVAL_MS_KEY, interval); + } + properties.put(ConsumerConstant.FILE_SAVE_DIR_KEY, "target/pull-subscription"); + pullConsumer = new SubscriptionPullConsumer(properties); + pullConsumer.open(); + return pullConsumer; + } + + public void createTopic_s( + String topicName, String pattern, String start, String end, boolean isTsfile) + throws IoTDBConnectionException, StatementExecutionException { + Properties properties = new Properties(); + if (pattern != null) { + properties.setProperty("path", pattern); + } + if (start != null) { + properties.setProperty("start-time", start); + } + if (end != null) { + properties.setProperty("end-time", end); + } + if (isTsfile) { + properties.setProperty("format", "TsFileHandler"); + } else { + properties.setProperty("format", "SessionDataSet"); + } + properties.setProperty("processor", "do-nothing-processor"); + subs.createTopic(topicName, properties); + } + + public void createTopic_s( + String topicName, + String pattern, + String start, + String end, + boolean isTsfile, + String mode, + String loose_range) + throws IoTDBConnectionException, StatementExecutionException { + Properties properties = new Properties(); + if (pattern != null) { + properties.setProperty(TopicConstant.PATH_KEY, pattern); + } + if (start != null) { + properties.setProperty(TopicConstant.START_TIME_KEY, start); + } + if (end != null) { + properties.setProperty(TopicConstant.END_TIME_KEY, end); + } + if (isTsfile) { + properties.setProperty(TopicConstant.FORMAT_KEY, "TsFileHandler"); + } else { + properties.setProperty(TopicConstant.FORMAT_KEY, "SessionDataSet"); + } + if (mode != null && !mode.isEmpty()) { + properties.setProperty(TopicConstant.MODE_KEY, mode); + } + if (loose_range != null && !loose_range.isEmpty()) { + properties.setProperty(TopicConstant.LOOSE_RANGE_KEY, loose_range); + } + subs.createTopic(topicName, properties); + } + + public static long getCount(Session session, String sql) + throws IoTDBConnectionException, StatementExecutionException { + SessionDataSet dataSet = session.executeQueryStatement(sql); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + long result = rowRecord.getFields().get(0).getLongV(); + return result; + } + return 0; + } + + public void check_count(int expect_count, String sql, String msg) + throws IoTDBConnectionException, StatementExecutionException { + assertEquals(getCount(session_dest, sql), expect_count, "Query count:" + msg); + } + + public void check_count2(int expect_count, String sql, String msg) + throws IoTDBConnectionException, StatementExecutionException { + assertEquals(getCount(session_dest2, sql), expect_count, "Query count:" + msg); + } + + public void consume_data(SubscriptionPullConsumer consumer, Session session) + throws TException, + IOException, + StatementExecutionException, + InterruptedException, + IoTDBConnectionException { + while (true) { + Thread.sleep(1000); + + List messages = consumer.poll(Duration.ofMillis(POLL_TIMEOUT_MS)); + if (messages.isEmpty()) { + break; + } + for (final SubscriptionMessage message : messages) { + for (final Iterator it = message.getSessionDataSetsHandler().tabletIterator(); + it.hasNext(); ) { + final Tablet tablet = it.next(); + session.insertTablet(tablet); + } + } + consumer.commitSync(messages); + } + } + + public List consume_tsfile_withFileCount( + SubscriptionPullConsumer consumer, String device) throws InterruptedException { + return consume_tsfile(consumer, Collections.singletonList(device)); + } + + public int consume_tsfile(SubscriptionPullConsumer consumer, String device) + throws InterruptedException { + return consume_tsfile(consumer, Collections.singletonList(device)).get(0); + } + + public List consume_tsfile(SubscriptionPullConsumer consumer, List devices) + throws InterruptedException { + List rowCounts = new ArrayList<>(devices.size()); + for (int i = 0; i < devices.size(); i++) { + rowCounts.add(new AtomicInteger(0)); + } + AtomicInteger onReceived = new AtomicInteger(0); + while (true) { + Thread.sleep(1000); + // That is, the consumer poll will keep pulling if no messages are fetched within the timeout, + // until a message is fetched or the time exceeds the timeout. + List messages = consumer.poll(Duration.ofMillis(POLL_TIMEOUT_MS)); + if (messages.isEmpty()) { + break; + } + for (final SubscriptionMessage message : messages) { + onReceived.incrementAndGet(); + // System.out.println(FORMAT.format(new Date()) + " onReceived=" + onReceived.get()); + final SubscriptionTsFileHandler tsFileHandler = message.getTsFileHandler(); + try (final TsFileReader tsFileReader = tsFileHandler.openReader()) { + for (int i = 0; i < devices.size(); i++) { + final Path path = new Path(devices.get(i), "s_0", true); + final QueryDataSet dataSet = + tsFileReader.query(QueryExpression.create(Collections.singletonList(path), null)); + while (dataSet.hasNext()) { + RowRecord next = dataSet.next(); + rowCounts.get(i).addAndGet(1); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + // System.out.println(FORMAT.format(new Date()) + " consume tsfile " + i + ":" + + // rowCounts.get(i).get()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + consumer.commitSync(messages); + } + List results = new ArrayList<>(devices.size()); + for (AtomicInteger rowCount : rowCounts) { + results.add(rowCount.get()); + } + results.add(onReceived.get()); + return results; + } + + public static void consume_data_long( + SubscriptionPullConsumer consumer, Session session, Long timeout) + throws StatementExecutionException, InterruptedException, IoTDBConnectionException { + timeout = System.currentTimeMillis() + timeout; + while (System.currentTimeMillis() < timeout) { + List messages = consumer.poll(Duration.ofMillis(POLL_TIMEOUT_MS)); + if (messages.isEmpty()) { + Thread.sleep(1000); + } + for (final SubscriptionMessage message : messages) { + for (final Iterator it = message.getSessionDataSetsHandler().tabletIterator(); + it.hasNext(); ) { + final Tablet tablet = it.next(); + session.insertTablet(tablet); + } + } + consumer.commitSync(messages); + } + } + + public void consume_data(SubscriptionPullConsumer consumer) + throws TException, + IOException, + StatementExecutionException, + InterruptedException, + IoTDBConnectionException { + consume_data(consumer, session_dest); + } + + //////////////////////////// strict assertions //////////////////////////// + + public static void assertEquals(int actual, int expected) { + Assert.assertEquals(expected, actual); + } + + public static void assertEquals(Integer actual, int expected) { + Assert.assertEquals(expected, (Object) actual); + } + + public static void assertEquals(int actual, Integer expected) { + Assert.assertEquals((Object) expected, actual); + } + + public static void assertEquals(Integer actual, Integer expected) { + Assert.assertEquals(expected, actual); + } + + public static void assertEquals(int actual, int expected, String message) { + Assert.assertEquals(message, expected, actual); + } + + public static void assertEquals(Integer actual, int expected, String message) { + Assert.assertEquals(message, expected, (Object) actual); + } + + public static void assertEquals(int actual, Integer expected, String message) { + Assert.assertEquals(message, (Object) expected, actual); + } + + public static void assertEquals(Integer actual, Integer expected, String message) { + Assert.assertEquals(message, expected, actual); + } + + public static void assertEquals(long actual, long expected, String message) { + Assert.assertEquals(message, expected, actual); + } + + public static void assertEquals(boolean actual, boolean expected, String message) { + Assert.assertEquals(message, expected, actual); + } + + public static void assertTrue(boolean condition) { + Assert.assertTrue(condition); + } + + public static void assertTrue(boolean condition, String message) { + Assert.assertTrue(message, condition); + } + + //////////////////////////// non-strict assertions //////////////////////////// + + public static void assertGte(int actual, int expected) { + assertGte(actual, expected, null); + } + + public static void assertGte(int actual, int expected, String message) { + assertGte((long) actual, expected, message); + } + + public static void assertGte(long actual, long expected, String message) { + assertTrue(actual >= expected); + if (!(actual == expected)) { + String skipMessage = actual + " should be equals to " + expected; + if (Objects.nonNull(message)) { + skipMessage += ", message: " + message; + } + LOGGER.warn(skipMessage); + Assume.assumeTrue(skipMessage, actual == expected); + } + } + + public void check_count_non_strict(int expect_count, String sql, String msg) + throws IoTDBConnectionException, StatementExecutionException { + assertGte(getCount(session_dest, sql), expect_count, "Query count: " + msg); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/auto_create_db/IoTDBDefaultPullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/auto_create_db/IoTDBDefaultPullConsumerDataSetIT.java new file mode 100644 index 000000000000..b9e1fb0f1dd5 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/auto_create_db/IoTDBDefaultPullConsumerDataSetIT.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.auto_create_db; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBDefaultPullConsumerDataSetIT extends AbstractSubscriptionRegressionIT { + public static SubscriptionPullConsumer consumer; + private int deviceCount = 3; + private static final String databasePrefix = "root.DefaultPullConsumerDataSet"; + private static String topicName = "topic_autodb_DefaultPullConsumerDataSet"; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createTopic_s(topicName, null, null, null, false); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + for (int i = 0; i < deviceCount; i++) { + session_src.executeNonQueryStatement("create database " + databasePrefix + i); + session_dest.executeNonQueryStatement("create database " + databasePrefix + i); + } + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + for (int i = 0; i < deviceCount; i++) { + session_src.executeNonQueryStatement("drop database " + databasePrefix + i); + session_dest.executeNonQueryStatement("drop database " + databasePrefix + i); + } + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20 + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + List devices = new ArrayList<>(deviceCount); + for (int i = 0; i < deviceCount; i++) { + devices.add(databasePrefix + i + ".d_0"); + } + consumer = create_pull_consumer("pull_auto_create_db", "default_pattern_dataset", false, null); + for (int i = 0; i < deviceCount; i++) { + // Write data before subscribing + insert_data(1706659200000L, devices.get(i)); // 2024-01-31 08:00:00+08:00 + } + // Subscribe + consumer.subscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after subscription"); + for (int i = 0; i < deviceCount; i++) { + insert_data(System.currentTimeMillis(), devices.get(i)); + } + String sql = "select count(s_0) from " + databasePrefix + "0.d_0"; + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + // Consumption data + consume_data(consumer, session_dest); + for (int i = 0; i < deviceCount; i++) { + check_count(10, "select count(s_0) from " + devices.get(i), i + ":Consumption Data:s_0"); + } + // Unsubscribe + consumer.unsubscribe(topicName); + // Unsubscribe and then write data + for (int i = 0; i < deviceCount; i++) { + insert_data(1707782400000L, devices.get(i)); // 2024-02-13 08:00:00+08:00 + } + consumer.subscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after re-subscribing"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + consume_data(consumer, session_dest); + for (int i = 0; i < deviceCount; i++) { + check_count(15, "select count(s_0) from " + devices.get(i), i + ":consume data again:s_0"); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/auto_create_db/IoTDBDefaultTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/auto_create_db/IoTDBDefaultTsfilePushConsumerIT.java new file mode 100644 index 000000000000..2b55879f2d98 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/auto_create_db/IoTDBDefaultTsfilePushConsumerIT.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.auto_create_db; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * pattern: root.** + * TsFile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBDefaultTsfilePushConsumerIT extends AbstractSubscriptionRegressionIT { + private SubscriptionPushConsumer consumer; + private int deviceCount = 3; + private static final String databasePrefix = "root.DefaultTsfilePushConsumer"; + private static String topicName = "topicDefaultTsfilePushConsumer"; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createTopic_s(topicName, null, null, null, true); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + for (int i = 0; i < deviceCount; i++) { + session_src.executeNonQueryStatement("create database " + databasePrefix + i); + } + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + for (int i = 0; i < deviceCount; i++) { + session_src.executeNonQueryStatement("drop database " + databasePrefix + i); + } + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20 + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + List devices = new ArrayList<>(deviceCount); + List paths = new ArrayList<>(deviceCount); + for (int i = 0; i < deviceCount; i++) { + devices.add(databasePrefix + i + ".d_0"); + paths.add(new Path(devices.get(i), "s_0", true)); + } + System.out.println("### Before Subscription Write Data ###"); + for (int i = 0; i < deviceCount; i++) { + insert_data(1706659200000L, devices.get(i)); // 2024-01-31 08:00:00+08:00 + } + session_src.executeNonQueryStatement("flush;"); + final AtomicInteger onReceiveCount = new AtomicInteger(0); + List rowCounts = new ArrayList<>(deviceCount); + for (int i = 0; i < deviceCount; i++) { + rowCounts.add(new AtomicInteger(0)); + } + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("default_pattern_TsFile_consumer") + .consumerGroupId("push_auto_create_db") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + onReceiveCount.incrementAndGet(); + System.out.println( + FORMAT.format(new Date()) + " ######## onReceived: " + onReceiveCount.get()); + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + for (int i = 0; i < deviceCount; i++) { + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(paths.get(i)), null)); + while (dataset.hasNext()) { + rowCounts.get(i).addAndGet(1); + RowRecord next = dataset.next(); + // System.out.println(format.format(new + // Date())+" "+next.getTimestamp()+","+next.getFields()); + } + System.out.println( + FORMAT.format(new Date()) + + " rowCounts_" + + i + + ":" + + rowCounts.get(i).get()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions(topicName).forEach(System.out::println); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "After subscribing: show subscriptions"); + for (int i = 0; i < deviceCount; i++) { + insert_data(System.currentTimeMillis(), devices.get(i)); + System.out.println( + FORMAT.format(new Date()) + + " src " + + i + + ":" + + getCount(session_src, "select count(s_0) from " + devices.get(i))); + } + session_src.executeNonQueryStatement("flush;"); + AWAIT.untilAsserted( + () -> { + for (int i = 0; i < deviceCount; i++) { + assertEquals(rowCounts.get(i).get(), 10, devices.get(i) + ".s_0"); + } + }); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + System.out.println("### Subscribe and write data ###"); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "After subscribing again: show subscriptions"); + for (int i = 0; i < deviceCount; i++) { + insert_data(1707782400000L, devices.get(i)); // 2024-02-13 08:00:00+08:00 + System.out.println( + FORMAT.format(new Date()) + + " src " + + i + + ":" + + getCount(session_src, "select count(s_0) from " + devices.get(i))); + } + session_src.executeNonQueryStatement("flush;"); + + // Unsubscribe, then it will consume all again. + AWAIT.untilAsserted( + () -> { + for (int i = 0; i < deviceCount; i++) { + assertEquals(rowCounts.get(i).get(), 25, devices.get(i) + ".s_0"); + } + }); + System.out.println(FORMAT.format(new Date()) + " onReceived: " + onReceiveCount.get()); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/auto_create_db/IoTDBRootDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/auto_create_db/IoTDBRootDatasetPushConsumerIT.java new file mode 100644 index 000000000000..caacf6757ca8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/auto_create_db/IoTDBRootDatasetPushConsumerIT.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.auto_create_db; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * pattern: root + * DataSet + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBRootDatasetPushConsumerIT extends AbstractSubscriptionRegressionIT { + private String pattern = "root.**"; + public static SubscriptionPushConsumer consumer; + private int deviceCount = 3; + private static final String databasePrefix = "root.RootDatasetPushConsumer"; + private static final String database2 = "root.RootDatasetPushConsumer2.test"; + private static String topicName = "topicAutoCreateDB_RootDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createTopic_s(topicName, pattern, null, null, false); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(databasePrefix + "*.*"); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20 + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + List devices = new ArrayList<>(deviceCount); + for (int i = 0; i < deviceCount - 1; i++) { + devices.add(databasePrefix + i + ".d_0"); + } + devices.add(database2 + ".d_2"); + for (int i = 0; i < deviceCount; i++) { + // Write data before subscribing + insert_data(1706659200000L, devices.get(i)); // 2024-01-31 08:00:00+08:00 + } + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("root_dataset_consumer") + .consumerGroupId("push_auto_create_db") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions(topicName).size(), 1, "subscribe:show subscriptions"); + for (int i = 0; i < deviceCount; i++) { + insert_data(System.currentTimeMillis(), devices.get(i)); + } + String sql = "select count(s_0) from " + databasePrefix + "0.d_0"; + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + AWAIT.untilAsserted( + () -> { + check_count(10, "select count(s_0) from " + devices.get(0), "0:consume data:s_0"); + for (int i = 1; i < deviceCount; i++) { + check_count(10, "select count(s_0) from " + devices.get(i), i + ":consume data:s_0"); + } + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + for (int i = 0; i < deviceCount; i++) { + insert_data(1707782400000L, devices.get(i)); // 2024-02-13 08:00:00+08:00 + } + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "After subscribing again: show subscriptions"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count( + 15, + "select count(s_0) from " + devices.get(0), + "0:After subscribing again:consume data:s_0"); + for (int i = 1; i < deviceCount; i++) { + check_count( + 15, + "select count(s_0) from " + devices.get(i), + i + ":After subscribing again:consume data:s_0"); + } + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/auto_create_db/IoTDBRootPullConsumeTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/auto_create_db/IoTDBRootPullConsumeTsfileIT.java new file mode 100644 index 000000000000..ad7363013570 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/auto_create_db/IoTDBRootPullConsumeTsfileIT.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.auto_create_db; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer + * pattern: db + * Tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBRootPullConsumeTsfileIT extends AbstractSubscriptionRegressionIT { + private static final String pattern = "root.**"; + private static final String device = "root.auto_create_db.RootPullConsumeTsfile.d_0"; + private static final String device2 = "root.RootPullConsumeTsfile.d_1"; + public static SubscriptionPullConsumer consumer; + private static String topicName = "topicAutoCreateDB_RootPullConsumeTsfile"; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createTopic_s(topicName, pattern, null, null, true); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + session_src.executeNonQueryStatement("create database root.auto_create_db"); + session_src.executeNonQueryStatement("create database root.RootPullConsumeTsfile"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + session_src.executeNonQueryStatement("drop database root.auto_create_db"); + session_src.executeNonQueryStatement("drop database root.RootPullConsumeTsfile"); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20 + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionPullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("root_tsfile") + .consumerGroupId("pull_auto_create_db") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + consumer.subscribe(topicName); + subs.getSubscriptions(topicName).forEach(System.out::println); + assertEquals(subs.getSubscriptions(topicName).size(), 1, "subscribe:show subscriptions"); + insert_data(System.currentTimeMillis(), device); + insert_data(System.currentTimeMillis(), device2); + List devices = new ArrayList<>(2); + devices.add(device); + devices.add(device2); + List results = consume_tsfile(consumer, devices); + assertEquals(results.get(0), 10); + assertEquals(results.get(1), 10); + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions(topicName).size(), 0, "unsubscribe:show subscriptions"); + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "subscribe again:show subscriptions"); + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + results = consume_tsfile(consumer, devices); + assertEquals( + results.get(0), + 15, + "Unsubscribing and then re-subscribing will not retain progress. Full synchronization."); + assertEquals( + results.get(1), + 15, + "Unsubscribing and then re-subscribing will not retain progress. Full synchronization."); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/mix/IoTDBPushConsumerPullConsumerWith1TopicShareProcessMixIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/mix/IoTDBPushConsumerPullConsumerWith1TopicShareProcessMixIT.java new file mode 100644 index 000000000000..be44aa833602 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/mix/IoTDBPushConsumerPullConsumerWith1TopicShareProcessMixIT.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.mix; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * pattern: db + * Dataset + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBPushConsumerPullConsumerWith1TopicShareProcessMixIT + extends AbstractSubscriptionRegressionIT { + private static String topicName = "`1-group.1-consumer.db`"; + private static List schemaList = new ArrayList<>(); + private final String database = "root.PushConsumerPullConsumerWith1TopicShareProcessMix"; + private final String device = database + ".d_0"; + private final String pattern = database + ".**"; + private SubscriptionPushConsumer consumer; + private SubscriptionPullConsumer consumer2; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createTopic_s(topicName, pattern, null, null, false); + createDB(database); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest2.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest2.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + consumer2.close(); + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + Thread thread = + new Thread( + () -> { + long timestamp = 1706659200000L; // 2024-01-31 08:00:00+08:00 + for (int i = 0; i < 100; i++) { + try { + insert_data(timestamp); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } + timestamp += 20000; + } + }); + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("dataset_push_consumer_1") + .consumerGroupId("db_pull_push_mix") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + consumer.subscribe(topicName); + + consumer2 = + new SubscriptionPullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("dataset_pull_consumer_2") + .consumerGroupId("db_pull_push_mix") + .buildPullConsumer(); + consumer2.open(); + consumer2.subscribe(topicName); + + thread.start(); + thread.join(); + consume_data(consumer2, session_dest2); + System.out.println("After subscribing:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions"); + + Thread.sleep(3000); + // The first 5 entries may have duplicate data + String sql = "select count(s_0) from " + device; + System.out.println("src push consumer: " + getCount(session_src, sql)); + System.out.println("dest push consumer: " + getCount(session_dest, sql)); + System.out.println("dest2 pull consumer: " + getCount(session_dest2, sql)); + AWAIT.untilAsserted( + () -> { + assertEquals( + getCount(session_dest, sql) + getCount(session_dest2, sql), + getCount(session_src, sql), + "share process"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/autocommit/IoTDBTestAutoCommitFalseDataSetPullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/autocommit/IoTDBTestAutoCommitFalseDataSetPullConsumerIT.java new file mode 100644 index 000000000000..800bc5ef69bd --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/autocommit/IoTDBTestAutoCommitFalseDataSetPullConsumerIT.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.autocommit; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Date; +import java.util.Iterator; +import java.util.List; + +/*** + * If autoCommit is set to false, not using commit to submit consumption progress will lead to repeated consumption; + * If set to true, it will not cause duplicate consumption (but considering the underlying implementation of the data subscription pipe is at-least-once semantics, there may also be cases of duplicate data) + * autoCommit itself is just an auxiliary feature for at least once semantics. The commit itself is only related to restart recovery. In normal situations without bugs (pure log/batch), whether to commit or not, it will still be sent once. + * Now the data subscription so-called progress information and the Pipe's progress information is a concept. + * For the semantics related to data subscription commit, here are some supplements: * + * In the implementation of data subscription, after a batch of messages is polled by the consumer, if they are not committed within a certain period of time (hardcoded value), the consumers under this subscription can repoll this batch of messages. This mechanism is designed to prevent messages from accumulating on the server side when the consumer does not explicitly commit with autoCommit set to false. However, this mechanism is not part of the functional definition (not exposed to the user), but only to handle some exceptional cases on the client side, such as when a consumer crashes after polling a batch of messages, or forgets to commit. + * Tests can be more focused on situations where autoCommit is true. In this case, if not explicitly committed, it is expected that all messages successfully polled by the consumer should be committed before the consumer is closed (reflected in the Pipe as no accumulated resources). + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTestAutoCommitFalseDataSetPullConsumerIT + extends AbstractSubscriptionRegressionIT { + private static final String database = "root.TestAutoCommitFalseDataSetPullConsumer"; + private static final String device = database + ".d_0"; + private static final String topicName = "Topic_auto_commit_false"; + private String pattern = device + ".**"; + private static SubscriptionPullConsumer consumer; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest2.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest2.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + private void consume_data_noCommit(SubscriptionPullConsumer consumer, Session session) + throws InterruptedException, + TException, + IOException, + StatementExecutionException, + IoTDBConnectionException { + while (true) { + Thread.sleep(1000); + List messages = consumer.poll(Duration.ofMillis(10000)); + if (messages.isEmpty()) { + break; + } + for (final SubscriptionMessage message : messages) { + for (final Iterator it = message.getSessionDataSetsHandler().tabletIterator(); + it.hasNext(); ) { + final Tablet tablet = it.next(); + session.insertTablet(tablet); + System.out.println( + FORMAT.format(new Date()) + " consume data no commit:" + tablet.rowSize); + } + } + } + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_commit", "Auto_commit_FALSE", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + assertEquals(subs.getSubscriptions().size(), 0, "Before subscription show subscriptions"); + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + String sql1 = "select count(s_0) from " + device; + String sql2 = "select count(s_1) from " + device; + consume_data_noCommit(consumer, session_dest); + System.out.println(FORMAT.format(new Date()) + " src sql1: " + getCount(session_src, sql1)); + System.out.println("dest sql1: " + getCount(session_dest, sql1)); + System.out.println("dest2 sql1: " + getCount(session_dest2, sql1)); + check_count(4, sql1, "dest consume subscription before data:s_0"); + check_count(4, sql2, "dest consume subscription before data:s_1"); + check_count2(0, sql2, "dest2 consumption subscription previous data: s_1"); + + // Subscribe and then write data + insert_data(System.currentTimeMillis()); + consume_data_noCommit(consumer, session_dest2); + System.out.println("src sql1: " + getCount(session_src, sql1)); + System.out.println("dest sql1: " + getCount(session_dest, sql1)); + System.out.println("dest2 sql1: " + getCount(session_dest2, sql1)); + check_count(4, sql1, "dest consume subscription data 2:s_0"); + check_count2(4, sql1, "dest2 consumption subscription data 2:s_0"); + check_count2(4, sql2, "dest2 consumption subscription data 2:s_1"); + + // insert_data(1706659300000L); //2024-01-31 08:00:00+08:00 + // Will consume again + consume_data_noCommit(consumer, session_dest); + System.out.println("src sql1: " + getCount(session_src, sql1)); + System.out.println("dest sql1: " + getCount(session_dest, sql1)); + System.out.println("dest2 sql1: " + getCount(session_dest2, sql1)); + check_count(4, sql1, "dest consumption subscription before data3:s_0"); + check_count(4, sql2, "dest consume subscription before data3:s_1"); + check_count2(4, sql2, "dest2 consumption subscription before count 3 data:s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/autocommit/IoTDBTestAutoCommitTrueDataSetPullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/autocommit/IoTDBTestAutoCommitTrueDataSetPullConsumerIT.java new file mode 100644 index 000000000000..d5d72d34137e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/autocommit/IoTDBTestAutoCommitTrueDataSetPullConsumerIT.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.autocommit; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Date; +import java.util.Iterator; +import java.util.List; + +/*** + * PullConsumer DataSet + * pattern: device + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTestAutoCommitTrueDataSetPullConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.TestAutoCommitTrueDataSetPullConsumer"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_auto_commit_true"; + private String pattern = device + ".**"; + private static SubscriptionPullConsumer consumer; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest2.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest2.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } finally { + subs.dropTopic(topicName); + dropDB(database); + } + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + private void consume_data_noCommit(SubscriptionPullConsumer consumer, Session session) + throws InterruptedException, + TException, + IOException, + StatementExecutionException, + IoTDBConnectionException { + while (true) { + Thread.sleep(1000); + List messages = consumer.poll(Duration.ofMillis(10000)); + if (messages.isEmpty()) { + break; + } + for (final SubscriptionMessage message : messages) { + for (final Iterator it = message.getSessionDataSetsHandler().tabletIterator(); + it.hasNext(); ) { + final Tablet tablet = it.next(); + session.insertTablet(tablet); + System.out.println( + FORMAT.format(new Date()) + " consume data no commit:" + tablet.rowSize); + } + } + } + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = "select count(s_0) from " + device; + consumer = create_pull_consumer("pull_commit", "Auto_commit_true", true, 1000L); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + assertEquals(subs.getSubscriptions().size(), 0, "Before subscription show subscriptions"); + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + System.out.println("First consumption"); + consume_data_noCommit(consumer, session_dest); + System.out.println(FORMAT.format(new Date()) + ", " + getCount(session_src, sql)); + check_count(4, sql, "Consumption subscription previous data: s_0"); + check_count(4, "select count(s_1) from " + device, "Consumption subscription before data: s_1"); + // Subscribe and then write data + insert_data(System.currentTimeMillis()); + consume_data_noCommit(consumer, session_dest2); + System.out.println("second consumption"); + check_count2(4, "select count(s_0) from " + device, "Consumption subscription data 2: s_0"); + check_count2(4, "select count(s_1) from " + device, "Consumption subscription data 2:s_1"); + // Consumed data will not be consumed again + consume_data_noCommit(consumer, session_dest); + System.out.println("Third consumption"); + check_count(4, "select count(s_0) from " + device, "Consumption subscription before data: s_0"); + check_count( + 4, "select count(s_1) from " + device, "Consumption subscription previous data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/format/IoTDBDBDataSetPullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/format/IoTDBDBDataSetPullConsumerIT.java new file mode 100644 index 000000000000..6610c1b129a4 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/format/IoTDBDBDataSetPullConsumerIT.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.format; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer DataSet + * pattern: db + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBDBDataSetPullConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.DBDataSetPullConsumer"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_format_pull_dataset"; + private static final String pattern = database + ".**"; + private static SubscriptionPullConsumer consumer; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionPullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts") + .consumerGroupId("pull_mode_dataset") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + // Consumption data + consume_data(consumer, session_dest); + + String sql = "select count(s_0) from " + device; + System.out.println("src " + getCount(session_src, sql)); + System.out.println("dest " + getCount(session_dest, sql)); + check_count(8, sql, "consume after subscription"); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "After cancellation, show subscriptions"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + consume_data(consumer, session_dest2); + System.out.println("src " + getCount(session_src, sql)); + System.out.println("dest " + getCount(session_dest, sql)); + System.out.println("dest2 " + getCount(session_dest2, sql)); + check_count2( + 12, sql, "Unsubscribe and resubscribe, progress is not retained. Full synchronization."); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/format/IoTDBDBTsfilePullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/format/IoTDBDBTsfilePullConsumerIT.java new file mode 100644 index 000000000000..8a3da6afd6e4 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/format/IoTDBDBTsfilePullConsumerIT.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.format; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer Tsfile + * pattern: db + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBDBTsfilePullConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.DBTsfilePullConsumer"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_format_pull_tsfile"; + private static final String pattern = database + ".**"; + private static SubscriptionPullConsumer consumer; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createTopic_s(topicName, pattern, null, null, true); + createDB(database); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionPullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("db") + .consumerGroupId("mode_pull_tsfile") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + // insert_data(1706659200000L); //2024-01-31 08:00:00+08:00 + insert_data(System.currentTimeMillis()); + // Consumption data + List results = consume_tsfile_withFileCount(consumer, device); + assertEquals(results.get(0), 10); + assertEquals(results.get(1), 2, "number of received files"); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "Show subscriptions after unsubscription"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + results = consume_tsfile_withFileCount(consumer, device); + assertEquals( + results.get(0), + 15, + "Unsubscribe and resubscribe, progress is not retained. Full synchronization."); + assertEquals( + results.get(1), + 3, + "Number of received files: After unsubscribing and resubscribing, progress is not retained. Full synchronization."); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBAllTsDatasetPullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBAllTsDatasetPullConsumerIT.java new file mode 100644 index 000000000000..07405a8970fc --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBAllTsDatasetPullConsumerIT.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * pull consumer + * pattern: ts + * accurate time + * format: dataset + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBAllTsDatasetPullConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.AllTsDatasetPullConsumer"; + private static final String database2 = "root.test.AllTsDatasetPullConsumer"; + private static final String device = database + ".d_0"; + private String device2 = database + ".d_1"; + private static final String pattern = device + ".s_0"; + private static final String topicName = "topic_loose_range_all_pull_dataset"; + private List schemaList = new ArrayList<>(); + private SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-03-31T23:59:59+08:00", + false, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_ALL_VALUE); + session_src.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-03-31T23:59:59+08:00"; + consumer = create_pull_consumer("ts_accurate_dataset_pull", "loose_range_all", false, null); + // Write data before subscribing + insert_data(1704038396000L, device); // 2023-12-31 23:59:56+08:00 + insert_data(1704038396000L, device2); // 2023-12-31 23:59:56+08:00 + System.out.println("src filter:" + getCount(session_src, sql)); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + // Before consumption subscription data + consume_data(consumer, session_dest); + check_count_non_strict( + 3, "select count(s_0) from " + device, "Start time boundary data: s_0 " + device); + check_count(0, "select count(s_1) from " + device, "Start time boundary data: s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "Start time boundary data: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "Start time boundary data: s_1 " + device2); + + insert_data(System.currentTimeMillis(), device); // now + insert_data(System.currentTimeMillis(), device2); // now + System.out.println("src filter:" + getCount(session_src, sql)); + consume_data(consumer, session_dest); + check_count_non_strict( + 3, "select count(s_0) from " + device, "Write some real-time data later:s_0 " + device); + check_count( + 0, "select count(s_1) from " + device, "Write some real-time data later: s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "not in range: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "not in range: s_1 " + device2); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + consume_data(consumer, session_dest); + System.out.println("src filter:" + getCount(session_src, sql)); + check_count_non_strict( + 8, "select count(s_0) from " + device, "data within the time range: s_0 " + device); + check_count(0, "select count(s_1) from " + device, "data within the time range: s_1 " + device); + check_count( + 0, "select count(s_0) from " + device2, "data within the time range: s_0 " + device2); + check_count( + 0, "select count(s_1) from " + device2, "data within the time range: s_1 " + device2); + + insert_data(1711814398000L, device); // 2024-03-30 23:59:58+08:00 + insert_data(1711814398000L, device2); // 2024-03-30 23:59:58+08:00 + System.out.println("src filter:" + getCount(session_src, sql)); + consume_data(consumer, session_dest); + check_count_non_strict( + 13, "select count(s_0) from " + device, "End time limit data: s_0 " + device); + check_count(0, "select count(s_1) from " + device, "End time limit data: s_1" + device); + check_count(0, "select count(s_0) from " + device2, "End time limit data: s_0" + device2); + check_count(0, "select count(s_1) from " + device2, "End time limit data: s_1" + device2); + + insert_data(1711900798000L, device); // 2024-03-31 23:59:58+08:00 + insert_data(1711900798000L, device2); // 2024-03-31 23:59:58+08:00 + System.out.println("src filter:" + getCount(session_src, sql)); + consume_data(consumer, session_dest); + check_count_non_strict( + 14, "select count(s_0) from " + device, "End time limit data 2:s_0 " + device); + check_count(0, "select count(s_1) from " + device, "End time limit data 2:s_1" + device); + check_count(0, "select count(s_0) from " + device2, "End time limit data 2: s_0" + device2); + check_count(0, "select count(s_1) from " + device2, "End time limit data 2: s_1" + device2); + + consumer.unsubscribe(topicName); + consumer.subscribe(topicName); + consume_data(consumer, session_dest); + check_count_non_strict( + 14, "select count(s_0) from " + device, "End time limit data 2:s_0 " + device); + check_count(0, "select count(s_1) from " + device, "End time limit data 2:s_1" + device); + check_count(0, "select count(s_0) from " + device2, "End time limit data 2: s_0" + device2); + check_count(0, "select count(s_1) from " + device2, "End time limit data 2: s_1" + device2); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBAllTsTsfilePullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBAllTsTsfilePullConsumerIT.java new file mode 100644 index 000000000000..3787c28345f1 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBAllTsTsfilePullConsumerIT.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +/*** + * PullConsumer + * pattern: ts + * format: tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBAllTsTsfilePullConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.AllTsTsfilePullConsumer"; + private static final String database2 = "root.AllTsTsfilePullConsumer"; + private static final String topicName = "TopicAllTsTsfilePullConsumer"; + private static final String device = database + ".d_0"; + private static final String pattern = device + ".s_0"; + private static final String device2 = database + ".d_1"; + private static SubscriptionPullConsumer consumer; + private List schemaList = new ArrayList<>(); + private long nowTimestamp; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + nowTimestamp = System.currentTimeMillis(); + createTopic_s( + topicName, + pattern, + null, + String.valueOf(nowTimestamp), + true, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_ALL_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + device2 + "(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + + consumer = + new SubscriptionPullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts_pattern_history_tsfile_pull") + .consumerGroupId("loose_range_all") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + insert_data(nowTimestamp - 4000, device); + insert_data(nowTimestamp - 4000, device2); + System.out.println( + FORMAT.format(new Date()) + + " src filter:" + + getCount( + session_src, "select count(s_0) from " + device + " where time <=" + nowTimestamp)); + + // Consumption data + List paths = new ArrayList<>(3); + paths.add(device); + paths.add(device2); + paths.add(database2 + ".d_2"); + List rowCountList = consume_tsfile(consumer, paths); + assertGte(rowCountList.get(0), 8); + assertEquals(rowCountList.get(1), 0); + assertEquals(rowCountList.get(2), 0); + + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "Show subscriptions after cancellation"); + + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + + // Consumption data: Progress is not retained after unsubscribing and resubscribing. Full + // synchronization. + rowCountList = consume_tsfile(consumer, paths); + assertGte( + rowCountList.get(0), + 13, + "Unsubscribe and then resubscribe, progress is not retained. Full synchronization."); + assertEquals(rowCountList.get(1), 0, "Unsubscribe and then resubscribe," + device2); + assertEquals(rowCountList.get(2), 0, "Unsubscribe and then resubscribe," + database2 + ".d_2"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBAllTsfilePullConsumerSnapshotIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBAllTsfilePullConsumerSnapshotIT.java new file mode 100644 index 000000000000..95d0bce419da --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBAllTsfilePullConsumerSnapshotIT.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +/*** + * PullConsumer + * pattern: ts + * format: tsfile + * loose-range: all + * mode: snapshot + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBAllTsfilePullConsumerSnapshotIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.AllTsfilePullConsumerSnapshot"; + private static final String database2 = "root.test.AllTsfilePullConsumerSnapshot"; + private static final String device = database + ".d_0"; + private static final String topicName = "TopicAllTsfilePullConsumerSnapshot"; + private static final String pattern = device + ".s_0"; + private static final String device2 = database + ".d_1"; + private static SubscriptionPullConsumer consumer; + private List schemaList = new ArrayList<>(); + private long nowTimestamp; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + nowTimestamp = System.currentTimeMillis(); + createTopic_s( + topicName, + pattern, + null, + String.valueOf(nowTimestamp), + true, + TopicConstant.MODE_SNAPSHOT_VALUE, + TopicConstant.LOOSE_RANGE_ALL_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + device2 + "(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + + consumer = + new SubscriptionPullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("snapshot_ts_pattern_tsfile_pull") + .consumerGroupId("loose_range_all") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + insert_data(nowTimestamp - 4000, device); + insert_data(nowTimestamp - 4000, device2); + System.out.println( + FORMAT.format(new Date()) + + " src filter:" + + getCount( + session_src, "select count(s_0) from " + device + " where time <=" + nowTimestamp)); + + // Consumption data + List paths = new ArrayList<>(3); + paths.add(device); + paths.add(device2); + paths.add(database2 + ".d_2"); + List rowCountList = consume_tsfile(consumer, paths); + // Subscribe and write without consuming + assertEquals(rowCountList.get(0), 5, "Write without consume after subscription"); + assertEquals(rowCountList.get(1), 0); + assertEquals(rowCountList.get(2), 0); + + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "After cancellation, show subscriptions"); + + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + rowCountList = consume_tsfile(consumer, paths); + assertEquals( + rowCountList.get(0), + 10, + "Unsubscribe and then resubscribe, progress is not retained. Full synchronization."); + assertEquals(rowCountList.get(1), 0, "Unsubscribe and then resubscribe," + device2); + assertEquals(rowCountList.get(2), 0, "Unsubscribe and then resubscribe," + database2 + ".d_2"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBPathDeviceDataSetPullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBPathDeviceDataSetPullConsumerIT.java new file mode 100644 index 000000000000..6e9c3e5158b5 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBPathDeviceDataSetPullConsumerIT.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * pull consumer + * format: dataset + * loose-range: path + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBPathDeviceDataSetPullConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.PathDeviceDataSetPullConsumer"; + private static final String database2 = "root.PathDeviceDataSetPullConsumer"; + private static final String topicName = "TopicPathDeviceDataSetPullConsumer"; + private static final String device = database + ".d_0"; + private String pattern = device + ".**"; + private String device2 = database + ".d_1"; + private static SubscriptionPullConsumer consumer; + private List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + null, + "now", + false, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_PATH_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 float;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 float;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = "select count(s_0) from " + device; + consumer = create_pull_consumer("device_pattern_dataset_pull", "loose_range_path", false, null); + // Write data before subscribing + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis() - 30000L, device); + insert_data(System.currentTimeMillis() - 30000L, device2); + // Consumption data + consume_data(consumer, session_dest); + System.out.println("src: " + getCount(session_src, sql)); + check_count(10, sql, "Consumption data: s_0 " + device); + check_count(10, "select count(s_1) from " + device, "Consumption data: s_1"); + check_count(0, "select count(s_0) from " + device2, "Consumption data:d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + insert_data(System.currentTimeMillis(), device); + insert_data(System.currentTimeMillis(), device2); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + System.out.println("src: " + getCount(session_src, sql)); + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(15, "select count(s_0) from " + device, "Consume data again:" + pattern); + check_count(15, "select count(s_1) from " + device, "Consumption data: s_1"); + check_count(0, "select count(s_0) from " + device2, "Consumption data:d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBPathDeviceTsfilePullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBPathDeviceTsfilePullConsumerIT.java new file mode 100644 index 000000000000..25928e6ff7d7 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBPathDeviceTsfilePullConsumerIT.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer + * pattern: device + * format: tsfile + * loose-range: time + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBPathDeviceTsfilePullConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.PathDeviceTsfilePullConsumer"; + private static final String database2 = "root.PathDeviceTsfilePullConsumer"; + private static final String topicName = "TopicPathDeviceTsfilePullConsumer"; + private static final String device = database + ".d_0"; + private static final String pattern = device + ".**"; + private static final String device2 = database + ".d_1"; + private static SubscriptionPullConsumer consumer; + private List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + null, + null, + true, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_PATH_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionPullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("pull_path_device_tsfile") + .consumerGroupId("loose_range_path") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis(), device); + insert_data(System.currentTimeMillis(), device2); + // Consumption data + List devices = new ArrayList<>(3); + devices.add(device); + devices.add(device2); + devices.add(database2 + ".d_2"); + List results = consume_tsfile(consumer, devices); + + assertEquals(results.get(0), 10); + assertEquals(results.get(1), 0); + assertEquals(results.get(2), 0); + assertEquals(results.get(3), 2, "number of received files"); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "show subscriptions after cancellation"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + results = consume_tsfile(consumer, devices); + assertEquals( + results.get(0), + 15, + "Unsubscribe and resubscribe, progress is not retained. Full synchronization."); + assertEquals(results.get(1), 0, "Unsubscribe and then subscribe again," + device2); + assertEquals(results.get(2), 0, "Subscribe again after cancellation," + database2 + ".d_2"); + assertEquals(results.get(3), 3, "Number of received files: resubscribe after unsubscribe"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBTimeTsDatasetPullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBTimeTsDatasetPullConsumerIT.java new file mode 100644 index 000000000000..111be2cf9723 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBTimeTsDatasetPullConsumerIT.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * pull consumer + * pattern: ts + * loose-range: time + * accurate time range + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTimeTsDatasetPullConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.TimeTsDatasetPullConsumer"; + private static final String database2 = "root.TimeTsDatasetPullConsumer"; + private static final String topicName = "TopicTimeTsDatasetPullConsumer"; + private String device = database + ".d_0"; + private String device2 = database + ".d_1"; + private String device3 = database2 + ".d_2"; + private String pattern = device + ".s_0"; + private List schemaList = new ArrayList<>(); + private SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-03-31T23:59:59+08:00", + false, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_TIME_VALUE); + session_src.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + device3 + ".s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + device3 + ".s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + device3 + ".s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + device3 + ".s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + device3 + "(time,s_0,s_1)values(1000,132,4567.89);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-03-31T23:59:59+08:00"; + consumer = + create_pull_consumer("pull_ts_pattern_accurate_dataset", "loose_range_time", false, null); + + // Write data before subscribing + insert_data(1704038396000L, device); // 2023-12-31 23:59:56+08:00 + insert_data(1704038396000L, device2); // 2023-12-31 23:59:56+08:00 + System.out.println("src filter:" + getCount(session_src, sql)); + + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + // Before consumption subscription data + consume_data(consumer, session_dest); + check_count_non_strict(3, sql, "Start time boundary data: s_0 " + device); + check_count(0, "select count(s_1) from " + device, "Start time boundary data: s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "Start time boundary data: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "Start time limit data: s_1 " + device2); + + insert_data(System.currentTimeMillis(), device); // not in range + insert_data(System.currentTimeMillis(), device2); // not in range + System.out.println("src filter:" + getCount(session_src, sql)); + + consume_data(consumer, session_dest); + check_count_non_strict(3, sql, "After writing some real-time data: s_0 " + device); + check_count( + 0, "select count(s_1) from " + device, "Write some real-time data later:s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "not in range: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "not in range: s_1 " + device2); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + consume_data(consumer, session_dest); + System.out.println("src filter:" + getCount(session_src, sql)); + + check_count_non_strict(8, sql, "Data within time range: s_0 " + device); + check_count(0, "select count(s_1) from " + device, "data within the time range: s_1 " + device); + check_count( + 0, "select count(s_0) from " + device2, "Data within the time range: s_0 " + device2); + check_count( + 0, "select count(s_1) from " + device2, "Data within the time range: s_1 " + device2); + + insert_data(1711814398000L, device); // 2024-03-30 23:59:58+08:00 + insert_data(1711814398000L, device2); // 2024-03-30 23:59:58+08:00 + System.out.println("src filter:" + getCount(session_src, sql)); + + consume_data(consumer, session_dest); + check_count_non_strict(13, sql, "End time limit data: s_0 " + device); + check_count(0, "select count(s_1) from " + device, "End time limit data: s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "End time limit data: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "End time limit data: s_1 " + device2); + + insert_data(1711900798000L, device); // 2024-03-31 23:59:58+08:00 + insert_data(1711900798000L, device2); // 2024-03-31 23:59:58+08:00 + System.out.println("src filter:" + getCount(session_src, sql)); + + consume_data(consumer, session_dest); + check_count_non_strict( + 14, "select count(s_0) from " + device, "End time limit data 2:s_0 " + device); + check_count(0, "select count(s_1) from " + device, "End time limit data 2:s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "End time limit data 2: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "End time limit data 2: s_1 " + device2); + + consumer.unsubscribe(topicName); + consumer.subscribe(topicName); + consume_data(consumer, session_dest); + check_count_non_strict( + 14, "select count(s_0) from " + device, "End time limit data 2:s_0 " + device); + check_count(0, "select count(s_1) from " + device, "End time limit data 2:s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "End time limit data 2: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "End time limit data 2: s_1 " + device2); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBTimeTsTsfilePullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBTimeTsTsfilePullConsumerIT.java new file mode 100644 index 000000000000..6695fde116db --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/loose_range/IoTDBTimeTsTsfilePullConsumerIT.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer + * pattern: ts + * format: tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTimeTsTsfilePullConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.TimeTsTsfilePullConsumer"; + private static final String database2 = "root.TimeTsTsfilePullConsumer"; + private static final String topicName = "TopicTimeTsTsfilePullConsumer"; + private static final String device = database + ".d_0"; + private static final String pattern = device + ".s_0"; + private static final String device2 = database + ".d_1"; + private static final String device3 = database2 + ".d_2"; + private static SubscriptionPullConsumer consumer; + private static List schemaList = new ArrayList<>(); + private static List rowCountList; + private long nowTimestamp; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + nowTimestamp = System.currentTimeMillis(); + createTopic_s( + topicName, + pattern, + null, + String.valueOf(nowTimestamp), + true, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_TIME_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + device3 + ".s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + device3 + ".s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + device3 + ".s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + device3 + ".s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + device3 + "(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + device2 + "(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionPullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts_pattern_tsfile_pull") + .consumerGroupId("loose_range_time") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions(topicName).forEach(System.out::println); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after subscription"); + insert_data(nowTimestamp - 4000, device); + insert_data(nowTimestamp - 4000, device2); + + String sql = "select count(s_0) from " + device + " where time <=" + nowTimestamp; + System.out.println("TimeTsTsfilePullConsumer src1 filter:" + getCount(session_src, sql)); + + // Consumption data + List paths = new ArrayList<>(3); + paths.add(device); + paths.add(device2); + paths.add(device3); + + rowCountList = consume_tsfile(consumer, paths); + assertGte(rowCountList.get(0), 8, device); + assertEquals(rowCountList.get(1), 0, device2); + assertEquals(rowCountList.get(2), 0, device3); + + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "Show subscriptions after unsubscription"); + + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after re-subscribing"); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + System.out.println("TimeTsTsfilePullConsumer src2 filter:" + getCount(session_src, sql)); + + rowCountList = consume_tsfile(consumer, paths); + assertGte( + rowCountList.get(0), + 13, + "Unsubscribe and then resubscribe, progress is not retained. Full synchronization. Actual=" + + rowCountList.get(0)); + assertEquals(rowCountList.get(1), 0, "Unsubscribe and subscribe again," + device2); + assertEquals(rowCountList.get(2), 0, "Unsubscribe and then resubscribe," + device3); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/mode/IoTDBSnapshotDevicePullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/mode/IoTDBSnapshotDevicePullConsumerDataSetIT.java new file mode 100644 index 000000000000..8f6f432558f8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/mode/IoTDBSnapshotDevicePullConsumerDataSetIT.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.mode; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBSnapshotDevicePullConsumerDataSetIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.SnapshotDevicePullConsumerDataSet"; + private static final String database2 = "root.SnapshotDevicePullConsumerDataSet"; + private static final String topicName = "topicSnapshotDevicePullConsumerDataSet"; + private String device = database + ".d_0"; + + private String pattern = device + ".**"; + private static SubscriptionPullConsumer consumer; + private List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, "now", false, TopicConstant.MODE_SNAPSHOT_VALUE, null); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 float;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 float;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_mode", "device_snapshot_dataset", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + Thread.sleep(1000); + insert_data(System.currentTimeMillis() - 30000L); + // Consumption data + consume_data(consumer, session_dest); + String sql = "select count(s_0) from " + device; + System.out.println("src: " + getCount(session_src, sql)); + check_count(4, sql, "Consumption data:" + pattern); + check_count(4, "select count(s_1) from " + device, "Consumption data: s_1"); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption Data: d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + insert_data(System.currentTimeMillis()); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(8, "select count(s_0) from " + device, "Consume data again:" + pattern); + check_count(8, "select count(s_1) from " + device, "Consumption data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/mode/IoTDBSnapshotDevicePullConsumerTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/mode/IoTDBSnapshotDevicePullConsumerTsfileIT.java new file mode 100644 index 000000000000..6f8d49f4fc47 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/mode/IoTDBSnapshotDevicePullConsumerTsfileIT.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.mode; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer + * pattern: device + * tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBSnapshotDevicePullConsumerTsfileIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.SnapshotDevicePullConsumerTsfile"; + private static final String database2 = "root.SnapshotDevicePullConsumerTsfile"; + private static final String device = database + ".d_0"; + private static final String topicName = "topicSnapshotDevicePullConsumerTsfile"; + private static final String pattern = device + ".**"; + private static SubscriptionPullConsumer consumer; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, true, TopicConstant.MODE_SNAPSHOT_VALUE, null); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + + consumer = + new SubscriptionPullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("device_tsfile_snapshot") + .consumerGroupId("pull_mode") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + insert_data(System.currentTimeMillis()); + System.out.println("src :" + getCount(session_src, "select count(s_0) from " + device)); + + // Consumption data + List devices = new ArrayList<>(3); + devices.add(device); + devices.add(database + ".d_1"); + devices.add(database2 + ".d_2"); + List rowCounts = consume_tsfile(consumer, devices); + + assertEquals(rowCounts.get(0), 5); + assertEquals(rowCounts.get(1), 0); + assertEquals(rowCounts.get(2), 0); + + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "Show subscriptions after unsubscription"); + + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + + // Consumption data: Progress is not retained after unsubscribing and then re-subscribing. Full + // synchronization. + rowCounts = consume_tsfile(consumer, devices); + assertEquals( + rowCounts.get(0), + 10, + "Unsubscribe and resubscribe, progress is not retained. Full synchronization."); + assertEquals(rowCounts.get(1), 0, "Unsubscribe and resubscribe," + database + ".d_1"); + assertEquals( + rowCounts.get(2), 0, "Cancel subscription and subscribe again," + database2 + ".d_2"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/multi/IoTDBConsumer2With1TopicShareProcessDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/multi/IoTDBConsumer2With1TopicShareProcessDataSetIT.java new file mode 100644 index 000000000000..4af1f7f3b681 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/multi/IoTDBConsumer2With1TopicShareProcessDataSetIT.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBConsumer2With1TopicShareProcessDataSetIT + extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.Consumer2With1TopicShareProcessDataSet"; + private static final String device = database + ".d_0"; + private static final String topicName = "topicConsumer2With1TopicShareProcessDataSet"; + private String pattern = device + ".**"; + private SubscriptionPullConsumer consumer2; + private static SubscriptionPullConsumer consumer; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest2.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest2.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + consumer2.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("g1", "c1", false, null); + consumer2 = create_pull_consumer("g1", "c2", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + assertEquals(subs.getSubscriptions().size(), 0, "Before subscribing, show subscriptions"); + consumer.subscribe(topicName); + consumer2.subscribe(topicName); + System.out.println("After subscription:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + consume_data(consumer, session_dest); + check_count( + 5, "select count(s_0) from " + device, "Consumption subscription before data: s_0 "); + check_count( + 5, "select count(s_1) from " + device, "Consumption subscription before data: s_1 "); + // Subscribe and then write data + insert_data(System.currentTimeMillis()); + consume_data(consumer2, session_dest2); + check_count2(5, "select count(s_0) from " + device, "Consumption subscription data: s_0"); + check_count2(5, "select count(s_1) from " + device, "Consumption subscription data: s_1"); + + // Consumed data will not be consumed again + consume_data(consumer, session_dest); + check_count(5, "select count(s_0) from " + device, "Consumption subscription before data: s_0"); + check_count( + 5, "select count(s_1) from " + device, "Consumption subscription previous data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/multi/IoTDBConsumer2With1TopicShareProcessTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/multi/IoTDBConsumer2With1TopicShareProcessTsfileIT.java new file mode 100644 index 000000000000..934071921f82 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/multi/IoTDBConsumer2With1TopicShareProcessTsfileIT.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * format: tsfile + * pattern:device + * Same group pull consumer share progress + * About 1/5 chance, over 1000 + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBConsumer2With1TopicShareProcessTsfileIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.Consumer2With1TopicShareProcessTsfile"; + private static final String device = database + ".d_0"; + private static final String topicName = "topicConsumer2With1TopicShareProcessTsfile"; + private static List schemaList = new ArrayList<>(); + private String pattern = device + ".**"; + private SubscriptionPullConsumer consumer2; + private static SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createTopic_s(topicName, pattern, null, null, true); + createDB(database); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest2.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest2.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + consumer2.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("tsfile_group_share_process", "c1", false, null); + consumer2 = create_pull_consumer("tsfile_group_share_process", "c2", false, 10L); + // Subscribe + assertEquals( + subs.getSubscriptions(topicName).size(), 0, "Show subscriptions before subscribing"); + consumer.subscribe(topicName); + consumer2.subscribe(topicName); + subs.getSubscriptions(topicName).forEach((System.out::println)); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after subscription"); + // insert 1000 records + Thread thread = + new Thread( + () -> { + for (int i = 0; i < 200; i++) { + try { + insert_data(1706659200000L); + Thread.sleep(1000); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }); + AtomicInteger rowCount1 = new AtomicInteger(0); + AtomicInteger rowCount2 = new AtomicInteger(0); + Thread thread1 = + new Thread( + () -> { + try { + rowCount1.addAndGet(consume_tsfile(consumer, device)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + Thread thread2 = + new Thread( + () -> { + try { + rowCount2.addAndGet(consume_tsfile(consumer2, device)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + + thread1.start(); + thread2.start(); + thread.start(); + thread1.join(); + thread2.join(); + thread.join(); + + System.out.println("src :" + getCount(session_src, "select count(s_0) from " + device)); + System.out.println("rowCount1=" + rowCount1.get()); + System.out.println("rowCount2=" + rowCount2.get()); + AWAIT.untilAsserted( + () -> { + assertGte( + rowCount1.get() + rowCount2.get(), + 1000, + "consumer share process rowCount1=" + + rowCount1.get() + + " rowCount2=" + + rowCount2.get() + + " src=" + + getCount(session_src, "select count(s_0) from " + device)); + assertTrue(rowCount1.get() > 0); + assertTrue(rowCount2.get() > 0); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/multi/IoTDBMultiGroupVsMultiConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/multi/IoTDBMultiGroupVsMultiConsumerIT.java new file mode 100644 index 000000000000..ecae393ebfe6 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/multi/IoTDBMultiGroupVsMultiConsumerIT.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBMultiGroupVsMultiConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.pullMultiGroupVsMultiConsumer"; + private static final String device = database + ".d_0"; + private static List schemaList = new ArrayList<>(); + private String topicNamePrefix = "TopicPullMultiGroupVsMultiConsumer_"; + private int tsCount = 10; + private int consumertCount = 10; + private List consumers = new ArrayList<>(consumertCount); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + for (int i = 0; i < tsCount; i++) { + createTopic_s(topicNamePrefix + i, device + ".s_" + i, null, null, false); + session_src.createTimeseries( + device + ".s_" + i, TSDataType.INT32, TSEncoding.RLE, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_" + i, TSDataType.INT32, TSEncoding.RLE, CompressionType.LZMA2); + session_dest2.createTimeseries( + device + ".s_" + i, TSDataType.INT32, TSEncoding.RLE, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_" + i, TSDataType.INT32)); + } + System.out.println("topics:" + subs.getTopics()); + } + + @Override + @After + public void tearDown() throws Exception { + for (SubscriptionPullConsumer c : consumers) { + try { + c.close(); + } catch (Exception e) { + } + } + for (int i = 0; i < tsCount; i++) { + subs.dropTopic(topicNamePrefix + i); + } + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + for (int i = 0; i < tsCount; i++) { + tablet.addValue( + schemaList.get(i).getMeasurementId(), rowIndex, (row + 1) * 20 + i * 1000 + row); + } + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + /*** + * |c0|t0|g1| + * |c1|t0|g1| + * |c2|t1|g1| + * |c3|t1|g1| + * |c4|t2,t3|g2| + * |c5|t3,t4|g2| + * |c6|t2,t4|g2| + * |c7|t0,t3|g3| + * |c8|t6|g3| + * |c9|t0,t3|g3| + */ + @Test + public void do_test() + throws TException, + IoTDBConnectionException, + IOException, + StatementExecutionException, + InterruptedException { + int i = 0; + for (i = 0; i < 4; i++) { + consumers.add( + new SubscriptionPullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_" + i) + .consumerGroupId("pull_group_id_1") + .buildPullConsumer()); + } + for (; i < 7; i++) { + consumers.add( + new SubscriptionPullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_" + i) + .consumerGroupId("pull_group_id_2") + .buildPullConsumer()); + } + for (; i < consumertCount; i++) { + consumers.add( + new SubscriptionPullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_" + i) + .consumerGroupId("pull_group_id_3") + .buildPullConsumer()); + } + for (int j = 0; j < consumertCount; j++) { + consumers.get(j).open(); + } + + consumers.get(0).subscribe(topicNamePrefix + 0); + consumers.get(1).subscribe(topicNamePrefix + 0); + consumers.get(2).subscribe(topicNamePrefix + 1); + consumers.get(3).subscribe(topicNamePrefix + 1); + consumers.get(4).subscribe(topicNamePrefix + 2, topicNamePrefix + 3); + consumers.get(5).subscribe(topicNamePrefix + 3, topicNamePrefix + 4); + consumers.get(6).subscribe(topicNamePrefix + 2, topicNamePrefix + 4); + consumers.get(7).subscribe(topicNamePrefix + 0, topicNamePrefix + 3); + consumers.get(8).subscribe(topicNamePrefix + 6); + consumers.get(9).subscribe(topicNamePrefix + 0, topicNamePrefix + 3); + + subs.getSubscriptions().forEach(System.out::println); + // Write data + System.out.println("Write data 1"); + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consume_data(consumers.get(0), session_dest); + System.out.println("src:" + getCount(session_src, "select count(s_0) from " + device)); + check_count(5, "select count(s_0) from " + device, "1 post-consumption check: s_0"); + for (i = 1; i < tsCount; i++) { + check_count(0, "select count(s_" + i + ") from " + device, "1 pre-consumption check: s_" + i); + } + consume_data(consumers.get(2), session_dest); + check_count(5, "select count(s_1) from " + device, "1 post-consumption check: s_1"); + consume_data(consumers.get(4), session_dest); + check_count(5, "select count(s_2) from " + device, "1 post-consumption check: s_2"); + check_count(5, "select count(s_3) from " + device, "1 post-consumption check: s_3"); + for (i = 4; i < tsCount; i++) { + check_count(0, "select count(s_" + i + ") from " + device, "1 pre-consumption check: s_" + i); + } + + System.out.println("Write data 2"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + System.out.println("src:" + getCount(session_src, "select count(s_0) from " + device)); + consume_data(consumers.get(1), session_dest2); + consume_data(consumers.get(3), session_dest2); + consume_data(consumers.get(5), session_dest2); + consume_data(consumers.get(8), session_dest2); + + for (i = 0; i < 4; i++) { + if (i == 2) continue; + check_count2( + 5, "select count(s_" + i + ") from " + device, "2 pre-consumption check: s_" + i); + } + check_count2(10, "select count(s_4) from " + device, "2 check dest2:s_4 after consumption"); + check_count2(10, "select count(s_6) from " + device, "2 check dest2:s_6 after consumption"); + + consume_data(consumers.get(7), session_dest); + check_count(10, "select count(s_0) from " + device, "2 post-consumption check: s_0"); + check_count(10, "select count(s_3) from " + device, "2 post-consumption check: s_3"); + + System.out.println("Write data 3"); + insert_data(System.currentTimeMillis()); + consume_data(consumers.get(7), session_dest2); + check_count2(10, "select count(s_0) from " + device, "3 check dest2:s_0 after consumption"); + check_count2(10, "select count(s_3) from " + device, "3 consume check dest2:s_3"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/multi/IoTDBOneConsumerMultiTopicsDatasetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/multi/IoTDBOneConsumerMultiTopicsDatasetIT.java new file mode 100644 index 000000000000..261361b4a8a8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/multi/IoTDBOneConsumerMultiTopicsDatasetIT.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * 1 consumer subscribes to 2 topics: fixed time range + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBOneConsumerMultiTopicsDatasetIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.OneConsumerMultiTopicsDataset"; + private static final String device = database + ".d_0"; + private static List schemaList = new ArrayList<>(); + + private String pattern = device + ".s_0"; + private String pattern2 = device + ".s_1"; + private String topicName = "topic1_OneConsumerMultiTopicsDataset"; + private String topicName2 = "topic2_OneConsumerMultiTopicsDataset"; + private static SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, pattern, "2024-01-01T00:00:00+08:00", "2024-03-01T00:00:00+08:00", false); + createTopic_s( + topicName2, pattern2, "2024-01-01T00:00:00+08:00", "2024-03-13T00:00:00+08:00", false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + subs.dropTopic(topicName2); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + Thread.sleep(1000); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1710288000000,313,6.78);"); // 2024-03-13 08:00:00+08:00 + // Subscribe + consumer = create_pull_consumer("multi_1consumer_multiTopics", "c1", false, null); + assertEquals(subs.getSubscriptions().size(), 0, "Before subscription show subscriptions"); + + consumer.subscribe(topicName, topicName2); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 2, "show subscriptions after subscription"); + + Thread thread = + new Thread( + new Runnable() { + @Override + public void run() { + try { + insert_data(System.currentTimeMillis()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + thread.start(); + thread.join(); + String sql1 = "select count(s_0) from " + device; + String sql2 = "select count(s_1) from " + device; + System.out.println("src s_0:" + getCount(session_src, sql1)); + System.out.println("src s_1:" + getCount(session_src, sql2)); + // Consumption data + consume_data(consumer, session_dest); + System.out.println("dest s_0:" + getCount(session_dest, sql1)); + System.out.println("dest s_1:" + getCount(session_dest, sql2)); + AWAIT.untilAsserted( + () -> { + check_count(5, sql1, "Consumption data:" + pattern); + check_count(5, sql2, "Consumption data:" + pattern2); + }); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + // Subscribe and then write data + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1703980800000,1231,321.45);"); // 2023-12-31 08:00:00+08:00 + + // Consumption data + consume_data(consumer, session_dest); + AWAIT.untilAsserted( + () -> { + check_count(5, sql1, "consume data again:" + pattern); + check_count(10, sql2, "consume data again:" + pattern2); + }); + // Unsubscribe + consumer.unsubscribe(topicName, topicName2); + System.out.println("###### Query after unsubscription again:"); + subs.getSubscriptions().forEach((System.out::println)); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/multi/IoTDBOneConsumerMultiTopicsMixIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/multi/IoTDBOneConsumerMultiTopicsMixIT.java new file mode 100644 index 000000000000..3e77496db47f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/multi/IoTDBOneConsumerMultiTopicsMixIT.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * 1 consumer subscribes to 2 topics: Historical data + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBOneConsumerMultiTopicsMixIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.OneConsumerMultiTopicsMix"; + private static final String device = database + ".d_0"; + private String pattern = device + ".s_0"; + private String pattern2 = "root.**"; + private String topicName = "topic1_OneConsumerMultiTopicsMix"; + private String topicName2 = "topic2_OneConsumerMultiTopicsMix"; + private List schemaList = new ArrayList<>(); + private SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, "now", false); + createTopic_s(topicName2, pattern2, null, "now", true); + session_src.createTimeseries( + device + ".s_0", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.TEXT, TSEncoding.DICTIONARY, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.FLOAT)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.TEXT)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + assertTrue(subs.getTopic(topicName2).isPresent(), "Create show topics 2"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + subs.dropTopic(topicName2); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row + 2.45f); + tablet.addValue("s_1", rowIndex, "rowIndex" + rowIndex); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1710288000000,313,'2024-03-13 08:00:00+08:00');"); // 2024-03-13 + // 08:00:00+08:00 + session_src.executeNonQueryStatement("flush;"); + // Subscribe + consumer = create_pull_consumer("multi_1consumer_mix", "tsfile_dataset", false, null); + System.out.println("###### Before Subscription Query:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 0, "Before subscription show subscriptions"); + consumer.subscribe(topicName, topicName2); + long timestamp = System.currentTimeMillis(); + System.out.println("###### Subscription Query:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 2, "show subscriptions after subscription"); + // Subscribe and then write data + Thread thread = + new Thread( + new Runnable() { + @Override + public void run() { + try { + insert_data(System.currentTimeMillis()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + thread.start(); + + AtomicInteger rowCount = new AtomicInteger(0); + Thread thread2 = + new Thread( + () -> { + while (true) { + List messages = consumer.poll(Duration.ofMillis(10000)); + if (messages.isEmpty()) { + break; + } + + for (final SubscriptionMessage message : messages) { + final short messageType = message.getMessageType(); + if (SubscriptionMessageType.isValidatedMessageType(messageType)) { + switch (SubscriptionMessageType.valueOf(messageType)) { + case SESSION_DATA_SETS_HANDLER: + for (final Iterator it = + message.getSessionDataSetsHandler().tabletIterator(); + it.hasNext(); ) { + final Tablet tablet = it.next(); + try { + session_dest.insertTablet(tablet); + // + // System.out.println(format.format(new Date())+" "+tablet.rowSize); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + break; + case TS_FILE_HANDLER: + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(new Path(device, "s_1", true)), + null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println( + device + ".s_1:" + next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + consumer.commitSync(messages); + break; + } + } + } + } + }); + Thread thread3 = + new Thread( + () -> { + while (true) { + List messages = consumer.poll(Duration.ofMillis(10000)); + if (messages.isEmpty()) { + break; + } + + for (final SubscriptionMessage message : messages) { + final short messageType = message.getMessageType(); + if (SubscriptionMessageType.isValidatedMessageType(messageType)) { + switch (SubscriptionMessageType.valueOf(messageType)) { + case SESSION_DATA_SETS_HANDLER: + for (final Iterator it = + message.getSessionDataSetsHandler().tabletIterator(); + it.hasNext(); ) { + final Tablet tablet = it.next(); + try { + session_dest.insertTablet(tablet); + System.out.println(FORMAT.format(new Date()) + " " + tablet.rowSize); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + break; + case TS_FILE_HANDLER: + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(new Path(device, "s_1", true)), + null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println( + device + ":" + next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + consumer.commitSync(messages); + break; + } + } + } + } + }); + thread2.start(); + thread.join(); + thread2.join(5000); + String sql1 = "select count(s_0) from " + device + " where time <= " + timestamp; + String sql2 = "select count(s_1) from " + device + " where time <= " + timestamp; + + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql1)); + AWAIT.untilAsserted( + () -> { + // Consumption data + check_count(6, sql1, "Consumption data:" + pattern); + check_count(0, sql2, "Consumption data:" + pattern2); + assertEquals(rowCount.get(), 6, "tsfile consumer"); + }); + // Unsubscribe + consumer.unsubscribe(topicName); + System.out.println("###### After unsubscribing query:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "Unsubscribe 1 and then show subscriptions"); + + // Unsubscribe and then write data + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1703980800000,3.45,'2023-12-31 08:00:00+08:00');"); // 2023-12-31 08:00:00+08:00 + insert_data(System.currentTimeMillis()); + session_src.executeNonQueryStatement("flush;"); + System.out.println( + FORMAT.format(new Date()) + + " Unsubscribe after writing data src:" + + getCount(session_src, sql1)); + + thread3.start(); + thread3.join(5000); + System.out.println(FORMAT.format(new Date())); + AWAIT.untilAsserted( + () -> { + assertEquals( + rowCount.get(), 12, "Re-consume data: tsfile consumer " + FORMAT.format(new Date())); + check_count(6, sql1, "consume data again:" + pattern); + check_count(0, sql2, "Reconsume data:" + pattern2); + }); + // close + consumer.close(); + try { + consumer.subscribe(topicName, topicName2); + } catch (Exception e) { + System.out.println("subscribe again after close, expecting an exception"); + } + assertEquals(subs.getSubscriptions().size(), 0, "show subscriptions after close"); + subs.getSubscriptions().forEach((System.out::println)); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/multi/IoTDBOneConsumerMultiTopicsTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/multi/IoTDBOneConsumerMultiTopicsTsfileIT.java new file mode 100644 index 000000000000..9dc310554ccf --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/multi/IoTDBOneConsumerMultiTopicsTsfileIT.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +/*** + * 1 consumer subscribes to 2 topics: historical data + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBOneConsumerMultiTopicsTsfileIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.OneConsumerMultiTopicsTsfile"; + private static final String device = database + ".d_0"; + private static List schemaList = new ArrayList<>(); + private String pattern = database + ".**"; + private String database2 = "root.OneConsumerMultiTopicsTsfile"; + private String pattern2 = database2 + ".**"; + private String device2 = database2 + ".d_0"; + private String topicName = "topic1_OneConsumerMultiTopicsTsfile"; + private String topicName2 = "topic2_OneConsumerMultiTopicsTsfile"; + private SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, "now", null, true); + createTopic_s(topicName2, pattern2, "now", null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.TEXT, TSEncoding.DICTIONARY, CompressionType.LZMA2); + session_src.createTimeseries( + device2 + ".s_0", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device2 + ".s_1", TSDataType.TEXT, TSEncoding.DICTIONARY, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.FLOAT)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.TEXT)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + assertTrue(subs.getTopic(topicName2).isPresent(), "Create show topics 2"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + subs.dropTopic(topicName2); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row + 2.45f); + tablet.addValue("s_1", rowIndex, "rowIndex" + rowIndex); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + Thread.sleep(1000); + // Write data before subscribing + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1710288000000,313,'2024-03-13 08:00:00+08:00');"); // 2024-03-13 + // 08:00:00+08:00 + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer = create_pull_consumer("multi_tsfile_2topic", "1_consumer", false, null); + System.out.println("###### Subscription Query Before:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 0, "Before subscription show subscriptions"); + consumer.subscribe(topicName, topicName2); + System.out.println("###### Subscribe and query:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 2, "subscribe then show subscriptions"); + + final AtomicInteger rowCount = new AtomicInteger(); + Thread thread1 = + new Thread( + () -> { + List devices = new ArrayList<>(2); + devices.add(device); + devices.add(device2); + try { + List results = consume_tsfile(consumer, devices); + System.out.println(results); + rowCount.addAndGet(results.get(0)); + rowCount.addAndGet(results.get(1)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + thread1.start(); + // Subscribe and then write data + Thread thread = + new Thread( + () -> { + long timestamp = System.currentTimeMillis(); + for (int i = 0; i < 20; i++) { + try { + insert_data(timestamp, device); + insert_data(timestamp, device2); + timestamp += 30000; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + thread.start(); + thread.join(); + thread1.join(); + + System.out.println( + "src insert " + device + " :" + getCount(session_src, "select count(s_0) from " + device)); + System.out.println( + "src insert " + + device2 + + " :" + + getCount(session_src, "select count(s_0) from " + device2)); + assertEquals(rowCount.get(), 200, "After first consumption"); + // Unsubscribe + consumer.unsubscribe(topicName); + System.out.println("###### After cancellation query:"); + subs.getSubscriptions(topicName).forEach((System.out::println)); + assertEquals( + subs.getSubscriptions(topicName).size(), 0, "Unsubscribe 1 and then show subscriptions"); + + // Unsubscribe and then write data + insert_data(System.currentTimeMillis(), device2); + int result = consume_tsfile(consumer, device2); + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1703980800000,3.45,'2023-12-31 08:00:00+08:00');"); // 2023-12-31 08:00:00+08:00 + assertEquals(result, 5, "After the second consumption"); + + // close + consumer.close(); + try { + consumer.subscribe(topicName, topicName2); + } catch (Exception e) { + System.out.println("subscribe again after close, expecting an exception"); + } + assertEquals(subs.getSubscriptions(topicName).size(), 0, "show subscriptions after close"); + subs.getSubscriptions(topicName).forEach((System.out::println)); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBDBPatternPullConsumeTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBDBPatternPullConsumeTsfileIT.java new file mode 100644 index 000000000000..bfa0d3b71fbe --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBDBPatternPullConsumeTsfileIT.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer + * pattern: db + * Tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBDBPatternPullConsumeTsfileIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.DBPatternPullConsumeTsfile"; + private static final String database2 = "root.DBPatternPullConsumeTsfile"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topicDBPatternPullConsumeTsfile"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = database + ".**"; + private static SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + device2 + "(time,s_0,s_1)values(2000,232,567.891);"); + + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionPullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("db_tsfile") + .consumerGroupId("pull_pattern") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions(topicName).forEach(System.out::println); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after subscription"); + // insert_data(1706659200000L); //2024-01-31 08:00:00+08:00 + insert_data(System.currentTimeMillis()); + // Consumption data + List results = consume_tsfile_withFileCount(consumer, device); + assertEquals(results.get(0), 10); + assertEquals(results.get(1), 3, "number of files received"); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 0, "Show subscriptions after unsubscription"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + + // Consumption data: Progress is not retained after unsubscribing and resubscribing. Full + // synchronization. + results = consume_tsfile_withFileCount(consumer, device); + assertEquals( + results.get(0), + 15, + "Unsubscribe and then resubscribe, progress is not retained. Full synchronization."); + assertEquals(results.get(1), 4, "Number of files received: resubscribe after unsubscribe"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBDBPatternPullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBDBPatternPullConsumerDataSetIT.java new file mode 100644 index 000000000000..bbb0f74ab87f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBDBPatternPullConsumerDataSetIT.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * format: dataset + * pattern:db + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBDBPatternPullConsumerDataSetIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.DBPatternPullConsumerDataSet"; + private static final String database2 = "root.DBPatternPullConsumerDataSet"; + private static final String device = database + ".d_0"; + private static final String topicName = "topicDBPatternPullConsumerDataSet"; + private static List schemaList = new ArrayList<>(); + private String pattern = database + ".**"; + private static SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("db_dataset_snapshot", "pull_mode", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + // Consumption data + consume_data(consumer, session_dest); + check_count(8, "select count(s_0) from " + device, "Consumption Data: s_0"); + check_count(8, "select count(s_1) from " + device, "Consumption Data: s_1"); + check_count(1, "select count(s_0) from " + database + ".d_1", "Consumption data:d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_1"); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(12, "select count(s_0) from " + device, "consume data again:s_0"); + check_count(12, "select count(s_1) from " + device, "Consumption Data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBDefaultPatternPullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBDefaultPatternPullConsumerDataSetIT.java new file mode 100644 index 000000000000..a04d05ada701 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBDefaultPatternPullConsumerDataSetIT.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBDefaultPatternPullConsumerDataSetIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.DefaultPatternPullConsumerDataSet"; + private static final String database2 = "root.DefaultPatternPullConsumerDataSet"; + private static final String device = database + ".d_0"; + private static final String topicName = "topicDefaultPatternPullConsumerDataSet"; + private static SubscriptionPullConsumer consumer; + private static List schemaList = new ArrayList<>(); + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, null, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_pattern", "default_pattern_dataset", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + // Consumption data + consume_data(consumer, session_dest); + check_count(8, "select count(s_0) from " + device, "Consumption data: s_0"); + check_count(8, "select count(s_1) from " + device, "Consumption data: s_1"); + check_count(1, "select count(s_0) from " + database + ".d_1", "Consumption data:d_1"); + check_count(1, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after unsubscribing and resubscribing. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(12, "select count(s_0) from " + device, "consume data again:s_0"); + check_count(12, "select count(s_1) from " + device, "Consumption data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBDevicePatternPullConsumeTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBDevicePatternPullConsumeTsfileIT.java new file mode 100644 index 000000000000..7e9bcf723ca4 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBDevicePatternPullConsumeTsfileIT.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer + * pattern: device + * Tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBDevicePatternPullConsumeTsfileIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.DevicePatternPullConsumeTsfile"; + private static final String database2 = "root.DevicePatternPullConsumeTsfile"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String device3 = database2 + ".d_2"; + private static final String topicName = "topicDevicePatternPullConsumeTsfile"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = device + ".**"; + private static SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionPullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("device_pattern_tsfile") + .consumerGroupId("pull_pattern") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + // Consumption data + List devices = new ArrayList<>(3); + devices.add(device); + devices.add(device2); + devices.add(device3); + List results = consume_tsfile(consumer, devices); + assertEquals(results.get(0), 10); + assertEquals(results.get(1), 0); + assertEquals(results.get(2), 0); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "Show subscriptions after unsubscribe"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after unsubscribing and resubscribing. Full + // synchronization. + results = consume_tsfile(consumer, devices); + assertEquals( + results.get(0), + 15, + "After unsubscribing and resubscribing, progress is not retained. Full synchronization."); + assertEquals(results.get(1), 0, "Cancel subscription and subscribe again," + device2); + assertEquals(results.get(2), 0, "Unsubscribe and subscribe again," + device3); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBDevicePatternPullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBDevicePatternPullConsumerDataSetIT.java new file mode 100644 index 000000000000..58d5c1b54036 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBDevicePatternPullConsumerDataSetIT.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBDevicePatternPullConsumerDataSetIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.DevicePatternPullConsumerDataSet"; + private static final String database2 = "root.DevicePatternPullConsumerDataSet"; + private static final String device = database + ".d_0"; + private static final String device2 = database2 + ".d_2"; + private static final String topicName = "topicDevicePatternPullConsumerDataSet"; + private static List schemaList = new ArrayList<>(); + + private String pattern = device + ".**"; + public static SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, "now", false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + device2 + ".s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + device2 + ".s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + device2 + ".s_1 float;"); + session_dest.executeNonQueryStatement("create timeseries " + device2 + ".s_1 float;"); + session_src.executeNonQueryStatement( + "insert into " + device2 + "(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } finally { + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + } + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_pattern", "device_dataset", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis() - 30000L); + // Consumption data + consume_data(consumer, session_dest); + String sql = "select count(s_0) from " + device; + System.out.println("src: " + getCount(session_src, sql)); + check_count(8, sql, "Consumption data:" + pattern); + check_count(8, "select count(s_1) from " + device, "Consumption data: s_1"); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption data:d_1"); + check_count(0, "select count(s_0) from " + device2, "Consumption data:d_2"); + insert_data(System.currentTimeMillis()); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + System.out.println("src: " + getCount(session_src, sql)); + // Consumption data: Progress is not retained after unsubscribing and then re-subscribing. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(12, "select count(s_0) from " + device, "consume data again:s_0"); + check_count(12, "select count(s_1) from " + device, "Consumption data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBMiddleMatch2PatternPullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBMiddleMatch2PatternPullConsumerDataSetIT.java new file mode 100644 index 000000000000..25c832c71136 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBMiddleMatch2PatternPullConsumerDataSetIT.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * pattern: root.*.d_*.* + * format: dataset + * time-range: history + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBMiddleMatch2PatternPullConsumerDataSetIT + extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.MiddleMatch2PatternPullConsumerDataSet"; + private static final String database2 = "root.MiddleMatch2PatternPullConsumerDataSet"; + private static List devices = new ArrayList<>(3); + private static final String device = database + ".d_0"; + private static final String device2 = database2 + ".sd_1"; + private static final String device3 = database2 + ".d_2"; + private static final String topicName = "topicMiddleMatch2PatternPullConsumerDataSet"; + private static List schemaList = new ArrayList<>(); + + private String pattern = "root.*.d_*.*"; + private static SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, "now", false); + devices.add(device); + devices.add(device2); + devices.add(device3); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + device3 + ".s_0 int64;"); + session_dest.executeNonQueryStatement("create timeseries " + device3 + ".s_0 int64;"); + session_src.executeNonQueryStatement("create timeseries " + device3 + ".s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + device3 + ".s_1 double;"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = + create_pull_consumer("pull_pattern", "MiddleMatchPatternHistory_DataSet", false, null); + // Write data before subscribing + for (int i = 0; i < 3; i++) { + insert_data(1706659200000L, devices.get(i)); // 2024-01-31 08:00:00+08:00 + } + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + for (int i = 0; i < 3; i++) { + insert_data(System.currentTimeMillis() - 30000L, devices.get(i)); + } + // Consumption data + consume_data(consumer, session_dest); + String sql = "select count(s_0) from "; + for (int i = 0; i < 3; i++) { + System.out.println( + "src " + devices.get(i) + ": " + getCount(session_src, sql + devices.get(i))); + } + check_count(0, sql + device, "Consumption data: s_0 " + device); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "Consumption data: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "Consumption data: s_1 " + device2); + check_count(10, "select count(s_0) from " + device3, "Consumption data: s_0 " + device3); + check_count(10, "select count(s_1) from " + device3, "Consumption data: s_1 " + device3); + for (int i = 0; i < 3; i++) { + insert_data(System.currentTimeMillis(), devices.get(i)); + } + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + for (int i = 0; i < 3; i++) { + System.out.println( + "src " + devices.get(i) + ": " + getCount(session_src, sql + devices.get(i))); + } + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(0, "select count(s_0) from " + device, "consume data again:" + device); + check_count(0, "select count(s_0) from " + device2, "consume data again:" + device2); + check_count(10, "select count(s_1) from " + device3, "Consumption data:" + device3); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBMiddleMatchPatternPullConsumeTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBMiddleMatchPatternPullConsumeTsfileIT.java new file mode 100644 index 000000000000..6c00913f8231 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBMiddleMatchPatternPullConsumeTsfileIT.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer + * pattern: device + * Tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBMiddleMatchPatternPullConsumeTsfileIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.MiddleMatchPatternPullConsumeTsfile"; + private static final String database2 = "root.MiddleMatchPatternPullConsumeTsfile"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topicMiddleMatchPatternPullConsumeTsfile"; + private static List schemaList = new ArrayList<>(); + private static final String pattern = "root.**.d_*.**"; + private static SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, "now", true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionPullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("MiddleMatchPatternHistory_tsfile") + .consumerGroupId("pull_pattern") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis() - 20000); + // Consumption data + List devices = new ArrayList<>(3); + devices.add(device); + devices.add(device2); + devices.add(database2 + ".d_2"); + + List rowCounts = consume_tsfile(consumer, devices); + assertEquals(rowCounts.get(0), 10); + assertEquals(rowCounts.get(1), 1); + assertEquals(rowCounts.get(2), 1); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "Show subscriptions after cancellation"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after canceling and re-subscribing. Full + // synchronization. + rowCounts = consume_tsfile(consumer, devices); + + assertEquals( + rowCounts.get(0), + 15, + "Unsubscribe and resubscribe, progress is not retained. Full synchronization."); + assertEquals( + rowCounts.get(1), 1, "Cancel subscription and subscribe again," + database + ".d_1"); + assertEquals(rowCounts.get(2), 1, "Unsubscribe and resubscribe," + database2 + ".d_2"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBMiddleMatchPatternPullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBMiddleMatchPatternPullConsumerDataSetIT.java new file mode 100644 index 000000000000..5840a4c9a910 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBMiddleMatchPatternPullConsumerDataSetIT.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBMiddleMatchPatternPullConsumerDataSetIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.MiddleMatchPatternPullConsumerDataSet"; + private static final String database2 = "root.MiddleMatchPatternPullConsumerDataSet"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topicMiddleMatchPatternPullConsumerDataSet"; + private static List schemaList = new ArrayList<>(); + + private String pattern = "root.**.d_*.s_0"; + public static SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, "now", false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 float;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 float;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = + create_pull_consumer("pull_pattern", "MiddleMatchPatternHistory_DataSet", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis() - 30000L); + // Consumption data + consume_data(consumer, session_dest); + String sql = "select count(s_0) from " + device; + System.out.println("src " + database + ".d_0.s_0: " + getCount(session_src, sql)); + System.out.println( + "src " + + database + + ".d_0.s_1: " + + getCount(session_src, "select count(s_1) from " + device)); + System.out.println( + "src " + + database + + ".d_1.s_0: " + + getCount(session_src, "select count(s_0) from " + database + ".d_1")); + System.out.println( + "src " + + database2 + + ".d_2.s_0: " + + getCount(session_src, "select count(s_0) from " + database2 + ".d_2")); + System.out.println("dest " + database + ".d_0.s_0: " + getCount(session_dest, sql)); + System.out.println( + "dest " + + database + + ".d_0.s_1: " + + getCount(session_dest, "select count(s_1) from " + device)); + System.out.println( + "dest " + + database + + ".d_1.s_0: " + + getCount(session_dest, "select count(s_0) from " + database + ".d_1")); + System.out.println( + "dest " + + database2 + + ".s_0: " + + getCount(session_dest, "select count(s_0) from " + database2 + ".d_2")); + check_count(10, sql, "Consumption Data:s_0"); + check_count(0, "select count(s_1) from " + device, "Consumption Data: s_1"); + check_count(1, "select count(s_0) from " + database + ".d_1", "Consumption Data:d_1"); + check_count(1, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + insert_data(System.currentTimeMillis()); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + System.out.println("src: " + getCount(session_src, sql)); + // Consumption data: Progress is not preserved if you unsubscribe and then resubscribe. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(15, "select count(s_0) from " + device, "Consume data again:s_0"); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBRootPatternPullConsumeTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBRootPatternPullConsumeTsfileIT.java new file mode 100644 index 000000000000..8e91c4aa3881 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBRootPatternPullConsumeTsfileIT.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer + * pattern: db + * Tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBRootPatternPullConsumeTsfileIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.RootPatternPullConsumeTsfile"; + private static final String device = database + ".d_0"; + private static final String topicName = "topicRootPatternPullConsumeTsfile"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = "root.**"; + public static SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionPullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("root_tsfile") + .consumerGroupId("pull_pattern") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + // insert_data(1706659200000L); //2024-01-31 08:00:00+08:00 + insert_data(System.currentTimeMillis()); + // Consumption data + List results = consume_tsfile_withFileCount(consumer, device); + assertEquals(results.get(0), 10, "Number of consumption data rows"); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "show subscriptions after unsubscribe"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + results = consume_tsfile_withFileCount(consumer, device); + assertEquals( + results.get(0), + 15, + "After unsubscribing and resubscribing, progress is not retained. Full synchronization."); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBTSPatternPullConsumeTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBTSPatternPullConsumeTsfileIT.java new file mode 100644 index 000000000000..0bfd0e488aa9 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBTSPatternPullConsumeTsfileIT.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * PullConsumer + * pattern: TS + * history + * Tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTSPatternPullConsumeTsfileIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.DBPatternPullConsumeTsfile"; + private static final String database2 = "root.DBPatternPullConsumeTsfile"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topicDBPatternPullConsumeTsfile"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = device + ".s_0"; + + public static SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, "now", true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_0 int32;"); + session_src.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_dest.executeNonQueryStatement("create timeseries " + database2 + ".d_2.s_1 double;"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionPullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts_tsfile") + .consumerGroupId("pull_pattern") + .autoCommit(false) + .fileSaveDir("target/pull-subscription") // hack for license check + .buildPullConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + // insert_data(1706659200000L); //2024-01-31 08:00:00+08:00 + insert_data(System.currentTimeMillis() - 30000L); + // Consumption data + List devices = new ArrayList<>(3); + devices.add(device); + devices.add(device2); + devices.add(database2 + ".d_2"); + List results = consume_tsfile(consumer, devices); + assertEquals(results.get(0), 10); + assertEquals(results.get(1), 0); + assertEquals(results.get(2), 0); + assertEquals(results.get(3), 1, "number of received files"); + insert_data(System.currentTimeMillis()); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "Show subscriptions after unsubscription"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + results = consume_tsfile(consumer, devices); + + assertEquals( + results.get(0), + 15, + "Unsubscribe and resubscribe, progress is not retained. Full synchronization."); + assertEquals(results.get(1), 0, "Subscribe again after unsubscribe," + database + ".d_1"); + assertEquals(results.get(2), 0, "Unsubscribe and then subscribe again," + database2 + ".d_2"); + assertEquals(results.get(3), 1, "Number of received files: resubscribe after unsubscription"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBTSPatternPullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBTSPatternPullConsumerDataSetIT.java new file mode 100644 index 000000000000..d7a38e8304a4 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/pattern/IoTDBTSPatternPullConsumerDataSetIT.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTSPatternPullConsumerDataSetIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.TSPatternPullConsumerDataSet"; + private static final String device = database + ".d_0"; + private static final String topicName = "topicTSPatternPullConsumerDataSet"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = device + ".s_0"; + public static SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + Thread.sleep(1000); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_pattern", "ts_dataset", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + // Consumption data + consume_data(consumer, session_dest); + check_count(8, "select count(s_0) from " + device, "Consumption data: s_0"); + check_count(0, "select count(s_1) from " + device, "Consumption Data: s_1"); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "show subscriptions after unsubscription"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(12, "select count(s_0) from " + device, "consume data again:s_0"); + check_count(0, "select count(s_1) from " + device, "Consumption Data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/time/IoTDBAllPullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/time/IoTDBAllPullConsumerDataSetIT.java new file mode 100644 index 000000000000..4401b1e582f6 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/time/IoTDBAllPullConsumerDataSetIT.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBAllPullConsumerDataSetIT extends AbstractSubscriptionRegressionIT { + private String database = "root.AllPullConsumerDataSet"; + private String device = database + ".d_0"; + private String pattern = device + ".s_0"; + // private String topicName = "`1-group.1-consumer.ts`"; + private String topicName = "topic_AllPullConsumerDataSet"; + private List schemaList = new ArrayList<>(); + private SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_time", "ts_time_default_dataset", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + Thread.sleep(10000); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + // Consumption data + consume_data(consumer, session_dest); + check_count(8, "select count(s_0) from " + device, "Consumption data:" + pattern); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1"); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(12, "select count(s_0) from " + device, "consume data again:" + pattern); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/time/IoTDBHistoryPullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/time/IoTDBHistoryPullConsumerDataSetIT.java new file mode 100644 index 000000000000..9ffe87dfd3e8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/time/IoTDBHistoryPullConsumerDataSetIT.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBHistoryPullConsumerDataSetIT extends AbstractSubscriptionRegressionIT { + private String database = "root.HistoryPullConsumerDataSet"; + private String device = database + ".d_0"; + private String pattern = device + ".s_0"; + // private String topicName = "`1-group.1-consumer.ts`"; + private String topicName = "topic_HistoryPullConsumerDataSet"; + private List schemaList = new ArrayList<>(); + private SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, "now", false); + session_src.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest2.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest2.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_time", "ts_history_dataset", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + // Consumption data + consume_data(consumer, session_dest); + + // System.out.println("Check consumption data"); + check_count(4, "select count(s_0) from " + device, "Consumption data:" + pattern); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1"); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + // System.out.println("After resubscribing:"); + // subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after canceling and re-subscribing. Full + // synchronization. + consume_data(consumer, session_dest2); + check_count2(8, "select count(s_0) from " + device, "consume data again:" + pattern); + check_count2(0, "select count(s_1) from " + device, "Consumption Data: s_1"); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/time/IoTDBRealTimePullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/time/IoTDBRealTimePullConsumerDataSetIT.java new file mode 100644 index 000000000000..b3b73d21766c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/time/IoTDBRealTimePullConsumerDataSetIT.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBRealTimePullConsumerDataSetIT extends AbstractSubscriptionRegressionIT { + private String database = "root.RealTimePullConsumerDataSet"; + private String device = database + ".d_0"; + private String pattern = device + ".s_0"; + private String topicName = "topic_RealTimePullConsumerDataSet"; + private List schemaList = new ArrayList<>(); + private SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + + createDB(database); + createTopic_s(topicName, pattern, "now", null, false); + session_src.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest2.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest2.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_time", "ts_realtime_dataset", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + // Consumption data + consume_data(consumer, session_dest); + check_count(0, "select count(s_0) from " + device, "Consumption data:" + pattern); + insert_data(System.currentTimeMillis()); + consume_data(consumer, session_dest); + check_count(4, "select count(s_0) from " + device, "Consumption data:" + pattern); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + insert_data(System.currentTimeMillis()); // now + consumer.subscribe(topicName); + System.out.println("After re-subscribing:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + // Consumption data + consume_data(consumer, session_dest2); + check_count(4, "select count(s_0) from " + device, "Consume data again:" + pattern); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/time/IoTDBTimeRangeAccuratePullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/time/IoTDBTimeRangeAccuratePullConsumerDataSetIT.java new file mode 100644 index 000000000000..1c21a3a3eb97 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/time/IoTDBTimeRangeAccuratePullConsumerDataSetIT.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTimeRangeAccuratePullConsumerDataSetIT extends AbstractSubscriptionRegressionIT { + private String database = "root.TimeRangeAccuratePullConsumerDataSet"; + private String device = database + ".d_0"; + private String pattern = device + ".s_0"; + private String topicName = "topic_TimeRangeAccuratePullConsumerDataSet"; + private List schemaList = new ArrayList<>(); + private SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, pattern, "2024-01-01T00:00:00+08:00", "2024-03-31T23:59:59+08:00", false); + session_src.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_time", "ts_accurate_dataset", false, null); + // Write data before subscribing + insert_data(1704038396000L); // 2023-12-31 23:59:56+08:00 + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + // Before consumption subscription data + consume_data(consumer, session_dest); + check_count(2, "select count(s_0) from " + device, "Start time boundary data:" + pattern); + + insert_data(System.currentTimeMillis()); // now + consume_data(consumer, session_dest); + check_count( + 2, "select count(s_0) from " + device, "Write some real-time data after:" + pattern); + + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + consume_data(consumer, session_dest); + check_count(6, "select count(s_0) from " + device, "Data within the time range:" + pattern); + + insert_data(1711814398000L); // 2024-03-30 23:59:58+08:00 + consume_data(consumer, session_dest); + check_count(10, "select count(s_0) from " + device, "End time boundary data:" + pattern); + + insert_data(1711900798000L); // 2024-03-31 23:59:58+08:00 + consume_data(consumer, session_dest); + check_count(11, "select count(s_0) from " + device, "End time limit data 2:" + pattern); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/time/IoTDBTimeRangePullConsumerDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/time/IoTDBTimeRangePullConsumerDataSetIT.java new file mode 100644 index 000000000000..66b57279fa2f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pullconsumer/time/IoTDBTimeRangePullConsumerDataSetIT.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pullconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * Start time, end time are both closed intervals. If not specified, the time will be 00:00:00. + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTimeRangePullConsumerDataSetIT extends AbstractSubscriptionRegressionIT { + private String database = "root.TimeRangePullConsumerDataSet"; + private String device = database + ".d_0"; + private String pattern = device + ".s_0"; + private String topicName = "topic_TimeRangePullConsumerDataSet"; + private List schemaList = new ArrayList<>(); + private SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, pattern, "2024-01-01T00:00:00+08:00", "2024-03-31T00:00:00+08:00", false); + session_src.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + consumer.close(); + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("pull_time", "ts_time_range_dataset", false, null); + // Write data before subscribing + insert_data(1704038396000L); // 2023-12-31 23:59:56+08:00 + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + // Before consumption subscription data + consume_data(consumer, session_dest); + check_count(2, "select count(s_0) from " + device, "Start time boundary data:" + pattern); + + insert_data(System.currentTimeMillis()); // now + consume_data(consumer, session_dest); + check_count( + 2, "select count(s_0) from " + device, "Write some real-time data later:" + pattern); + + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + consume_data(consumer, session_dest); + check_count(6, "select count(s_0) from " + device, "Data within the time range:" + pattern); + + insert_data(1711814398000L); // 2024-03-30 23:59:58+08:00 + consume_data(consumer, session_dest); + // Because the end time is 2024-03-31 00:00:00, closed interval + check_count(8, "select count(s_0) from " + device, "End time limit data:" + pattern); + + insert_data(1711900798000L); // 2024-03-31 23:59:58+08:00 + consume_data(consumer, session_dest); + check_count(8, "select count(s_0) from " + device, "End time boundary data:" + pattern); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/format/IoTDBTestPushConsumeDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/format/IoTDBTestPushConsumeDataSetIT.java new file mode 100644 index 000000000000..f8c446c1182a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/format/IoTDBTestPushConsumeDataSetIT.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.format; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer: BEFORE_CONSUME + * DataSet + * pattern: root.** + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTestPushConsumeDataSetIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.TestPushConsumeDataSet"; + private static final String topicName = "topic_TestPushConsumeDataSet"; + private static List schemaList = new ArrayList<>(); + private static final String pattern = "root.**"; + private static SubscriptionPushConsumer consumer; + private static final String device = database + ".d_push_dataset"; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + + final AtomicInteger rowCount = new AtomicInteger(0); + consumer = + new SubscriptionPushConsumer.Builder() + .nodeUrls(Collections.singletonList(SRC_HOST + ":" + SRC_PORT)) + .consumerId("root_dataset_consumer") + .consumerGroupId("push_format") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .fileSaveDir("target/iotdb-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + String sql0 = "select count(s_0) from " + device; + String sql1 = "select count(s_1) from " + device; + long expectCount = getCount(session_src, sql0); + System.out.println("###### src " + expectCount); + long finalExpectCount = expectCount; + AWAIT.untilAsserted( + () -> { + assertEquals(getCount(session_dest, sql0), finalExpectCount, "First result:s_0"); + assertEquals(getCount(session_dest, sql1), finalExpectCount, "First result:s_1"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "After cancellation, show subscriptions"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + expectCount = getCount(session_src, sql0); + System.out.println("###### src2 " + expectCount); + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + long finalExpectCount1 = expectCount; + AWAIT.untilAsserted( + () -> { + assertEquals(getCount(session_dest, sql0), finalExpectCount1, "Second result: s_0"); + assertEquals(getCount(session_dest, sql1), finalExpectCount1, "Second result: s_1"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/format/IoTDBTestPushConsumeNoTargetDirTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/format/IoTDBTestPushConsumeNoTargetDirTsfileIT.java new file mode 100644 index 000000000000..182ba3c3a00b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/format/IoTDBTestPushConsumeNoTargetDirTsfileIT.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.format; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer:BEFORE_CONSUME + * TsFileHandler + * pattern: db + * target: no + * consumer_id: no + * group_id: no + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTestPushConsumeNoTargetDirTsfileIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.TestPushConsumeNoTargetDirTsfile"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_TestPushConsumeNoTargetDirTsfile"; + private static List schemaList = new ArrayList<>(); + private static final String pattern = database + ".**"; + private static SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + System.out.println("### TestPushConsumeNoTargetDirTsfile ###"); + + final AtomicInteger onReceiveCount = new AtomicInteger(0); + final AtomicInteger rowCount = new AtomicInteger(0); + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .consumeListener( + message -> { + onReceiveCount.incrementAndGet(); + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + Path path = new Path(device, "s_0", true); + QueryDataSet dataset = + reader.query(QueryExpression.create(Collections.singletonList(path), null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + dataset.next(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceiveCount.get(), 2, "should 2 tsfile"); + assertEquals(rowCount.get(), 10, "should process 8 rows data"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "show subscriptions after unsubscription"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + AWAIT.untilAsserted( + () -> { + // Consumption data: Progress is not retained after canceling and re-subscribing. Full + // synchronization. + assertEquals(onReceiveCount.get(), 5, "should 5 tsfiles include 2 duplicates"); + assertEquals(rowCount.get(), 25, "should process 20 rows data, include "); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/format/IoTDBTestPushConsumeTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/format/IoTDBTestPushConsumeTsfileIT.java new file mode 100644 index 000000000000..9c43e976f3e9 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/format/IoTDBTestPushConsumeTsfileIT.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.format; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer:AFTER_CONSUME + * TsFileHandler + * pattern: db + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTestPushConsumeTsfileIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.TestPushConsumeTsfile"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_TestPushConsumeTsfile"; + private static List schemaList = new ArrayList<>(); + private static final String pattern = database + ".**"; + private static SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + + final AtomicInteger onReceiveCount = new AtomicInteger(0); + final AtomicInteger rowCount = new AtomicInteger(0); + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("DB_TsFile_specify_target_dir_consumer") + .consumerGroupId("push_format") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + onReceiveCount.incrementAndGet(); + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + Path path = new Path(device, "s_0", true); + QueryDataSet dataset = + reader.query(QueryExpression.create(Collections.singletonList(path), null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + String sql = "select count(s_0) from " + device; + System.out.println("src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertEquals(rowCount.get(), 8, "should process 8 rows data"); + }); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "After cancellation, show subscriptions"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + System.out.println("src: " + getCount(session_src, sql)); + AWAIT.untilAsserted( + () -> { + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + assertEquals(rowCount.get(), 20, "should process 12 rows data, include "); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBLooseAllTsDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBLooseAllTsDatasetPushConsumerIT.java new file mode 100644 index 000000000000..6a34c24f0817 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBLooseAllTsDatasetPushConsumerIT.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: ts + * loose-range: all + * mode: live + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBLooseAllTsDatasetPushConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.LooseAllTsDatasetPushConsumer"; + private static final String database2 = "root.LooseAllTsDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_LooseAllTsDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + private String pattern = device + ".**"; + private static SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-02-13T08:00:02+08:00", + false, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_ALL_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-02-13T08:00:02+08:00"; + + // Write data before subscribing + insert_data(1704038399000L, device); // 2023-12-31 23:59:59+08:00 + insert_data(1704038399000L, device2); // 2023-12-31 23:59:59+08:00 + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println("LooseAllTsDatasetPushConsumer src1: " + getCount(session_src, sql)); + + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("device_accurate_dataset_push_snapshot") + .consumerGroupId("loose_range_all") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + insert_data(1706745600000L, device); // 2024-02-01 08:00:00+08:00 + insert_data(1706745600000L, device2); // 2024-02-01 08:00:00+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println("LooseAllTsDatasetPushConsumer src2: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 14, "select count(s_0) from " + device, "Consumption data: s_0 " + device); + check_count_non_strict( + 14, "select count(s_1) from " + device, "Consumption data: s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "Consumption data: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "Consumption data: s_1 " + device2); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println("LooseAllTsDatasetPushConsumer src3: " + getCount(session_src, sql)); + + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 16, "select count(s_0) from " + device, "consume data again: s_0 " + device); + check_count_non_strict( + 16, "select count(s_1) from " + device, "Consumption data: s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "Consumption data: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "Consumption data: s_1 " + device2); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBLooseAllTsDatasetPushConsumerSnapshotIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBLooseAllTsDatasetPushConsumerSnapshotIT.java new file mode 100644 index 000000000000..0204d2510993 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBLooseAllTsDatasetPushConsumerSnapshotIT.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: ts + * loose-range: all + * mode: snapshot + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBLooseAllTsDatasetPushConsumerSnapshotIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.LooseAllTsDatasetPushConsumerSnapshot"; + private static final String database2 = "root.LooseAllTsDatasetPushConsumerSnapshot"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_LooseAllTsDatasetPushConsumerSnapshot"; + private static List schemaList = new ArrayList<>(); + private String pattern = device + ".**"; + private static SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-02-13T08:00:02+08:00", + false, + TopicConstant.MODE_SNAPSHOT_VALUE, + TopicConstant.LOOSE_RANGE_ALL_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-02-13T08:00:02+08:00"; + + // Write data before subscribing + insert_data(1704038399000L, device); // 2023-12-31 23:59:59+08:00 + insert_data(1704038399000L, device2); // 2023-12-31 23:59:59+08:00 + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println("src: " + getCount(session_src, sql)); + + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("device_accurate_dataset_push_snapshot") + .consumerGroupId("loose_range_all") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + insert_data(1706745600000L, device); // 2024-02-01 08:00:00+08:00 + insert_data(1706745600000L, device2); // 2024-02-01 08:00:00+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println("src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 9, "select count(s_0) from " + device, "Consumption data: s_0 " + device); + check_count_non_strict( + 9, "select count(s_1) from " + device, "Consumption data: s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "Consumption data: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "Consumption data: s_1 " + device2); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println("src: " + getCount(session_src, sql)); + + // Consumption data: Progress is not retained after cancellation and re-subscription. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 11, "select count(s_0) from " + device, "consume data again: s_0 " + device); + check_count_non_strict( + 11, "select count(s_1) from " + device, "Consumption data: s_1 " + device); + check_count(0, "select count(s_0) from " + device2, "Consumption data: s_0 " + device2); + check_count(0, "select count(s_1) from " + device2, "Consumption data: s_1 " + device2); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBLooseAllTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBLooseAllTsfilePushConsumerIT.java new file mode 100644 index 000000000000..4b0adb059e4a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBLooseAllTsfilePushConsumerIT.java @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * push consumer + * mode: live + * pattern: db + * loose-range: all + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBLooseAllTsfilePushConsumerIT extends AbstractSubscriptionRegressionIT { + private String database = "root.LooseAllTsfilePushConsumer"; + private String device = database + ".d_0"; + private String device2 = database + ".d_1"; + private String pattern = database + ".**"; + private String topicName = "topic_LooseAllTsfilePushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-03-31T00:00:00+08:00", + true, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_ALL_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-03-31T00:00:00+08:00"; + + List rowCounts = new ArrayList<>(2); + rowCounts.add(new AtomicInteger(0)); + rowCounts.add(new AtomicInteger(0)); + final AtomicInteger onReceive = new AtomicInteger(0); + + // Write data before subscribing + insert_data(1704038396000L, device); // 2023-12-31 23:59:56+08:00 + insert_data(1704038396000L, device2); // 2023-12-31 23:59:56+08:00 + session_src.executeNonQueryStatement("flush;"); + + List paths = new ArrayList<>(2); + paths.add(device); + paths.add(device2); + + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("time_range_ts_tsfile_push") + .consumerGroupId("loose_range_all") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + onReceive.addAndGet(1); + TsFileReader reader = message.getTsFileHandler().openReader(); + for (int i = 0; i < 2; i++) { + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(new Path(paths.get(i), "s_0", true)), + null)); + while (dataset.hasNext()) { + rowCounts.get(i).addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + } + System.out.println( + FORMAT.format(new Date()) + + " " + + rowCounts.get(0).get() + + "," + + rowCounts.get(1).get()); + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCounts.get(0).get(), 3); + assertGte(rowCounts.get(1).get(), 3); + }); + + insert_data(System.currentTimeMillis(), device); // now, not in range + insert_data(System.currentTimeMillis(), device2); // now, not in range + session_src.executeNonQueryStatement("flush;"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCounts.get(0).get(), 3); + assertGte(rowCounts.get(1).get(), 3); + }); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCounts.get(0).get(), 8); + assertGte(rowCounts.get(1).get(), 8); + }); + + insert_data(1711814398000L, device); // 2024-03-30 23:59:58+08:00 + insert_data(1711814398000L, device2); // 2024-03-30 23:59:58+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCounts.get(0).get(), 10); + assertGte(rowCounts.get(1).get(), 10); + }); + + insert_data(1711900798000L, device); // 2024-03-31 23:59:58+08:00, not in range + insert_data(1711900798000L, device2); // 2024-03-31 23:59:58+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCounts.get(0).get(), 10, "Inserted data is out of range"); + assertGte(rowCounts.get(1).get(), 10); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBPathLooseDeviceTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBPathLooseDeviceTsfilePushConsumerIT.java new file mode 100644 index 000000000000..878608c16f57 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBPathLooseDeviceTsfilePushConsumerIT.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * loose range: path + * pattern: device + * push consumer + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBPathLooseDeviceTsfilePushConsumerIT extends AbstractSubscriptionRegressionIT { + private String database = "root.PathLooseDeviceTsfilePushConsumer"; + private String device = database + ".d_0"; + private String device2 = database + ".d_1"; + private String pattern = device + ".**"; + private String topicName = "topic_PathLooseDeviceTsfilePushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-03-31T00:00:00+08:00", + true, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_PATH_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + final AtomicInteger onReceive = new AtomicInteger(0); + List rowCounts = new ArrayList<>(2); + rowCounts.add(new AtomicInteger(0)); + rowCounts.add(new AtomicInteger(0)); + + List paths = new ArrayList<>(2); + paths.add(new Path(device, "s_0", true)); + paths.add(new Path(device2, "s_0", true)); + + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-03-31T00:00:00+08:00"; + + // Write data before subscribing + insert_data(1704038396000L, device); // 2023-12-31 23:59:56+08:00 + insert_data(1704038396000L, device2); // 2023-12-31 23:59:56+08:00 + session_src.executeNonQueryStatement("flush;"); + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("time_range_accurate_device_tsfile_push") + .consumerGroupId("loose_range_path") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + onReceive.addAndGet(1); + TsFileReader reader = message.getTsFileHandler().openReader(); + for (int i = 0; i < 2; i++) { + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(paths.get(i)), null)); + while (dataset.hasNext()) { + rowCounts.get(i).addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + } + + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + System.out.println(FORMAT.format(new Date()) + " src :" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceive.get(), 1); + assertGte(rowCounts.get(0).get(), 3, "Write data before subscription" + device); + assertGte(rowCounts.get(0).get(), 3, "Write data before subscription" + device2); + }); + + insert_data(System.currentTimeMillis(), device); // now, not in range + insert_data(System.currentTimeMillis(), device2); // now, not in range + session_src.executeNonQueryStatement("flush;"); + System.out.println(FORMAT.format(new Date()) + " src :" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceive.get(), 1); + assertGte(rowCounts.get(0).get(), 3, "Write out-of-range data" + device); + assertGte(rowCounts.get(0).get(), 3, "Write out-of-range data" + device2); + }); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println(FORMAT.format(new Date()) + " src :" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceive.get(), 2); + assertGte(rowCounts.get(0).get(), 8, "write data" + device); + assertGte(rowCounts.get(0).get(), 8, "write data " + device2); + }); + + insert_data(1711814398000L, device); // 2024-03-30 23:59:58+08:00 + insert_data(1711814398000L, device2); // 2024-03-30 23:59:58+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println(FORMAT.format(new Date()) + " src :" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceive.get(), 3); + assertGte(rowCounts.get(0).get(), 10, "Write data: end boundary at " + device); + assertGte(rowCounts.get(0).get(), 10, "Write data: end boundary at " + device2); + }); + + insert_data(1711900798000L, device); // 2024-03-31 23:59:58+08:00 + insert_data(1711900798000L, device2); // 2024-03-31 23:59:58+08:00 + System.out.println(FORMAT.format(new Date()) + " src :" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceive.get(), 3); + assertGte(rowCounts.get(0).get(), 10, "Write data: > end " + device); + assertGte(rowCounts.get(0).get(), 10, "Write data: > end " + device2); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBPathLooseTsDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBPathLooseTsDatasetPushConsumerIT.java new file mode 100644 index 000000000000..e441a853e161 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBPathLooseTsDatasetPushConsumerIT.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * mode: DataSet + * pattern: ts + * loose-range: path + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBPathLooseTsDatasetPushConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.PathLooseTsDatasetPushConsumer"; + private static final String database2 = "root.PathLooseTsDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_PathLooseTsDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = database + ".d_0.s_0"; + private static SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-02-13T08:00:02+08:00", + false, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_PATH_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-02-13T08:00:02+08:00"; + // Write data before subscribing + insert_data(1704038399000L, device); // 2023-12-31 23:59:59+08:00 + insert_data(1704038399000L, device2); // 2023-12-31 23:59:59+08:00 + + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts_accurate_dataset_consumer") + .consumerGroupId("loose_range_path") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 4, + "select count(s_0) from " + device, + "Subscribe before writing data: s_0 " + device); + check_count( + 0, + "select count(s_1) from " + device, + "Subscribe before writing data: s_1 " + device); + check_count( + 0, + "select count(s_0) from " + device2, + "Subscribe before writing data: s_0 " + device2); + check_count( + 0, + "select count(s_1) from " + device2, + "Subscribe before writing data: s_1 " + device2); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption data:d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 9, "select count(s_0) from " + device, "Consumption data: s_0" + device); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1" + device); + check_count(0, "select count(s_0) from " + device2, "Consumption data: s_0" + device2); + check_count(0, "select count(s_1) from " + device2, "Consumption data: s_1" + device2); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption Data: d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + insert_data(System.currentTimeMillis(), device); // not in range + insert_data(System.currentTimeMillis(), device2); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 9, "select count(s_0) from " + device, "Out-of-range data: s_0" + device); + check_count(0, "select count(s_1) from " + device, "Out-of-range data: s_1" + device); + check_count(0, "select count(s_0) from " + device2, "Out-of-range data: s_0" + device2); + check_count(0, "select count(s_1) from " + device2, "Out-of-range data: s_1" + device2); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption data:d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + // Consumption data: Progress is not preserved if you unsubscribe and then resubscribe. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 11, "select count(s_0) from " + device, "consume data again:s_0" + device); + check_count(0, "select count(s_1) from " + device, "Consumption Data: s_1" + device); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBPathLooseTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBPathLooseTsfilePushConsumerIT.java new file mode 100644 index 000000000000..157212cb1788 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBPathLooseTsfilePushConsumerIT.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * loose range: path + * push consumer + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBPathLooseTsfilePushConsumerIT extends AbstractSubscriptionRegressionIT { + private String database = "root.PathLooseTsfilePushConsumer"; + private String device = database + ".d_0"; + private String device2 = database + ".d_1"; + private String pattern = database + ".**"; + private String topicName = "topic_PathLooseTsfilePushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-03-31T00:00:00+08:00", + true, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_PATH_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + final AtomicInteger rowCount = new AtomicInteger(0); + final AtomicInteger onReceive = new AtomicInteger(0); + + // Write data before subscribing + insert_data(1704038396000L, device); // 2023-12-31 23:59:56+08:00 + insert_data(1704038396000L, device2); // 2023-12-31 23:59:56+08:00 + session_src.executeNonQueryStatement("flush;"); + + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("time_range_accurate_db_tsfile_push") + .consumerGroupId("loose_range_path") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + onReceive.addAndGet(1); + TsFileReader reader = message.getTsFileHandler().openReader(); + List paths = new ArrayList<>(2); + for (int i = 0; i < 2; i++) { + paths.add(new Path(device, "s_" + i, true)); + } + QueryDataSet dataset = reader.query(QueryExpression.create(paths, null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCount.get(), 3); + }); + + insert_data(System.currentTimeMillis(), device); // now, not in range + insert_data(System.currentTimeMillis(), device2); // now, not in range + session_src.executeNonQueryStatement("flush;"); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCount.get(), 3); + }); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement("flush;"); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCount.get(), 8); + }); + + insert_data(1711814398000L, device); // 2024-03-30 23:59:58+08:00 + insert_data(1711814398000L, device2); // 2024-03-30 23:59:58+08:00 + session_src.executeNonQueryStatement("flush;"); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCount.get(), 10); + }); + + insert_data(1711900798000L, device); // 2024-03-31 23:59:58+08:00 + insert_data(1711900798000L, device2); // 2024-03-31 23:59:58+08:00 + + AWAIT.untilAsserted( + () -> { + assertGte(rowCount.get(), 10); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBPathTsLooseDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBPathTsLooseDatasetPushConsumerIT.java new file mode 100644 index 000000000000..0ad43be14617 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBPathTsLooseDatasetPushConsumerIT.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: ts + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBPathTsLooseDatasetPushConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.PathTsLooseDatasetPushConsumer"; + private static final String database2 = "root.PathTsLooseDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_PathTsLooseDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = database + ".d_0.s_0"; + public static SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-02-13T08:00:02+08:00", + false, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_PATH_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1704038399000L, device); // 2023-12-31 23:59:59+08:00 + insert_data(1704038399000L, device2); // 2023-12-31 23:59:59+08:00 + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts_accurate_dataset_consumer") + .consumerGroupId("loose_range_path") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 9, "select count(s_0) from " + device, "Consumption Data: s_0" + device); + check_count_non_strict( + 0, "select count(s_1) from " + device, "Consumption data: s_1" + device); + check_count(0, "select count(s_0) from " + device2, "Consumption data: s_0" + device2); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1" + device2); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption data:d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + + // Consumption data: Progress is not retained after unsubscribing and resubscribing. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 11, "select count(s_0) from " + device, "consume data again:s_0" + device); + check_count(0, "select count(s_1) from " + device, "Consumption Data: s_1" + device); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBTimeLooseTsDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBTimeLooseTsDatasetPushConsumerIT.java new file mode 100644 index 000000000000..9d366f3d9b28 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBTimeLooseTsDatasetPushConsumerIT.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: ts + * loose-range: time + * live + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTimeLooseTsDatasetPushConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.TimeLooseTsDatasetPushConsumer"; + private static final String database2 = "root.TimeLooseTsDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_TimeLooseTsDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static String pattern = device + ".s_0"; + private static SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-02-13T08:00:02+08:00", + false, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_TIME_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-02-13T08:00:02+08:00"; + + // Write data before subscribing + insert_data(1704038399000L, device); // 2023-12-31 23:59:59+08:00 + insert_data(1704038399000L, device2); // 2023-12-31 23:59:59+08:00 + session_src.executeNonQueryStatement("flush;"); + + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("push_dataset_ts_dataset_consumer") + .consumerGroupId("loose_range_time") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 9, "select count(s_0) from " + device, "Consumption data: s_0" + device); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1" + device); + check_count(0, "select count(s_0) from " + device2, "Consumption data: s_0" + device2); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1" + device2); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption Data: d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count_non_strict( + 11, "select count(s_0) from " + device, "Consumption data: s_0" + device); + check_count(0, "select count(s_1) from " + device, "Consumption Data: s_1" + device); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBTimeLooseTsTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBTimeLooseTsTsfilePushConsumerIT.java new file mode 100644 index 000000000000..a3dd38dea192 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBTimeLooseTsTsfilePushConsumerIT.java @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * mode: live + * loose-range:path + * format: tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTimeLooseTsTsfilePushConsumerIT extends AbstractSubscriptionRegressionIT { + private String database = "root.TimeLooseTsTsfilePushConsumer"; + private String device = database + ".d_0"; + private String device2 = database + ".d_1"; + private String pattern = device + ".s_0"; + private String topicName = "topic_TimeLooseTsTsfilePushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-03-31T00:00:00+08:00", + true, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_TIME_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + final AtomicInteger onReceive = new AtomicInteger(0); + List rowCounts = new ArrayList<>(4); + for (int i = 0; i < 4; i++) { + rowCounts.add(new AtomicInteger(0)); + } + + List paths = new ArrayList<>(4); + paths.add(new Path(device, "s_0", true)); + paths.add(new Path(device, "s_1", true)); + paths.add(new Path(device2, "s_0", true)); + paths.add(new Path(device2, "s_1", true)); + + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-03-31T00:00:00+08:00"; + + // Write data before subscribing + insert_data(1704038396000L, device); // 2023-12-31 23:59:56+08:00 + insert_data(1704038396000L, device2); // 2023-12-31 23:59:56+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("time_range_accurate_ts_tsfile_push") + .consumerGroupId("loose_range_time") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + onReceive.addAndGet(1); + TsFileReader reader = message.getTsFileHandler().openReader(); + for (int i = 0; i < 4; i++) { + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(paths.get(i)), null)); + while (dataset.hasNext()) { + rowCounts.get(i).addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + System.out.println( + FORMAT.format(new Date()) + " " + i + " " + rowCounts.get(i).get()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + assertGte( + rowCounts.get(0).get(), + 3, + device + ".s_0, subscribe before writing data start boundary"); + assertEquals( + rowCounts.get(1).get(), + 0, + device + ".s_1, Subscription before writing data start boundary"); + assertEquals( + rowCounts.get(2).get(), + 0, + device2 + ".s_0, Subscribe before writing data start boundary"); + assertEquals( + rowCounts.get(3).get(), + 0, + device2 + ".s_1, Subscription before writing data start boundary"); + }); + + insert_data(System.currentTimeMillis(), device); // now, not in range + insert_data(System.currentTimeMillis(), device2); // now, not in range + session_src.executeNonQueryStatement("flush;"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCounts.get(0).get(), 3, device + ".s_0, Write out-of-range data"); + assertEquals(rowCounts.get(1).get(), 0, device + ".s_1, Write out-of-range data"); + assertEquals(rowCounts.get(2).get(), 0, device2 + ".s_0, Write out-of-range data"); + assertEquals(rowCounts.get(3).get(), 0, device2 + ".s_1, Write out-of-range data"); + }); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCounts.get(0).get(), 8, device + ".s_0, write normal data"); + assertEquals(rowCounts.get(1).get(), 0, device + ".s_1, write normal data"); + assertEquals(rowCounts.get(2).get(), 0, device2 + ".s_0, Write normal data"); + assertEquals(rowCounts.get(3).get(), 0, device2 + ".s_1, Write normal data"); + }); + + insert_data(1711814398000L, device); // 2024-03-30 23:59:58+08:00 + insert_data(1711814398000L, device2); // 2024-03-30 23:59:58+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCounts.get(0).get(), 10, device + ".s_0, write end boundary data"); + assertEquals(rowCounts.get(1).get(), 0, device + ".s_1, write end boundary data"); + assertEquals(rowCounts.get(2).get(), 0, device2 + ".s_0, write end boundary data"); + assertEquals(rowCounts.get(3).get(), 0, device2 + ".s_1, write end boundary data"); + }); + + insert_data(1711900798000L, device); // 2024-03-31 23:59:58+08:00 + insert_data(1711900798000L, device2); // 2024-03-31 23:59:58+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println(FORMAT.format(new Date()) + " src: " + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCounts.get(0).get(), 10, device + ".s_0, Write data outside end range"); + assertEquals(rowCounts.get(1).get(), 0, device + ".s_0, Write data outside of end range"); + assertEquals(rowCounts.get(2).get(), 0, device + ".s_0, Write data outside of end range"); + assertEquals(rowCounts.get(3).get(), 0, device + ".s_0, Write data outside of end range"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBTimeLooseTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBTimeLooseTsfilePushConsumerIT.java new file mode 100644 index 000000000000..1057c268d6b8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBTimeLooseTsfilePushConsumerIT.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTimeLooseTsfilePushConsumerIT extends AbstractSubscriptionRegressionIT { + private String database = "root.TimeLooseTsfilePushConsumer"; + private String device = database + ".d_0"; + private String device2 = database + ".d_1"; + private String pattern = device + ".**"; + private String topicName = "topic_TimeLooseTsfilePushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-03-31T00:00:00+08:00", + true, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_TIME_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-03-31T00:00:00+08:00"; + + final AtomicInteger rowCount = new AtomicInteger(0); + final AtomicInteger onReceive = new AtomicInteger(0); + + // Write data before subscribing + insert_data(1704038396000L, device); // 2023-12-31 23:59:56+08:00 + insert_data(1704038396000L, device2); // 2023-12-31 23:59:56+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("time_range_accurate_db_tsfile_push") + .consumerGroupId("loose_range_time") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + onReceive.addAndGet(1); + TsFileReader reader = message.getTsFileHandler().openReader(); + List paths = new ArrayList<>(2); + for (int i = 0; i < 2; i++) { + paths.add(new Path(device, "s_" + i, true)); + } + QueryDataSet dataset = reader.query(QueryExpression.create(paths, null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + System.out.println("onReceive=" + onReceive.get()); + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceive.get(), 1); + assertGte(rowCount.get(), 3); + }); + + insert_data(System.currentTimeMillis(), device); // now, not in range + insert_data(System.currentTimeMillis(), device2); // now, not in range + session_src.executeNonQueryStatement("flush;"); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceive.get(), 1); + assertGte(rowCount.get(), 3); + }); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceive.get(), 2); + assertGte(rowCount.get(), 8); + }); + + insert_data(1711814398000L, device); // 2024-03-30 23:59:58+08:00 + insert_data(1711814398000L, device2); // 2024-03-30 23:59:58+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceive.get(), 3); + assertGte(rowCount.get(), 10); + }); + + insert_data(1711900798000L, device); // 2024-03-31 23:59:58+08:00 + insert_data(1711900798000L, device2); // 2024-03-31 23:59:58+08:00 + session_src.executeNonQueryStatement("flush;"); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceive.get(), 3); + assertGte(rowCount.get(), 10); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBTimeTsLooseDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBTimeTsLooseDatasetPushConsumerIT.java new file mode 100644 index 000000000000..f5e98108d8c0 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/loose_range/IoTDBTimeTsLooseDatasetPushConsumerIT.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.loose_range; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: ts + * time loose + * live + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTimeTsLooseDatasetPushConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.TimeTsLooseDatasetPushConsumer"; + private static final String database2 = "root.TimeTsLooseDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_TimeTsLooseDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = device + ".s_0"; + public static SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-02-13T08:00:02+08:00", + false, + TopicConstant.MODE_LIVE_VALUE, + TopicConstant.LOOSE_RANGE_TIME_VALUE); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + device2 + "(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } finally { + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + } + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (row + 1) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1704038399000L, device); // 2023-12-31 23:59:59+08:00 + insert_data(1704038399000L, device2); // 2023-12-31 23:59:59+08:00 + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + session_src.executeNonQueryStatement("flush;"); + + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("push_dataset_ts_dataset_consumer") + .consumerGroupId("loose_range_time") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + assertGte( + getCount(session_dest, "select count(s_0) from " + device), + 9, + "Consumption data: s_0 " + device); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1" + device); + check_count(0, "select count(s_0) from " + device2, "Consumption data: s_0" + device2); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1" + device2); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption data:d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after re-subscribing"); + + insert_data(1707782400000L, device); // 2024-02-13 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement("flush;"); + + // Consumption data: Progress is not retained after canceling and re-subscribing. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + assertGte( + getCount(session_dest, "select count(s_0) from " + device), + 11, + "re-subscribing s_0 count=" + + getCount(session_dest, "select count(s_0) from " + device)); + check_count(0, "select count(s_1) from " + device, "Consumption Data: s_1" + device); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/mode/IoTDBSnapshotTSPatternDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/mode/IoTDBSnapshotTSPatternDatasetPushConsumerIT.java new file mode 100644 index 000000000000..3e45c599a8a9 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/mode/IoTDBSnapshotTSPatternDatasetPushConsumerIT.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.mode; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: ts + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBSnapshotTSPatternDatasetPushConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.SnapshotTSPatternDatasetPushConsumer"; + private static final String database2 = "root.SnapshotTSPatternDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_SnapshotTSPatternDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + private static final String pattern = device + ".s_0"; + private static SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, false, TopicConstant.MODE_SNAPSHOT_VALUE, null); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts_dataset_snapshot") + .consumerGroupId("push_mode") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions(topicName).forEach(System.out::println); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after subscription"); + + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + insert_data(System.currentTimeMillis()); + + AWAIT.untilAsserted( + () -> { + check_count(4, "select count(s_0) from " + device, "Consumption data: s_0 " + device); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1 " + device); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption Data:d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after re-subscribing"); + + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count(12, "select count(s_0) from " + device, "consume data again:s_0 " + device); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1 " + device); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/mode/IoTDBSnapshotTSPatternTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/mode/IoTDBSnapshotTSPatternTsfilePushConsumerIT.java new file mode 100644 index 000000000000..3197d343456a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/mode/IoTDBSnapshotTSPatternTsfilePushConsumerIT.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.mode; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * TsFile + * pattern: ts + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBSnapshotTSPatternTsfilePushConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.SnapshotTSPatternTsfilePushConsumer"; + private static final String database2 = "root.SnapshotTSPatternTsfilePushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_SnapshotTSPatternTsfilePushConsumer"; + private static List schemaList = new ArrayList<>(); + private static final String pattern = device + ".s_0"; + private static SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, + pattern, + "2024-01-01T00:00:00+08:00", + "2024-03-31T00:00:00+08:00", + true, + TopicConstant.MODE_SNAPSHOT_VALUE, + null); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-03-31T00:00:00+08:00"; + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + final AtomicInteger onReceiveCount = new AtomicInteger(0); + List rowCounts = new ArrayList<>(4); + for (int i = 0; i < 4; i++) { + rowCounts.add(new AtomicInteger(0)); + } + + Path path_d0s0 = new Path(device, "s_0", true); + Path path_d0s1 = new Path(device, "s_1", true); + Path path_d1s0 = new Path(database + ".d_1", "s_0", true); + Path path_other_d2 = new Path(database2 + ".d_2", "s_0", true); + List paths = new ArrayList<>(4); + paths.add(path_d0s0); + paths.add(path_d0s1); + paths.add(path_d1s0); + paths.add(path_other_d2); + + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts_tsfile_snapshot") + .consumerGroupId("push_mode") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + onReceiveCount.incrementAndGet(); + System.out.println("onReceiveCount=" + onReceiveCount.get()); + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + for (int i = 0; i < 4; i++) { + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(paths.get(i)), null)); + while (dataset.hasNext()) { + rowCounts.get(i).addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + System.out.println( + FORMAT.format(new Date()) + + " rowCounts_" + + i + + ":" + + rowCounts.get(i).get()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions(topicName).forEach(System.out::println); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after subscription"); + + insert_data(1707609600000L); // 2024-02-11 08:00:00+08:00 + insert_data(System.currentTimeMillis()); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceiveCount.get(), 1, "receive files"); + assertEquals(rowCounts.get(0).get(), 5, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 0, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 0, database + ".d_1.s_0"); + assertEquals(rowCounts.get(3).get(), 0, database2 + ".d_2.s_0"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals( + subs.getSubscriptions(topicName).size(), 1, "show subscriptions after re-subscribing"); + + AWAIT.untilAsserted( + () -> { + assertGte(onReceiveCount.get(), 2, "receive files over 2"); + assertEquals(rowCounts.get(0).get(), 15, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 0, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 0, database + ".d_1.s_0"); + assertEquals(rowCounts.get(3).get(), 0, database2 + ".d_2.s_0"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/multi/IoTDBConsumer2With1TopicShareProcessDataSetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/multi/IoTDBConsumer2With1TopicShareProcessDataSetIT.java new file mode 100644 index 000000000000..e59088d6a287 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/multi/IoTDBConsumer2With1TopicShareProcessDataSetIT.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * pattern: db + * Dataset + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBConsumer2With1TopicShareProcessDataSetIT + extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.Consumer2With1TopicShareProcessDataSet"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_Consumer2With1TopicShareProcessDataSet"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = database + ".**"; + private static SubscriptionPushConsumer consumer; + private static SubscriptionPushConsumer consumer2; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest2.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest2.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + consumer2.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + // insert_data(1706659200000L); + Thread thread = + new Thread( + () -> { + long timestamp = 1706659200000L; // 2024-01-31 08:00:00+08:00 + for (int i = 0; i < 20; i++) { + try { + insert_data(timestamp); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } + timestamp += 20000; + } + }); + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("db_dataset_consumer_1") + .consumerGroupId("push_multi_test") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + consumer.subscribe(topicName); + consumer2 = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("db_dataset_consumer_2") + .consumerGroupId("push_multi_test") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest2.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer2.open(); + consumer2.subscribe(topicName); + thread.start(); + + thread.join(); + System.out.println("After subscription:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + String sql = "select count(s_0) from " + device; + // The first 5 entries may have duplicate data + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertTrue(getCount(session_dest, sql) >= 0, "first consumer"); + assertTrue(getCount(session_dest2, sql) >= 0, "second Consumer"); + assertEquals( + getCount(session_dest, sql) + getCount(session_dest2, sql), + getCount(session_src, sql), + "share process"); + }); + System.out.println(FORMAT.format(new Date()) + " dest:" + getCount(session_dest, sql)); + System.out.println(FORMAT.format(new Date()) + " dest2:" + getCount(session_dest2, sql)); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/multi/IoTDBConsumer2With1TopicShareProcessTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/multi/IoTDBConsumer2With1TopicShareProcessTsfileIT.java new file mode 100644 index 000000000000..cd33091e4a08 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/multi/IoTDBConsumer2With1TopicShareProcessTsfileIT.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * pattern: db + * tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBConsumer2With1TopicShareProcessTsfileIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.Consumer2With1TopicShareProcessTsfile"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_Consumer2With1TopicShareProcessTsfile"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = database + ".**"; + private static SubscriptionPushConsumer consumer; + private static SubscriptionPushConsumer consumer2; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + consumer2.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + Thread thread = + new Thread( + () -> { + long timestamp = 1706659200000L; // 2024-01-31 08:00:00+08:00 + for (int i = 0; i < 20; i++) { + try { + insert_data(timestamp); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } + timestamp += 20000; + } + }); + AtomicInteger rowCount1 = new AtomicInteger(0); + AtomicInteger rowCount2 = new AtomicInteger(0); + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("db_tsfile_consumer_1") + .consumerGroupId("push_multi_Consumer2With1TopicShareProcessTsfile") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + List paths = new ArrayList<>(2); + for (int i = 0; i < 2; i++) { + paths.add(new Path(device, "s_" + i, true)); + } + QueryDataSet dataset = reader.query(QueryExpression.create(paths, null)); + while (dataset.hasNext()) { + rowCount1.addAndGet(1); + RowRecord next = dataset.next(); + // + // System.out.println(next.getTimestamp()+","+next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + consumer.subscribe(topicName); + consumer2 = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("db_tsfile_consumer_2") + .consumerGroupId("push_multi_Consumer2With1TopicShareProcessTsfile") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + List paths = new ArrayList<>(2); + for (int i = 0; i < 2; i++) { + paths.add(new Path(device, "s_" + i, true)); + } + QueryDataSet dataset = reader.query(QueryExpression.create(paths, null)); + while (dataset.hasNext()) { + rowCount2.addAndGet(1); + RowRecord next = dataset.next(); + // + // System.out.println(next.getTimestamp()+","+next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer2.open(); + consumer2.subscribe(topicName); + thread.start(); + + thread.join(); + System.out.println("After subscription:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + // The first 5 entries may have duplicate data + String sql = "select count(s_0) from " + device; + System.out.println("src " + getCount(session_src, sql)); + System.out.println("rowCount1.get()=" + rowCount1.get()); + System.out.println("rowCount2.get()=" + rowCount2.get()); + + AWAIT.untilAsserted( + () -> { + assertTrue(rowCount1.get() >= 0, "first consumer"); + assertTrue(rowCount2.get() >= 0, "second Consumer"); + assertGte(rowCount1.get() + rowCount2.get(), getCount(session_src, sql), "share process"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/multi/IoTDBMultiGroupVsMultiConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/multi/IoTDBMultiGroupVsMultiConsumerIT.java new file mode 100644 index 000000000000..d573afce21eb --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/multi/IoTDBMultiGroupVsMultiConsumerIT.java @@ -0,0 +1,603 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * pattern: device, different db + * |c0|t0|g1| tsfile databasePrefix+"0.**", "2024-01-01T00:00:00+08:00", "2024-03-31T23:59:59+08:00" + * |c1|t0|g1| tsfile + * |c2|t1|g1| dataset(dest) databasePrefix+"1.**" + * |c3|t1|g1| dataset(dest2) + * |c4|t2,t3|g2| dataset(dest) databasePrefix+"2.**", "now", null; databasePrefix+"3.**", null, "now" + * |c5|t3,t4|g2| dataset(dest2) databasePrefix+"4.**", null, "2024-03-31T23:59:59+08:00" + * |c6|t2,t4|g2| dataset(dest) + * |c7|t0,t3|g3| dataset(dest)/tsfile + * |c8|t6|g3| tsfile databasePrefix+"6.**", "now", null, + * |c9|t0,t3|g3| dataset(dest2)/tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBMultiGroupVsMultiConsumerIT extends AbstractSubscriptionRegressionIT { + + private String topicNamePrefix = "topic_pushMultiGroupVsMultiConsumer_"; + private String databasePrefix = "root.test.pushMultiGroupVsMultiConsumer_"; + private int tsCount = 10; + private int consumertCount = 10; + private static List schemaList = new ArrayList<>(); + + private List consumers = new ArrayList<>(consumertCount); + private AtomicInteger rowCount00 = new AtomicInteger(0); + private AtomicInteger rowCount10 = new AtomicInteger(0); + private AtomicInteger rowCount70 = new AtomicInteger(0); + private AtomicInteger rowCount90 = new AtomicInteger(0); + private AtomicInteger rowCount6 = new AtomicInteger(0); + private String sql1 = "select count(s_0) from " + databasePrefix + "1.d_0"; + private String sql2 = "select count(s_0) from " + databasePrefix + "2.d_0"; + private String sql3 = "select count(s_0) from " + databasePrefix + "3.d_0"; + private String sql4 = "select count(s_0) from " + databasePrefix + "4.d_0"; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + for (int i = 0; i < tsCount; i++) { + createDB(databasePrefix + i); + } + createTopic_s( + topicNamePrefix + 0, + databasePrefix + "0.**", + "2024-01-01T00:00:00+08:00", + "2024-03-31T23:59:59+08:00", + true); + createTopic_s(topicNamePrefix + 1, databasePrefix + "1.**", null, null, false); + + createTopic_s(topicNamePrefix + 2, databasePrefix + "2.**", "now", null, false); + createTopic_s(topicNamePrefix + 3, databasePrefix + "3.**", null, "now", false); + createTopic_s( + topicNamePrefix + 4, databasePrefix + "4.**", null, "2024-03-31T23:59:59+08:00", false); + + createTopic_s(topicNamePrefix + 6, databasePrefix + "6.**", "now", null, true); + + createTopic_s( + topicNamePrefix + 5, databasePrefix + "5.**", "2024-01-01T00:00:00+08:00", null, false); + createTopic_s( + topicNamePrefix + 7, databasePrefix + "7.**", null, "2024-03-31T23:59:59+08:00", true); + createTopic_s(topicNamePrefix + 8, databasePrefix + "8.**", null, "now", true); + createTopic_s( + topicNamePrefix + 9, databasePrefix + "9.**", "2024-01-01T00:00:00+08:00", null, true); + + subs.getTopics().forEach(System.out::println); + } + + @Override + @After + public void tearDown() throws Exception { + System.out.println(databasePrefix + "1.d_0:[src]" + getCount(session_src, sql1)); + System.out.println(databasePrefix + "1.d_0:[dest]" + getCount(session_dest, sql1)); + System.out.println(databasePrefix + "1.d_0:[dest2]" + getCount(session_dest2, sql1)); + System.out.println(databasePrefix + "2.d_0:[src]" + getCount(session_src, sql2)); + System.out.println(databasePrefix + "2.d_0:[dest]" + getCount(session_dest, sql2)); + System.out.println(databasePrefix + "2.d_0:[dest2]" + getCount(session_dest2, sql2)); + System.out.println(databasePrefix + "3.d_0:[src]" + getCount(session_src, sql3)); + System.out.println(databasePrefix + "3.d_0:[dest]" + getCount(session_dest, sql3)); + System.out.println(databasePrefix + "3.d_0:[dest2]" + getCount(session_dest2, sql3)); + System.out.println(databasePrefix + "4.d_0:[src]" + getCount(session_src, sql4)); + System.out.println(databasePrefix + "4.d_0:[dest]" + getCount(session_dest, sql4)); + System.out.println(databasePrefix + "4.d_0:[dest2]" + getCount(session_dest2, sql4)); + System.out.println("rowCount00.get()=" + rowCount00.get()); + System.out.println("rowCount10.get()=" + rowCount10.get()); + System.out.println("rowCount70.get()=" + rowCount70.get()); + System.out.println("rowCount90.get()=" + rowCount90.get()); + System.out.println("rowCount6.get()=" + rowCount6.get()); + for (SubscriptionPushConsumer c : consumers) { + try { + c.close(); + } catch (Exception e) { + } + } + for (int i = 0; i < tsCount; i++) { + subs.dropTopic(topicNamePrefix + i); + } + dropDB(databasePrefix); + super.tearDown(); + } + + private void insert_data(long timestamp, String device, int rows) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, rows); + int rowIndex = 0; + for (int row = 0; row < rows; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, (row + 1) * 1400 + row); + tablet.addValue(schemaList.get(1).getMeasurementId(), rowIndex, (row + 1) * 100 + 0.5); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + /*** + * |c0|t0|g1| tsfile databasePrefix+"0.**", "2024-01-01T00:00:00+08:00", "2024-03-31T23:59:59+08:00" + * |c1|t0|g1| tsfile + * |c2|t1|g1| dataset(dest) databasePrefix+"1.**" + * |c3|t1|g1| dataset(dest2) + * |c4|t2,t3|g2| dataset(dest) databasePrefix+"2.**", "now", null; databasePrefix+"3.**", null, "now" + * |c5|t3,t4|g2| dataset(dest2) databasePrefix+"4.**", null, "2024-03-31T23:59:59+08:00" + * |c6|t2,t4|g2| dataset(dest) + * |c7|t0,t3|g3| dataset(dest)/tsfile + * |c8|t6|g3| tsfile databasePrefix+"6.**", "now", null, + * |c9|t0,t3|g3| dataset(dest2)/tsfile + */ + @Test + public void do_test() + throws TException, + IoTDBConnectionException, + IOException, + StatementExecutionException, + InterruptedException { + consumers.add( + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_0") + .consumerGroupId("push_group_id_1") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList( + new Path(databasePrefix + "0.d_0", "s_0", true)), + null)); + while (dataset.hasNext()) { + rowCount00.addAndGet(1); + RowRecord next = dataset.next(); + // System.out.println("c0,g1:[rowCount00]" + + // next.getTimestamp() + "," + next.getFields()); + } + // + // System.out.println("c0,g1,rowCount00="+rowCount00.get()); + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + consumers.add( + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_1") + .consumerGroupId("push_group_id_1") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList( + new Path(databasePrefix + "0.d_0", "s_0", true)), + null)); + while (dataset.hasNext()) { + rowCount10.addAndGet(1); + RowRecord next = dataset.next(); + // + // System.out.println(databasePrefix+0+".d_0:[rowCount10]" + + // next.getTimestamp() + "," + next.getFields()); + } + // + // System.out.println("c1,g1,rowCount10="+rowCount00.get()); + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + consumers.add( + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_2") + .consumerGroupId("push_group_id_1") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + consumers.add( + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_3") + .consumerGroupId("push_group_id_1") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest2.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + consumers.add( + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_4") + .consumerGroupId("push_group_id_2") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + consumers.add( + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_5") + .consumerGroupId("push_group_id_2") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest2.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + consumers.add( + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_6") + .consumerGroupId("push_group_id_2") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + consumers.add( + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_7") + .consumerGroupId("push_group_id_3") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + final short messageType = message.getMessageType(); + if (SubscriptionMessageType.isValidatedMessageType(messageType)) { + switch (SubscriptionMessageType.valueOf(messageType)) { + case SESSION_DATA_SETS_HANDLER: + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + break; + case TS_FILE_HANDLER: + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList( + new Path(databasePrefix + "0.d_0", "s_0", true)), + null)); + while (dataset.hasNext()) { + rowCount70.addAndGet(1); + RowRecord next = dataset.next(); + // + // System.out.println(databasePrefix+"0.d_0:[rowCount70]" + + // next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + break; + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + consumers.add( + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_8") + .consumerGroupId("push_group_id_3") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList( + new Path(databasePrefix + "6.d_0", "s_0", true)), + null)); + while (dataset.hasNext()) { + rowCount6.addAndGet(1); + RowRecord next = dataset.next(); + // System.out.println(databasePrefix+6+".d_0:" + + // next.getTimestamp() + "," + next.getFields()); + } + // + // System.out.println("c8,g3,rowCount00="+rowCount6.get()); + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + consumers.add( + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("consumer_id_9") + .consumerGroupId("push_group_id_3") + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + final short messageType = message.getMessageType(); + if (SubscriptionMessageType.isValidatedMessageType(messageType)) { + switch (SubscriptionMessageType.valueOf(messageType)) { + case SESSION_DATA_SETS_HANDLER: + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest2.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + break; + case TS_FILE_HANDLER: + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList( + new Path(databasePrefix + "0.d_0", "s_0", true)), + null)); + while (dataset.hasNext()) { + rowCount90.addAndGet(1); + RowRecord next = dataset.next(); + // + // System.out.println(databasePrefix+"0.d_0:[rowCount90]" + + // next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + break; + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer()); + + for (int j = 0; j < consumers.size(); j++) { + consumers.get(j).open(); + } + + consumers.get(0).subscribe(topicNamePrefix + 0); + consumers.get(1).subscribe(topicNamePrefix + 0); + consumers.get(2).subscribe(topicNamePrefix + 1); + consumers.get(3).subscribe(topicNamePrefix + 1); + consumers.get(4).subscribe(topicNamePrefix + 2, topicNamePrefix + 3); + consumers.get(5).subscribe(topicNamePrefix + 3, topicNamePrefix + 4); + consumers.get(6).subscribe(topicNamePrefix + 2, topicNamePrefix + 4); + consumers.get(7).subscribe(topicNamePrefix + 0, topicNamePrefix + 3); + consumers.get(8).subscribe(topicNamePrefix + 6); + consumers.get(9).subscribe(topicNamePrefix + 0, topicNamePrefix + 3); + subs.getSubscriptions().forEach((System.out::println)); + + // Write data + Thread thread = + new Thread( + () -> { + long timestamp = 1706659200000L; // 2024-01-31 08:00:00+08:00 + for (int i = 0; i < 20; i++) { + for (int k = 0; k < 10; k++) { + String device = databasePrefix + k + ".d_0"; + try { + insert_data(timestamp, device, 20); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } + } + timestamp += 40000; + } + for (int i = 0; i < 10; i++) { + String device = databasePrefix + i + ".d_0"; + try { + insert_data(System.currentTimeMillis(), device, 5); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } + } + String device = databasePrefix + 2 + ".d_0"; + for (int i = 0; i < 20; i++) { + try { + insert_data(System.currentTimeMillis(), device, 5); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } + } + }); + + thread.start(); + thread.join(); + + System.out.println(databasePrefix + "1.d_0:[src]" + getCount(session_src, sql1)); + System.out.println(databasePrefix + "2.d_0:[src]" + getCount(session_src, sql2)); + System.out.println(databasePrefix + "3.d_0:[src]" + getCount(session_src, sql3)); + System.out.println(databasePrefix + "4.d_0:[src]" + getCount(session_src, sql4)); + + AWAIT.untilAsserted( + () -> { + assertGte(rowCount00.get() + rowCount10.get(), 400, "c0,c1,topic0,tsfile"); + assertEquals( + getCount(session_dest, sql1) + getCount(session_dest2, sql1), + getCount(session_src, sql1), + "c2,c3,topic1,group1"); + assertEquals( + getCount(session_dest, sql2) + getCount(session_dest2, sql2), + getCount(session_src, sql2) - 400, + "c4,c6,topic2,group2"); + final long topic3Total = getCount(session_dest, sql3) + getCount(session_dest2, sql3); + assertTrue(400 <= topic3Total && topic3Total <= 800, "c4,c5|c7,c9|topic3"); + assertEquals( + getCount(session_dest, sql4) + getCount(session_dest2, sql4), + 400, + "c5,c6,topic4,group3"); + assertGte(rowCount70.get() + rowCount90.get(), 400, "c7,c9,topic0,tsfile"); + assertEquals(rowCount6.get(), 5, "c8,topic6,tsfile"); + // assertTrue(rowCount00.get()>0); + // assertTrue(rowCount10.get()>0); + // assertTrue(rowCount70.get()>0); + // assertTrue(rowCount90.get()>0); + }); + } +} +/*** + * Expected result: + * root.test.MultiGroupVsMultiConsumer_1.d_0:[src]405 + * root.test.MultiGroupVsMultiConsumer_1.d_0:[dest]305 + * root.test.MultiGroupVsMultiConsumer_1.d_0:[dest2]100 + * root.test.MultiGroupVsMultiConsumer_2.d_0:[src]505 + * root.test.MultiGroupVsMultiConsumer_2.d_0:[dest]105 + * root.test.MultiGroupVsMultiConsumer_2.d_0:[dest2]0 + * root.test.MultiGroupVsMultiConsumer_3.d_0:[src]405 + * root.test.MultiGroupVsMultiConsumer_3.d_0:[dest]220 + * root.test.MultiGroupVsMultiConsumer_3.d_0:[dest2]300 + * root.test.MultiGroupVsMultiConsumer_4.d_0:[src]405 + * root.test.MultiGroupVsMultiConsumer_4.d_0:[dest]300 + * root.test.MultiGroupVsMultiConsumer_4.d_0:[dest2]100 + * rowCount00.get()=200 + * rowCount10.get()=200 + * rowCount70.get()=240 + * rowCount90.get()=160 + * rowCount6.get()=5 + **/ diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/multi/IoTDBOneConsumerMultiTopicsDatasetIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/multi/IoTDBOneConsumerMultiTopicsDatasetIT.java new file mode 100644 index 000000000000..31eecb78d04b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/multi/IoTDBOneConsumerMultiTopicsDatasetIT.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * 1 consumer subscribes to 2 topics: fixed time range + * dataset + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBOneConsumerMultiTopicsDatasetIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.OneConsumerMultiTopicsDataset"; + private static final String database2 = "root.OneConsumerMultiTopicsDataset"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_OneConsumerMultiTopicsDataset_1"; + private static List schemaList = new ArrayList<>(); + + private String pattern = database + ".**"; + private String pattern2 = database2 + ".**"; + private static final String device2 = database2 + ".d_1"; + private String topicName2 = "topic_OneConsumerMultiTopicsDataset_2"; + private static SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s( + topicName, pattern, "2024-01-01T00:00:00+08:00", "2024-03-01T00:00:00+08:00", false); + createTopic_s(topicName2, pattern2, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device2 + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device2 + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + subs.dropTopic(topicName2); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1707782400000L, device2); // 2024-02-13 + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1710288000000,313,6.78);"); // 2024-03-13 08:00:00+08:00 + // Subscribe + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("root_dataset_consumer") + .consumerGroupId("push_multi") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + assertEquals(subs.getSubscriptions().size(), 0, "Before subscription show subscriptions"); + + consumer.subscribe(topicName, topicName2); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 2, "show subscriptions after subscription"); + + Thread thread = + new Thread( + () -> { + try { + insert_data(System.currentTimeMillis(), device); + insert_data(System.currentTimeMillis(), device2); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + thread.start(); + thread.join(); + String sql = + "select count(s_0) from " + + device + + " where time >= 2024-01-01T00:00:00+08:00 and time <= 2024-03-01T00:00:00+08:00"; + String sql2 = "select count(s_0) from " + device2; + System.out.println(FORMAT.format(new Date()) + " src device:" + getCount(session_src, sql)); + System.out.println(FORMAT.format(new Date()) + " src device:" + getCount(session_src, sql2)); + + AWAIT.untilAsserted( + () -> { + // Consumption data + check_count(5, "select count(s_0) from " + device, "Consumption data:" + pattern); + check_count(10, "select count(s_0) from " + device2, "Consumption data:" + pattern2); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/multi/IoTDBOneConsumerMultiTopicsMixIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/multi/IoTDBOneConsumerMultiTopicsMixIT.java new file mode 100644 index 000000000000..00a3a973b110 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/multi/IoTDBOneConsumerMultiTopicsMixIT.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionMessageType; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * 1 consumer subscribes to 2 topics: historical data + * The timing of flush is very critical. If the data inside the filter and the data outside the filter are within one tsfile, they will all be extracted. + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBOneConsumerMultiTopicsMixIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.OneConsumerMultiTopicsMix"; + private static final String database2 = "root.OneConsumerMultiTopicsMix"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_OneConsumerMultiTopicsMix_1"; + private String pattern = database + ".**"; + private String pattern2 = database2 + ".**"; + private String device2 = database2 + ".d_0"; + private String topicName2 = "topic_OneConsumerMultiTopicsMix_2"; + private List schemaList = new ArrayList<>(); + private SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, false); + createTopic_s(topicName2, pattern2, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.TEXT, TSEncoding.DICTIONARY, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.TEXT, TSEncoding.DICTIONARY, CompressionType.LZMA2); + session_src.createTimeseries( + device2 + ".s_0", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device2 + ".s_1", TSDataType.TEXT, TSEncoding.DICTIONARY, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.FLOAT)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.TEXT)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + subs.dropTopic(topicName2); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row + 2.45f); + tablet.addValue("s_1", rowIndex, "rowIndex" + rowIndex); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1710288000000,313,'2024-03-13 08:00:00+08:00');"); // 2024-03-13 + // 08:00:00+08:00 + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1703980800000,133.45,'2023-12-31 08:00:00+08:00');"); // 2023-12-31 08:00:00+08:00 + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + AtomicInteger rowCount1 = new AtomicInteger(0); + AtomicInteger rowCount2 = new AtomicInteger(0); + // Subscribe + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("db_mix_consumer_2_topic") + .consumerGroupId("OneConsumerMultiTopicsMix") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + final short messageType = message.getMessageType(); + if (SubscriptionMessageType.isValidatedMessageType(messageType)) { + switch (SubscriptionMessageType.valueOf(messageType)) { + case SESSION_DATA_SETS_HANDLER: + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + break; + case TS_FILE_HANDLER: + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(new Path(device, "s_0", true)), + null)); + while (dataset.hasNext()) { + rowCount1.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println( + device + ":" + next.getTimestamp() + "," + next.getFields()); + } + dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(new Path(device2, "s_0", true)), + null)); + while (dataset.hasNext()) { + rowCount2.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println( + device2 + ":" + next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + break; + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + consumer.subscribe(topicName, topicName2); + + System.out.println("###### Subscription Query:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 2, "subscribe and show subscriptions"); + // Subscribe and then write data + Thread thread = + new Thread( + () -> { + try { + insert_data(System.currentTimeMillis(), device); + insert_data(System.currentTimeMillis(), device2); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + thread.start(); + thread.join(); + + AWAIT.untilAsserted( + () -> { + assertEquals(rowCount1.get(), 0, "pattern1"); + check_count(12, "select count(s_0) from " + device, "dataset pattern1"); + assertEquals(rowCount2.get(), 10, "pattern2"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/multi/IoTDBOneConsumerMultiTopicsTsfileIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/multi/IoTDBOneConsumerMultiTopicsTsfileIT.java new file mode 100644 index 000000000000..23e9484a9f4b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/multi/IoTDBOneConsumerMultiTopicsTsfileIT.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.multi; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * 1 consumer subscribes to 2 topics: historical data + * The timing of flush is very critical. If the data inside the filter and the data outside the filter are within one tsfile, they will all be extracted. + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBOneConsumerMultiTopicsTsfileIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.OneConsumerMultiTopicsTsfile"; + private static final String database2 = "root.OneConsumerMultiTopicsTsfile"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_OneConsumerMultiTopicsTsfile_1"; + private String pattern = database + ".**"; + private String pattern2 = database2 + ".**"; + private String device2 = database2 + ".d_0"; + private String topicName2 = "topic_OneConsumerMultiTopicsTsfile_2"; + private List schemaList = new ArrayList<>(); + private SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, "now", true); + createTopic_s(topicName2, pattern2, null, "now", true); + session_src.createTimeseries( + device + ".s_0", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.TEXT, TSEncoding.DICTIONARY, CompressionType.LZMA2); + session_src.createTimeseries( + device2 + ".s_0", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device2 + ".s_1", TSDataType.TEXT, TSEncoding.DICTIONARY, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.FLOAT)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.TEXT)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + subs.dropTopic(topicName2); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp, String device) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row + 2.45f); + tablet.addValue("s_1", rowIndex, "rowIndex" + rowIndex); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1710288000000,313,'2024-03-13 08:00:00+08:00');"); // 2024-03-13 + // 08:00:00+08:00 + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,s_0,s_1)values(1703980800000,133.45,'2023-12-31 08:00:00+08:00');"); // 2023-12-31 08:00:00+08:00 + insert_data(1706659200000L, device); // 2024-01-31 08:00:00+08:00 + insert_data(1706659200000L, device2); // 2024-01-31 08:00:00+08:00 + AtomicInteger rowCount1 = new AtomicInteger(0); + AtomicInteger rowCount2 = new AtomicInteger(0); + // Subscribe + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("db_tsfile_consumer_2_topic") + .consumerGroupId("OneConsumerMultiTopicsTsfile") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(new Path(device, "s_0", true)), null)); + while (dataset.hasNext()) { + rowCount1.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println( + device + ":" + next.getTimestamp() + "," + next.getFields()); + } + dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(new Path(device2, "s_0", true)), null)); + while (dataset.hasNext()) { + rowCount2.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println( + device2 + ":" + next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + consumer.subscribe(topicName, topicName2); + + System.out.println("###### Subscription query:"); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 2, "subscribe then show subscriptions"); + // Subscribe and then write data + Thread thread = + new Thread( + () -> { + try { + insert_data(System.currentTimeMillis(), device); + insert_data(System.currentTimeMillis(), device2); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + thread.start(); + thread.join(); + + AWAIT.untilAsserted( + () -> { + assertEquals(rowCount1.get(), 7, "pattern1"); + assertEquals(rowCount2.get(), 5, "pattern2"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBDBPatternDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBDBPatternDatasetPushConsumerIT.java new file mode 100644 index 000000000000..0cdf6f99fb4b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBDBPatternDatasetPushConsumerIT.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: db + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBDBPatternDatasetPushConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.DBPatternDatasetPushConsumer"; + private static final String database2 = "root.DBPatternDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_DBPatternDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static String pattern = database + ".**"; + private static SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("DB_dataset_consumer") + .consumerGroupId("push_pattern") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + + AWAIT.untilAsserted( + () -> { + check_count(8, "select count(s_0) from " + device, "Consumption Data: s_0"); + check_count(8, "select count(s_1) from " + device, "Consumption data: s_1"); + check_count(1, "select count(s_0) from " + database + ".d_1", "Consumption data:d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after canceling and re-subscribing. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count(12, "select count(s_0) from " + device, "consume data again:s_0"); + check_count(12, "select count(s_1) from " + device, "Consumption Data:s_1"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBDBPatternTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBDBPatternTsfilePushConsumerIT.java new file mode 100644 index 000000000000..4f5e5ce26634 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBDBPatternTsfilePushConsumerIT.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * TsFile + * pattern: db + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBDBPatternTsfilePushConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.DBPatternTsfilePushConsumer"; + private static final String database2 = "root.DBPatternTsfilePushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_DBPatternTsfilePushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static String pattern = database + ".**"; + private static SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + final AtomicInteger onReceiveCount = new AtomicInteger(0); + final AtomicInteger d0s0_rowCount = new AtomicInteger(0); + final AtomicInteger d0s1_rowCount = new AtomicInteger(0); + final AtomicInteger d1s0_rowCount = new AtomicInteger(0); + final AtomicInteger other_d2_rowCount = new AtomicInteger(0); + List rowCounts = new ArrayList<>(4); + rowCounts.add(d0s0_rowCount); + rowCounts.add(d0s1_rowCount); + rowCounts.add(d1s0_rowCount); + rowCounts.add(other_d2_rowCount); + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("DB_TsFile_consumer") + .consumerGroupId("push_pattern") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + onReceiveCount.incrementAndGet(); + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + Path path_d0s0 = new Path(device, "s_0", true); + Path path_d0s1 = new Path(device, "s_1", true); + Path path_d1s0 = new Path(database + ".d_1", "s_0", true); + Path path_other_d2 = new Path("" + database2 + ".d_2", "s_0", true); + List paths = new ArrayList<>(4); + paths.add(path_d0s0); + paths.add(path_d0s1); + paths.add(path_d1s0); + paths.add(path_other_d2); + for (int i = 0; i < 4; i++) { + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(paths.get(i)), null)); + while (dataset.hasNext()) { + rowCounts.get(i).addAndGet(1); + dataset.next(); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + AWAIT.untilAsserted( + () -> { + assertTrue(onReceiveCount.get() >= 2, "receive files over 2"); + assertEquals(rowCounts.get(0).get(), 8, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 8, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 1, database + ".d_1.s_0"); + assertEquals(rowCounts.get(3).get(), 0, "" + database2 + ".d_2.s_0"); + }); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + + AWAIT.untilAsserted( + () -> { + assertTrue(onReceiveCount.get() >= 5, "receive files over 2"); + assertEquals(rowCounts.get(0).get(), 20, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 20, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 2, database + ".d_1.s_0"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBDefaultPatternTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBDefaultPatternTsfilePushConsumerIT.java new file mode 100644 index 000000000000..d3353532d23d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBDefaultPatternTsfilePushConsumerIT.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * TsFile + * pattern: root.** + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBDefaultPatternTsfilePushConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.DefaultPatternTsfilePushConsumer"; + private static final String database2 = "root.DefaultPatternTsfilePushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_DefaultPatternTsfilePushConsumer"; + private static List schemaList = new ArrayList<>(); + + private SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, null, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + final AtomicInteger onReceiveCount = new AtomicInteger(0); + final AtomicInteger d0s0_rowCount = new AtomicInteger(0); + final AtomicInteger d0s1_rowCount = new AtomicInteger(0); + final AtomicInteger d1s0_rowCount = new AtomicInteger(0); + final AtomicInteger other_d2_rowCount = new AtomicInteger(0); + List rowCounts = new ArrayList<>(4); + rowCounts.add(d0s0_rowCount); + rowCounts.add(d0s1_rowCount); + rowCounts.add(d1s0_rowCount); + rowCounts.add(other_d2_rowCount); + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("default_pattern_TsFile_consumer") + .consumerGroupId("push_pattern") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + onReceiveCount.incrementAndGet(); + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + Path path_d0s0 = new Path(device, "s_0", true); + Path path_d0s1 = new Path(device, "s_1", true); + Path path_d1s0 = new Path(database + ".d_1", "s_0", true); + Path path_other_d2 = new Path("" + database2 + ".d_2", "s_0", true); + List paths = new ArrayList<>(4); + paths.add(path_d0s0); + paths.add(path_d0s1); + paths.add(path_d1s0); + paths.add(path_other_d2); + for (int i = 0; i < 4; i++) { + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(paths.get(i)), null)); + while (dataset.hasNext()) { + rowCounts.get(i).addAndGet(1); + dataset.next(); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + AWAIT.untilAsserted( + () -> { + assertEquals(rowCounts.get(0).get(), 8, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 8, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 1, database + ".d_1.s_0"); + assertEquals(rowCounts.get(3).get(), 1, "" + database2 + ".d_2.s_0"); + }); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + + // Unsubscribe, and it will consume all again. + AWAIT.untilAsserted( + () -> { + assertEquals(rowCounts.get(0).get(), 20, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 20, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 2, database + "d_1.s_0"); + assertEquals(rowCounts.get(3).get(), 2, "" + database2 + ".d_2.s_0"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBDevicePatternDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBDevicePatternDatasetPushConsumerIT.java new file mode 100644 index 000000000000..f837074051ce --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBDevicePatternDatasetPushConsumerIT.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: device + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBDevicePatternDatasetPushConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.DevicePatternDatasetPushConsumer"; + private static final String database2 = "root.DevicePatternDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_DevicePatternDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static String pattern = database + ".d_0.**"; + private static SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("device_dataset_consumer") + .consumerGroupId("push_pattern") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + + AWAIT.untilAsserted( + () -> { + check_count(8, "select count(s_0) from " + device, "Consumption data:" + pattern); + check_count(8, "select count(s_1) from " + device, "Consumption data: s_1"); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption Data: d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained when re-subscribing after cancellation. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count(12, "select count(s_0) from " + device, "Consume data again:" + pattern); + check_count(12, "select count(s_1) from " + device, "Consumption data: s_1"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBDevicePatternTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBDevicePatternTsfilePushConsumerIT.java new file mode 100644 index 000000000000..86485f20021e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBDevicePatternTsfilePushConsumerIT.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * TsFile + * pattern: device + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBDevicePatternTsfilePushConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.DevicePatternTsfilePushConsumer"; + private static final String database2 = "root.DevicePatternTsfilePushConsumer"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_DevicePatternTsfilePushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static String pattern = device + ".**"; + private SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, (1 + row) * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + String sql = "select count(s_0) from " + device; + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + final AtomicInteger onReceiveCount = new AtomicInteger(0); + final AtomicInteger d0s0_rowCount = new AtomicInteger(0); + final AtomicInteger d0s1_rowCount = new AtomicInteger(0); + final AtomicInteger d1s0_rowCount = new AtomicInteger(0); + final AtomicInteger other_d2_rowCount = new AtomicInteger(0); + List rowCounts = new ArrayList<>(4); + rowCounts.add(d0s0_rowCount); + rowCounts.add(d0s1_rowCount); + rowCounts.add(d1s0_rowCount); + rowCounts.add(other_d2_rowCount); + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("device_TsFile_consumer") + .consumerGroupId("push_pattern") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + onReceiveCount.incrementAndGet(); + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + Path path_d0s0 = new Path(device, "s_0", true); + Path path_d0s1 = new Path(device, "s_1", true); + Path path_d1s0 = new Path(database + ".d_1", "s_0", true); + Path path_other_d2 = new Path("" + database2 + ".d_2", "s_0", true); + List paths = new ArrayList<>(4); + paths.add(path_d0s0); + paths.add(path_d0s1); + paths.add(path_d1s0); + paths.add(path_other_d2); + for (int i = 0; i < 4; i++) { + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(paths.get(i)), null)); + while (dataset.hasNext()) { + rowCounts.get(i).addAndGet(1); + RowRecord next = dataset.next(); + System.out.println( + FORMAT.format(new Date()) + + ", " + + i + + ", " + + next.getTimestamp() + + "," + + next.getFields()); + } + } + System.out.println("onReceiveCount=" + onReceiveCount.get()); + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertEquals(rowCounts.get(0).get(), 10, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 10, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 0, database + ".d_1.s_0"); + assertEquals(rowCounts.get(3).get(), 0, "" + database2 + ".d_2.s_0"); + }); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertEquals(rowCounts.get(0).get(), 25, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 25, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 0, database + ".d_1.s_0"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBRootPatternDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBRootPatternDatasetPushConsumerIT.java new file mode 100644 index 000000000000..757857edab4f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBRootPatternDatasetPushConsumerIT.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: db + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBRootPatternDatasetPushConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.RootPatternDatasetPushConsumer"; + private static final String database2 = "root.RootPatternDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String device2 = database + ".d_1"; + private static final String topicName = "topic_RootPatternDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = "root.**"; + private static SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("root_dataset_consumer") + .consumerGroupId("push_pattern") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + + AWAIT.untilAsserted( + () -> { + check_count(10, "select count(s_0) from " + device, "Consumption data:" + pattern); + check_count(10, "select count(s_1) from " + device, "Consumption Data: s_1"); + check_count(1, "select count(s_0) from " + database + ".d_1", "Consumption data:d_1"); + check_count(1, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not preserved when re-subscribing after cancellation. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count(15, "select count(s_0) from " + device, "Consume data again:" + pattern); + check_count(15, "select count(s_1) from " + device, "Consumption Data: s_1"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBTSPatternDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBTSPatternDatasetPushConsumerIT.java new file mode 100644 index 000000000000..2133d40066f6 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBTSPatternDatasetPushConsumerIT.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * DataSet + * pattern: ts + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTSPatternDatasetPushConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.TSPatternDatasetPushConsumer"; + private static final String database2 = "root.TSPatternDatasetPushConsumer"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_TSPatternDatasetPushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static String pattern = database + ".d_0.s_0"; + private static SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts_dataset_consumer") + .consumerGroupId("push_pattern") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + session_dest.insertTablet(dataSet.getTablet()); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + + AWAIT.untilAsserted( + () -> { + check_count(8, "select count(s_0) from " + device, "Consumption data:" + pattern); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1"); + check_count(0, "select count(s_0) from " + database + ".d_1", "Consumption Data: d_1"); + check_count(0, "select count(s_0) from " + database2 + ".d_2", "Consumption data:d_2"); + }); + + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + AWAIT.untilAsserted( + () -> { + check_count(12, "select count(s_0) from " + device, "Consume data again:" + pattern); + check_count(0, "select count(s_1) from " + device, "Consumption Data: s_1"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBTSPatternTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBTSPatternTsfilePushConsumerIT.java new file mode 100644 index 000000000000..96e549a2334d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/pattern/IoTDBTSPatternTsfilePushConsumerIT.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.pattern; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * PushConsumer + * TsFile + * pattern: ts + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTSPatternTsfilePushConsumerIT extends AbstractSubscriptionRegressionIT { + private static final String database = "root.test.TSPatternTsfilePushConsumer"; + private static final String database2 = "root.TSPatternTsfilePushConsumer"; + private static final String device = database + ".d_0"; + private static final String topicName = "topic_TSPatternTsfilePushConsumer"; + private static List schemaList = new ArrayList<>(); + + private static final String pattern = device + ".s_0"; + private static SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createDB(database2); + createTopic_s(topicName, pattern, null, null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database + ".d_1(s_0 int64,s_1 double);"); + session_src.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_dest.executeNonQueryStatement( + "create aligned timeseries " + database2 + ".d_2(s_0 int32,s_1 float);"); + session_src.executeNonQueryStatement( + "insert into " + database2 + ".d_2(time,s_0,s_1)values(1000,132,4567.89);"); + session_src.executeNonQueryStatement( + "insert into " + database + ".d_1(time,s_0,s_1)values(2000,232,567.891);"); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + dropDB(database2); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + final AtomicInteger onReceiveCount = new AtomicInteger(0); + final AtomicInteger d0s0_rowCount = new AtomicInteger(0); + final AtomicInteger d0s1_rowCount = new AtomicInteger(0); + final AtomicInteger d1s0_rowCount = new AtomicInteger(0); + final AtomicInteger other_d2_rowCount = new AtomicInteger(0); + List rowCounts = new ArrayList<>(4); + rowCounts.add(d0s0_rowCount); + rowCounts.add(d0s1_rowCount); + rowCounts.add(d1s0_rowCount); + rowCounts.add(other_d2_rowCount); + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("ts_TsFile_consumer") + .consumerGroupId("push_pattern") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + onReceiveCount.incrementAndGet(); + System.out.println("onReceiveCount=" + onReceiveCount.get()); + try { + TsFileReader reader = message.getTsFileHandler().openReader(); + Path path_d0s0 = new Path(device, "s_0", true); + Path path_d0s1 = new Path(device, "s_1", true); + Path path_d1s0 = new Path(database + ".d_1", "s_0", true); + Path path_other_d2 = new Path("" + database2 + ".d_2", "s_0", true); + List paths = new ArrayList<>(4); + paths.add(path_d0s0); + paths.add(path_d0s1); + paths.add(path_d1s0); + paths.add(path_other_d2); + for (int i = 0; i < 4; i++) { + QueryDataSet dataset = + reader.query( + QueryExpression.create( + Collections.singletonList(paths.get(i)), null)); + while (dataset.hasNext()) { + rowCounts.get(i).addAndGet(1); + dataset.next(); + } + System.out.println("rowCounts_" + i + ":" + rowCounts.get(i).get()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + String sql = "select count(s_0) from " + device; + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceiveCount.get(), 1, "receive files 1"); + assertEquals(rowCounts.get(0).get(), 10, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 0, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 0, database + ".d_1.s_0"); + assertEquals(rowCounts.get(3).get(), 0, "" + database2 + ".d_2.s_0"); + }); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + + System.out.println(FORMAT.format(new Date()) + " src:" + getCount(session_src, sql)); + AWAIT.untilAsserted( + () -> { + assertEquals(onReceiveCount.get(), 2, "receive files over 2"); + assertEquals(rowCounts.get(0).get(), 25, device + ".s_0"); + assertEquals(rowCounts.get(1).get(), 0, device + ".s_1"); + assertEquals(rowCounts.get(2).get(), 0, database + ".d_1.s_0"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBHistoryRootDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBHistoryRootDatasetPushConsumerIT.java new file mode 100644 index 000000000000..01355f357392 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBHistoryRootDatasetPushConsumerIT.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * AFTER + * pattern: root + * dataset + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBHistoryRootDatasetPushConsumerIT extends AbstractSubscriptionRegressionIT { + private String database = "root.HistoryRootDatasetPushConsumer"; + private String device = database + ".d_0"; + private String pattern = "root.**"; + private String topicName = "topic_HistoryRootDatasetPushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, "now", false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + Thread.sleep(5000); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("root_history_dataset") + .consumerGroupId("push_time") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + Tablet tablet = dataSet.getTablet(); + session_dest.insertTablet(tablet); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + + AWAIT.untilAsserted( + () -> { + check_count(5, "select count(s_0) from " + device, "Consumption data:" + pattern); + check_count(5, "select count(s_1) from " + device, "Consumption Data: s_1"); + }); + // Unsubscribe + consumer.unsubscribe(topicName); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + + AWAIT.untilAsserted( + () -> { + // Consumption data: Progress is not retained after unsubscribing and re-subscribing. Full + // synchronization. + check_count(10, "select count(s_0) from " + device, "Consume data again:" + pattern); + check_count(10, "select count(s_1) from " + device, "Consumption data: s_1"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBHistoryRootTsFilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBHistoryRootTsFilePushConsumerIT.java new file mode 100644 index 000000000000..8319ba6c6c0c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBHistoryRootTsFilePushConsumerIT.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * Tsfile + * end_time: now + * pattern: root + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBHistoryRootTsFilePushConsumerIT extends AbstractSubscriptionRegressionIT { + private String database = "root.HistoryRootTsFilePushConsumer"; + private String device = database + ".d_0"; + private String pattern = "root.**"; + private String topicName = "topic_HistoryRootTsFilePushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, "2024-02-13T07:59:59+08:00", true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + final AtomicInteger rowCount = new AtomicInteger(0); + final AtomicInteger onReceive = new AtomicInteger(0); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerGroupId("push_time") + .consumerId("root_history_tsfile") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + onReceive.addAndGet(1); + TsFileReader reader = message.getTsFileHandler().openReader(); + List paths = new ArrayList<>(2); + for (int i = 0; i < 2; i++) { + paths.add(new Path(device, "s_" + i, true)); + } + QueryDataSet dataset = reader.query(QueryExpression.create(paths, null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + insert_data(System.currentTimeMillis()); + + AWAIT.untilAsserted( + () -> { + assertEquals(rowCount.get(), 4, "4 records"); + }); + + System.out.println("insert 2024-02-13 08:00:00+08:00"); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + AWAIT.untilAsserted( + () -> { + assertEquals(rowCount.get(), 4, "4 records"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBRealTimeDBDatasetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBRealTimeDBDatasetPushConsumerIT.java new file mode 100644 index 000000000000..1eedaa71ef2e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBRealTimeDBDatasetPushConsumerIT.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBRealTimeDBDatasetPushConsumerIT extends AbstractSubscriptionRegressionIT { + private String database = "root.RealTimeDBDatasetPushConsumer"; + private String device = database + ".d_0"; + private String pattern = database + ".**"; + private String topicName = "topic_RealTimeDBDatasetPushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, "now", null, false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("DB_realtime_dataset") + .consumerGroupId("push_time") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .fileSaveDir("target") + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + Tablet tablet = dataSet.getTablet(); + session_dest.insertTablet(tablet); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + check_count(0, "select count(s_0) from " + device, "Consumption data:" + pattern); + }); + insert_data(System.currentTimeMillis()); + AWAIT.untilAsserted( + () -> { + check_count(4, "select count(s_0) from " + device, "Consumption data:" + pattern); + }); + + // Subscribe and then write data + insert_data(System.currentTimeMillis() + 200000); // now + + AWAIT.untilAsserted( + () -> { + check_count(8, "select count(s_0) from " + device, "Consume data again:" + pattern); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBRealTimeDBTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBRealTimeDBTsfilePushConsumerIT.java new file mode 100644 index 000000000000..9e73b77d7cd1 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBRealTimeDBTsfilePushConsumerIT.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBRealTimeDBTsfilePushConsumerIT extends AbstractSubscriptionRegressionIT { + private String database = "root.RealTimeDBTsfilePushConsumer"; + private String device = database + ".d_0"; + private String pattern = database + ".**"; + private String topicName = "topic_RealTimeDBTsfilePushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, "2024-01-31T08:02:00+08:00", null, true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + final AtomicInteger rowCount = new AtomicInteger(0); + final AtomicInteger onReceive = new AtomicInteger(0); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("DB_realtime_tsfile") + .consumerGroupId("push_time") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + onReceive.addAndGet(1); + TsFileReader reader = message.getTsFileHandler().openReader(); + List paths = new ArrayList<>(2); + for (int i = 0; i < 2; i++) { + paths.add(new Path(device, "s_" + i, true)); + } + QueryDataSet dataset = reader.query(QueryExpression.create(paths, null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach((System.out::println)); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceive.get(), 0); + assertEquals(rowCount.get(), 0); + }); + insert_data(System.currentTimeMillis()); + AWAIT.untilAsserted( + () -> { + assertEquals(onReceive.get(), 1, "should process 1 file"); + assertEquals(rowCount.get(), 4, "4 records"); + }); + + // Subscribe and then write data + insert_data(System.currentTimeMillis() + 200000); // now + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceive.get(), 2, "should process 2 file"); + assertEquals(rowCount.get(), 8, "8 records"); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBTimeRangeAccurateDBDataSetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBTimeRangeAccurateDBDataSetPushConsumerIT.java new file mode 100644 index 000000000000..c431599aeced --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBTimeRangeAccurateDBDataSetPushConsumerIT.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTimeRangeAccurateDBDataSetPushConsumerIT + extends AbstractSubscriptionRegressionIT { + private String database = "root.TimeRangeAccurateDBDataSetPushConsumer"; + private String device = database + ".d_0"; + private String pattern = database + ".**"; + private String topicName = "topic_TimeRangeAccurateDBDataSetPushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, pattern, "2024-01-01T00:00:00+08:00", "2024-03-31T23:59:59+08:00", false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1704038396000L); // 2023-12-31 23:59:56+08:00 + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("DB_time_accurate_range_dataset") + .consumerGroupId("push_time") + .ackStrategy(AckStrategy.BEFORE_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + Tablet tablet = dataSet.getTablet(); + session_dest.insertTablet(tablet); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + // Before consumption subscription data + check_count(2, "select count(s_0) from " + device, "Start time boundary data:" + pattern); + }); + + insert_data(System.currentTimeMillis()); // now + AWAIT.untilAsserted( + () -> { + check_count( + 2, + "select count(s_0) from " + device, + "After writing some real-time data:" + pattern); + }); + + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + AWAIT.untilAsserted( + () -> { + check_count( + 6, "select count(s_0) from " + device, "Data within the time range:" + pattern); + }); + + insert_data(1711814398000L); // 2024-03-30 23:59:58+08:00 + AWAIT.untilAsserted( + () -> { + check_count(10, "select count(s_0) from " + device, "End time limit data:" + pattern); + }); + insert_data(1711900798000L); // 2024-03-31 23:59:58+08:00 + AWAIT.untilAsserted( + () -> { + check_count(11, "select count(s_0) from " + device, "End time limit data 2:" + pattern); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBTimeRangeDBDataSetPushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBTimeRangeDBDataSetPushConsumerIT.java new file mode 100644 index 000000000000..b9d2368188c2 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBTimeRangeDBDataSetPushConsumerIT.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * Start time, end time are both closed intervals. If not specified, the time will be 00:00:00. + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTimeRangeDBDataSetPushConsumerIT extends AbstractSubscriptionRegressionIT { + private String database = "root.TimeRangeDBDataSetPushConsumer"; + private String device = database + ".d_0"; + private String pattern = database + ".**"; + private String topicName = "topic_TimeRangeDBDataSetPushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, pattern, "2024-01-01T00:00:00+08:00", "2024-03-31T00:00:00+08:00", false); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1704038396000L); // 2023-12-31 23:59:56+08:00 + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("DB_time_range_dataset") + .consumerGroupId("push_time") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .consumeListener( + message -> { + for (final SubscriptionSessionDataSet dataSet : + message.getSessionDataSetsHandler()) { + try { + Tablet tablet = dataSet.getTablet(); + session_dest.insertTablet(tablet); + } catch (StatementExecutionException e) { + throw new RuntimeException(e); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + check_count(2, "select count(s_0) from " + device, "Start time boundary data:" + pattern); + }); + + insert_data(System.currentTimeMillis()); // now + AWAIT.untilAsserted( + () -> { + check_count( + 2, "select count(s_0) from " + device, "Write some real-time data after:" + pattern); + }); + + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + AWAIT.untilAsserted( + () -> { + check_count( + 6, "select count(s_0) from " + device, "Data within the time range:" + pattern); + }); + + insert_data(1711814398000L); // 2024-03-30 23:59:58+08:00 + AWAIT.untilAsserted( + () -> { + // Because the end time is 2024-03-31 00:00:00, closed interval + check_count(8, "select count(s_0) from " + device, "End time limit data:" + pattern); + }); + + insert_data(1711900798000L); // 2024-03-31 23:59:58+08:00 + AWAIT.untilAsserted( + () -> { + check_count(8, "select count(s_0) from " + device, "End time limit data:" + pattern); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBTimeRangeDBTsfilePushConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBTimeRangeDBTsfilePushConsumerIT.java new file mode 100644 index 000000000000..cccad773623e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/pushconsumer/time/IoTDBTimeRangeDBTsfilePushConsumerIT.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.pushconsumer.time; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.AckStrategy; +import org.apache.iotdb.session.subscription.consumer.ConsumeResult; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPushConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; + +/*** + * Start time, end time are both closed intervals. If not specified, the time will be 00:00:00. + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTimeRangeDBTsfilePushConsumerIT extends AbstractSubscriptionRegressionIT { + private String database = "root.TimeRangeDBTsfilePushConsumer"; + private String device = database + ".d_0"; + private String pattern = database + ".**"; + private String topicName = "topic_TimeRangeDBTsfilePushConsumer"; + private List schemaList = new ArrayList<>(); + private SubscriptionPushConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, pattern, "2024-01-01T00:00:00+08:00", "2024-03-31T00:00:00+08:00", true); + session_src.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + device + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + session_src.executeNonQueryStatement("flush;"); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + final AtomicInteger rowCount = new AtomicInteger(0); + final AtomicInteger onReceive = new AtomicInteger(0); + // Write data before subscribing + insert_data(1704038396000L); // 2023-12-31 23:59:56+08:00 + consumer = + new SubscriptionPushConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .consumerId("db_time_range_accurate_tsfile") + .consumerGroupId("push_time") + .ackStrategy(AckStrategy.AFTER_CONSUME) + .fileSaveDir("target/push-subscription") + .consumeListener( + message -> { + try { + onReceive.addAndGet(1); + TsFileReader reader = message.getTsFileHandler().openReader(); + List paths = new ArrayList<>(2); + for (int i = 0; i < 2; i++) { + paths.add(new Path(device, "s_" + i, true)); + } + QueryDataSet dataset = reader.query(QueryExpression.create(paths, null)); + while (dataset.hasNext()) { + rowCount.addAndGet(1); + RowRecord next = dataset.next(); + System.out.println(next.getTimestamp() + "," + next.getFields()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return ConsumeResult.SUCCESS; + }) + .buildPushConsumer(); + consumer.open(); + // Subscribe + consumer.subscribe(topicName); + subs.getSubscriptions().forEach(System.out::println); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + + AWAIT.untilAsserted( + () -> { + assertEquals(onReceive.get(), 1); + // loose-time should 2 records,get 4 records + assertTrue(rowCount.get() >= 2); + }); + + insert_data(System.currentTimeMillis()); // now, not in range + AWAIT.untilAsserted( + () -> { + assertEquals(onReceive.get(), 1); + assertTrue(rowCount.get() >= 2); + }); + + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + AWAIT.untilAsserted( + () -> { + assertEquals(onReceive.get(), 2); + assertTrue(rowCount.get() >= 6); + }); + + insert_data(1711814398000L); // 2024-03-30 23:59:58+08:00 + AWAIT.untilAsserted( + () -> { + // Because the end time is 2024-03-31 00:00:00, closed interval + assertEquals(onReceive.get(), 3); + assertTrue(rowCount.get() >= 8); + }); + + insert_data(1711900798000L); // 2024-03-31 23:59:58+08:00 + AWAIT.untilAsserted( + () -> { + assertEquals(onReceive.get(), 3); + assertTrue(rowCount.get() >= 8); + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/topic/IoTDBDataSet1TopicConsumerSpecialIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/topic/IoTDBDataSet1TopicConsumerSpecialIT.java new file mode 100644 index 000000000000..19ee191d1c7a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/topic/IoTDBDataSet1TopicConsumerSpecialIT.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.topic; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * Sequence-level topic, with start, end, tsfile + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBDataSet1TopicConsumerSpecialIT extends AbstractSubscriptionRegressionIT { + private String database = "root.test.ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz1"; + private String device = database + ".`#01`"; + private String pattern = device + ".`ABH#01`"; + private String topicName = "topic_DataSet1TopicConsumerSpecial"; + private List schemaList = new ArrayList<>(); + private SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s( + topicName, pattern, "2024-01-01T00:00:00+08:00", "2024-03-01T00:00:00+08:00", false); + session_src.createTimeseries( + pattern, TSDataType.INT32, TSEncoding.GORILLA, CompressionType.SNAPPY); + session_src.createTimeseries( + device + ".`BJ-ABH#01`", TSDataType.BOOLEAN, TSEncoding.RLE, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("`ABH#01`", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("`BJ-ABH#01`", TSDataType.BOOLEAN)); + System.out.println("topics:" + subs.getTopics()); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + Tablet tablet = new Tablet(device, schemaList, 10); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("`ABH#01`", rowIndex, row * 20 + row); + tablet.addValue("`BJ-ABH#01`", rowIndex, true); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,`ABH#01`,`BJ-ABH#01`)values(1710288000000,313,false);"); // 2024-03-13 + // 08:00:00+08:00 + // Subscribe + consumer = create_pull_consumer("`Group-ABH#01`", "`ABH#01`", false, 0L); + assertEquals(subs.getSubscriptions().size(), 0, "Before subscribing, show subscriptions"); + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + // Consumption data + consume_data(consumer); + check_count(4, "select count(`ABH#01`) from " + device, "Consumption data:" + pattern); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "After cancellation, show subscriptions"); + // Subscribe and then write data + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + session_src.executeNonQueryStatement( + "insert into " + + device + + "(time,`ABH#01`,`BJ-ABH#01`)values(1703980800000,1231,false);"); // 2023-12-31 + // 08:00:00+08:00 + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + // Consumption data + consume_data(consumer); + check_count(8, "select count(`ABH#01`) from " + device, "consume data again:" + pattern); + // Unsubscribe + consumer.unsubscribe(topicName); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/topic/IoTDBTestTopicNameIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/topic/IoTDBTestTopicNameIT.java new file mode 100644 index 000000000000..f80a819bf5c5 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/topic/IoTDBTestTopicNameIT.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.topic; + +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegression; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/*** + * Special topic name + */ +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegression.class}) +public class IoTDBTestTopicNameIT extends AbstractSubscriptionRegressionIT { + private String database = "root.TestTopicName"; + private String device = database + ".d_0"; + private String pattern = device + ".s_0"; + private String topicName = "`1-group.1-consumer.ts.topic`"; + private List schemaList = new ArrayList<>(); + private SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(database); + createTopic_s(topicName, pattern, null, null, false); + session_src.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + session_dest.createTimeseries( + pattern, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + device + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZMA2); + schemaList.add(new MeasurementSchema("s_0", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + subs.getTopics().forEach((System.out::println)); + assertTrue(subs.getTopic(topicName).isPresent(), "Create show topics"); + } + + @Override + @After + public void tearDown() throws Exception { + try { + consumer.close(); + } catch (Exception e) { + } + subs.dropTopic(topicName); + dropDB(database); + super.tearDown(); + } + + private void insert_data(long timestamp) + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + Tablet tablet = new Tablet(device, schemaList, 5); + int rowIndex = 0; + for (int row = 0; row < 5; row++) { + rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += 2000; + } + session_src.insertTablet(tablet); + Thread.sleep(1000); + } + + @Test + public void do_test() + throws InterruptedException, + TException, + IoTDBConnectionException, + IOException, + StatementExecutionException { + consumer = create_pull_consumer("g1", "c1", false, null); + // Write data before subscribing + insert_data(1706659200000L); // 2024-01-31 08:00:00+08:00 + // Subscribe + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after subscription"); + Thread.sleep(1000); + insert_data(System.currentTimeMillis()); + // Consumption data + consume_data(consumer, session_dest); + check_count(10, "select count(s_0) from " + device, "Consumption data:" + pattern); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1"); + // Unsubscribe + consumer.unsubscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 0, "Show subscriptions after unsubscribe"); + // Subscribe and then write data + consumer.subscribe(topicName); + assertEquals(subs.getSubscriptions().size(), 1, "show subscriptions after re-subscribing"); + Thread.sleep(1000); + insert_data(1707782400000L); // 2024-02-13 08:00:00+08:00 + // Consumption data: Progress is not preserved when re-subscribing after cancellation. Full + // synchronization. + consume_data(consumer, session_dest); + check_count(15, "select count(s_0) from " + device, "Consume data again:" + pattern); + check_count(0, "select count(s_1) from " + device, "Consumption data: s_1"); + } +}