From 920131021092825289582055287d41069bd8630f Mon Sep 17 00:00:00 2001 From: smallhibiscus <844981280> Date: Wed, 20 Dec 2023 16:55:30 +0800 Subject: [PATCH 1/5] [test](e2ecase) Optimize doris container log print --- .../java/org/apache/doris/flink/DorisTestBase.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java index 520d3d2ae..79173bf06 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java +++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java @@ -42,11 +42,11 @@ import java.util.stream.Stream; import static org.awaitility.Awaitility.given; +import static org.awaitility.Durations.ONE_SECOND; public abstract class DorisTestBase { protected static final Logger LOG = LoggerFactory.getLogger(DorisTestBase.class); - // protected static final String DORIS_12_DOCKER_IMAGE = "adamlee489/doris:1.2.7.1_arm"; - protected static final String DORIS_12_DOCKER_IMAGE = "adamlee489/doris:1.2.7.1_x86"; + protected static final String DORIS_12_DOCKER_IMAGE = "adamlee489/doris:2.0.3"; private static final String DRIVER_JAR = "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"; protected static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; @@ -68,6 +68,7 @@ public static void startContainers() { given().ignoreExceptions() .await() .atMost(120, TimeUnit.SECONDS) + .pollInterval(ONE_SECOND) .untilAsserted(DorisTestBase::initializeJdbcConnection); LOG.info("Containers are started."); } @@ -126,10 +127,10 @@ protected static void initializeJdbcConnection() throws SQLException, MalformedU } private static boolean isBeReady(ResultSet rs, Duration duration) throws SQLException { + LockSupport.parkNanos(duration.toNanos()); if (rs.next()) { - String isAlive = rs.getString(10).trim(); - String totalCap = rs.getString(16).trim(); - LockSupport.parkNanos(duration.toNanos()); + String isAlive = rs.getString("Alive").trim(); + String totalCap = rs.getString("TotalCapacity").trim(); return "true".equalsIgnoreCase(isAlive) && !"0.000".equalsIgnoreCase(totalCap); } return false; From 66e19cb87d766dfabe35e083efd36cff84ed9c2e Mon Sep 17 00:00:00 2001 From: smallhibiscus <844981280> Date: Thu, 21 Dec 2023 11:52:18 +0800 Subject: [PATCH 2/5] fix order of select result --- .../org/apache/doris/flink/tools/cdc/MySQLDorisE2ECase.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MySQLDorisE2ECase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MySQLDorisE2ECase.java index 0246e3930..053aef404 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MySQLDorisE2ECase.java +++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MySQLDorisE2ECase.java @@ -103,7 +103,7 @@ public void testMySQL2Doris() throws Exception { Arrays.asList("doris_3", 3)) .collect(Collectors.toSet()); String sql = - "select * from %s.%s union all select * from %s.%s union all select * from %s.%s order by 1"; + "select * from ( select * from %s.%s union all select * from %s.%s union all select * from %s.%s ) res order by 1,2"; checkResult(expected, sql, 2); // add incremental data @@ -135,7 +135,7 @@ public void testMySQL2Doris() throws Exception { Arrays.asList("doris_3_1", 12)) .collect(Collectors.toSet()); sql = - "select * from %s.%s union all select * from %s.%s union all select * from %s.%s order by 1"; + "select * from ( select * from %s.%s union all select * from %s.%s union all select * from %s.%s ) res order by 1"; checkResult(expected2, sql, 2); // mock schema change From 2cf129c75820591d741225226bdb1fb9e9c6c10d Mon Sep 17 00:00:00 2001 From: smallhibiscus <844981280> Date: Thu, 21 Dec 2023 11:56:14 +0800 Subject: [PATCH 3/5] fix --- .../org/apache/doris/flink/tools/cdc/MySQLDorisE2ECase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MySQLDorisE2ECase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MySQLDorisE2ECase.java index 053aef404..89870305d 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MySQLDorisE2ECase.java +++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MySQLDorisE2ECase.java @@ -103,7 +103,7 @@ public void testMySQL2Doris() throws Exception { Arrays.asList("doris_3", 3)) .collect(Collectors.toSet()); String sql = - "select * from ( select * from %s.%s union all select * from %s.%s union all select * from %s.%s ) res order by 1,2"; + "select * from ( select * from %s.%s union all select * from %s.%s union all select * from %s.%s ) res order by 1"; checkResult(expected, sql, 2); // add incremental data From 6368b3c5fd414f8ac99c0774b747532ceffbb639 Mon Sep 17 00:00:00 2001 From: smallhibiscus <844981280> Date: Tue, 26 Dec 2023 11:26:37 +0800 Subject: [PATCH 4/5] [test](e2ecase) add doris docker image 2.0 and 1.2.7 --- .github/workflows/run-e2ecase-12.yml | 44 +++++++++++++++++++ .../{run-e2ecase.yml => run-e2ecase-20.yml} | 2 +- .github/workflows/run-itcase-12.yml | 44 +++++++++++++++++++ .../{run-itcase.yml => run-itcase-20.yml} | 2 +- .../org/apache/doris/flink/DorisTestBase.java | 8 ++-- 5 files changed, 94 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/run-e2ecase-12.yml rename .github/workflows/{run-e2ecase.yml => run-e2ecase-20.yml} (92%) create mode 100644 .github/workflows/run-itcase-12.yml rename .github/workflows/{run-itcase.yml => run-itcase-20.yml} (92%) diff --git a/.github/workflows/run-e2ecase-12.yml b/.github/workflows/run-e2ecase-12.yml new file mode 100644 index 000000000..03d46a189 --- /dev/null +++ b/.github/workflows/run-e2ecase-12.yml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Run E2ECases +on: + pull_request: + push: + +jobs: + build-extension: + name: "Run E2ECases" + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@master + + - name: Setup java + uses: actions/setup-java@v2 + with: + distribution: adopt + java-version: '8' + + - name: Run E2ECases + run: | + cd flink-doris-connector && mvn test -Dtest="*E2ECase" -Dimage="adamlee489/doris:1.2.7.1" + diff --git a/.github/workflows/run-e2ecase.yml b/.github/workflows/run-e2ecase-20.yml similarity index 92% rename from .github/workflows/run-e2ecase.yml rename to .github/workflows/run-e2ecase-20.yml index ad76a3fa7..77492ae07 100644 --- a/.github/workflows/run-e2ecase.yml +++ b/.github/workflows/run-e2ecase-20.yml @@ -40,5 +40,5 @@ jobs: - name: Run E2ECases run: | - cd flink-doris-connector && mvn test -Dtest="*E2ECase" + cd flink-doris-connector && mvn test -Dtest="*E2ECase" -Dimage="adamlee489/doris:2.0.3" diff --git a/.github/workflows/run-itcase-12.yml b/.github/workflows/run-itcase-12.yml new file mode 100644 index 000000000..eb202b7d0 --- /dev/null +++ b/.github/workflows/run-itcase-12.yml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Run ITCases +on: + pull_request: + push: + +jobs: + build-extension: + name: "Run ITCases" + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@master + + - name: Setup java + uses: actions/setup-java@v2 + with: + distribution: adopt + java-version: '8' + + - name: Run ITCases + run: | + cd flink-doris-connector && mvn test -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1" + diff --git a/.github/workflows/run-itcase.yml b/.github/workflows/run-itcase-20.yml similarity index 92% rename from .github/workflows/run-itcase.yml rename to .github/workflows/run-itcase-20.yml index 624ccaaa9..d03ee646f 100644 --- a/.github/workflows/run-itcase.yml +++ b/.github/workflows/run-itcase-20.yml @@ -40,5 +40,5 @@ jobs: - name: Run ITCases run: | - cd flink-doris-connector && mvn test -Dtest="*ITCase" + cd flink-doris-connector && mvn test -Dtest="*ITCase" -Dimage="adamlee489/doris:2.0.3" diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java index 79173bf06..e3478da16 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java +++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java @@ -46,7 +46,7 @@ public abstract class DorisTestBase { protected static final Logger LOG = LoggerFactory.getLogger(DorisTestBase.class); - protected static final String DORIS_12_DOCKER_IMAGE = "adamlee489/doris:2.0.3"; + protected static final String DORIS_DOCKER_IMAGE = System.getProperty("image"); private static final String DRIVER_JAR = "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"; protected static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; @@ -67,7 +67,7 @@ public static void startContainers() { Startables.deepStart(Stream.of(DORIS_CONTAINER)).join(); given().ignoreExceptions() .await() - .atMost(120, TimeUnit.SECONDS) + .atMost(300, TimeUnit.SECONDS) .pollInterval(ONE_SECOND) .untilAsserted(DorisTestBase::initializeJdbcConnection); LOG.info("Containers are started."); @@ -82,7 +82,7 @@ public static void stopContainers() { public static GenericContainer createDorisContainer() { GenericContainer container = - new GenericContainer<>(DORIS_12_DOCKER_IMAGE) + new GenericContainer<>(DORIS_DOCKER_IMAGE) .withNetwork(Network.newNetwork()) .withNetworkAliases("DorisContainer") .withEnv("FE_SERVERS", "fe1:127.0.0.1:9010") @@ -95,7 +95,7 @@ public static GenericContainer createDorisContainer() { .withPrivilegedMode(true) .withLogConsumer( new Slf4jLogConsumer( - DockerLoggerFactory.getLogger(DORIS_12_DOCKER_IMAGE))); + DockerLoggerFactory.getLogger(DORIS_DOCKER_IMAGE))); container.setPortBindings( Lists.newArrayList( From 3d95520a168bc109c8d9fc3d97652a00e405d481 Mon Sep 17 00:00:00 2001 From: smallhibiscus <844981280> Date: Tue, 26 Dec 2023 11:37:12 +0800 Subject: [PATCH 5/5] [test](e2ecase) rename workflow --- .github/workflows/run-e2ecase-12.yml | 6 +++--- .github/workflows/run-e2ecase-20.yml | 4 ++-- .github/workflows/run-itcase-12.yml | 6 +++--- .github/workflows/run-itcase-20.yml | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/run-e2ecase-12.yml b/.github/workflows/run-e2ecase-12.yml index 03d46a189..fd89d20d8 100644 --- a/.github/workflows/run-e2ecase-12.yml +++ b/.github/workflows/run-e2ecase-12.yml @@ -16,14 +16,14 @@ # under the License. # --- -name: Run E2ECases +name: Run E2ECases 1.2 on: pull_request: push: jobs: build-extension: - name: "Run E2ECases" + name: "Run E2ECases 1.2" runs-on: ubuntu-latest defaults: run: @@ -40,5 +40,5 @@ jobs: - name: Run E2ECases run: | - cd flink-doris-connector && mvn test -Dtest="*E2ECase" -Dimage="adamlee489/doris:1.2.7.1" + cd flink-doris-connector && mvn test -Dtest="*E2ECase" -Dimage="adamlee489/doris:1.2.7.1_x86" diff --git a/.github/workflows/run-e2ecase-20.yml b/.github/workflows/run-e2ecase-20.yml index 77492ae07..ebf7ae62d 100644 --- a/.github/workflows/run-e2ecase-20.yml +++ b/.github/workflows/run-e2ecase-20.yml @@ -16,14 +16,14 @@ # under the License. # --- -name: Run E2ECases +name: Run E2ECases 2.0 on: pull_request: push: jobs: build-extension: - name: "Run E2ECases" + name: "Run E2ECases 2.0" runs-on: ubuntu-latest defaults: run: diff --git a/.github/workflows/run-itcase-12.yml b/.github/workflows/run-itcase-12.yml index eb202b7d0..cd31c3ad5 100644 --- a/.github/workflows/run-itcase-12.yml +++ b/.github/workflows/run-itcase-12.yml @@ -16,14 +16,14 @@ # under the License. # --- -name: Run ITCases +name: Run ITCases 1.2 on: pull_request: push: jobs: build-extension: - name: "Run ITCases" + name: "Run ITCases 1.2" runs-on: ubuntu-latest defaults: run: @@ -40,5 +40,5 @@ jobs: - name: Run ITCases run: | - cd flink-doris-connector && mvn test -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1" + cd flink-doris-connector && mvn test -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1_x86" diff --git a/.github/workflows/run-itcase-20.yml b/.github/workflows/run-itcase-20.yml index d03ee646f..ad9ef5a46 100644 --- a/.github/workflows/run-itcase-20.yml +++ b/.github/workflows/run-itcase-20.yml @@ -16,14 +16,14 @@ # under the License. # --- -name: Run ITCases +name: Run ITCases 2.0 on: pull_request: push: jobs: build-extension: - name: "Run ITCases" + name: "Run ITCases 2.0" runs-on: ubuntu-latest defaults: run: