diff --git a/.github/workflows/run-e2ecase-12.yml b/.github/workflows/run-e2ecase-12.yml new file mode 100644 index 000000000..fd89d20d8 --- /dev/null +++ b/.github/workflows/run-e2ecase-12.yml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Run E2ECases 1.2 +on: + pull_request: + push: + +jobs: + build-extension: + name: "Run E2ECases 1.2" + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@master + + - name: Setup java + uses: actions/setup-java@v2 + with: + distribution: adopt + java-version: '8' + + - name: Run E2ECases + run: | + cd flink-doris-connector && mvn test -Dtest="*E2ECase" -Dimage="adamlee489/doris:1.2.7.1_x86" + diff --git a/.github/workflows/run-e2ecase.yml b/.github/workflows/run-e2ecase-20.yml similarity index 88% rename from .github/workflows/run-e2ecase.yml rename to .github/workflows/run-e2ecase-20.yml index ad76a3fa7..ebf7ae62d 100644 --- a/.github/workflows/run-e2ecase.yml +++ b/.github/workflows/run-e2ecase-20.yml @@ -16,14 +16,14 @@ # under the License. # --- -name: Run E2ECases +name: Run E2ECases 2.0 on: pull_request: push: jobs: build-extension: - name: "Run E2ECases" + name: "Run E2ECases 2.0" runs-on: ubuntu-latest defaults: run: @@ -40,5 +40,5 @@ jobs: - name: Run E2ECases run: | - cd flink-doris-connector && mvn test -Dtest="*E2ECase" + cd flink-doris-connector && mvn test -Dtest="*E2ECase" -Dimage="adamlee489/doris:2.0.3" diff --git a/.github/workflows/run-itcase-12.yml b/.github/workflows/run-itcase-12.yml new file mode 100644 index 000000000..cd31c3ad5 --- /dev/null +++ b/.github/workflows/run-itcase-12.yml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Run ITCases 1.2 +on: + pull_request: + push: + +jobs: + build-extension: + name: "Run ITCases 1.2" + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@master + + - name: Setup java + uses: actions/setup-java@v2 + with: + distribution: adopt + java-version: '8' + + - name: Run ITCases + run: | + cd flink-doris-connector && mvn test -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1_x86" + diff --git a/.github/workflows/run-itcase.yml b/.github/workflows/run-itcase-20.yml similarity index 88% rename from .github/workflows/run-itcase.yml rename to .github/workflows/run-itcase-20.yml index 624ccaaa9..ad9ef5a46 100644 --- a/.github/workflows/run-itcase.yml +++ b/.github/workflows/run-itcase-20.yml @@ -16,14 +16,14 @@ # under the License. # --- -name: Run ITCases +name: Run ITCases 2.0 on: pull_request: push: jobs: build-extension: - name: "Run ITCases" + name: "Run ITCases 2.0" runs-on: ubuntu-latest defaults: run: @@ -40,5 +40,5 @@ jobs: - name: Run ITCases run: | - cd flink-doris-connector && mvn test -Dtest="*ITCase" + cd flink-doris-connector && mvn test -Dtest="*ITCase" -Dimage="adamlee489/doris:2.0.3" diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java index 520d3d2ae..e3478da16 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java +++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java @@ -42,11 +42,11 @@ import java.util.stream.Stream; import static org.awaitility.Awaitility.given; +import static org.awaitility.Durations.ONE_SECOND; public abstract class DorisTestBase { protected static final Logger LOG = LoggerFactory.getLogger(DorisTestBase.class); - // protected static final String DORIS_12_DOCKER_IMAGE = "adamlee489/doris:1.2.7.1_arm"; - protected static final String DORIS_12_DOCKER_IMAGE = "adamlee489/doris:1.2.7.1_x86"; + protected static final String DORIS_DOCKER_IMAGE = System.getProperty("image"); private static final String DRIVER_JAR = "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"; protected static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; @@ -67,7 +67,8 @@ public static void startContainers() { Startables.deepStart(Stream.of(DORIS_CONTAINER)).join(); given().ignoreExceptions() .await() - .atMost(120, TimeUnit.SECONDS) + .atMost(300, TimeUnit.SECONDS) + .pollInterval(ONE_SECOND) .untilAsserted(DorisTestBase::initializeJdbcConnection); LOG.info("Containers are started."); } @@ -81,7 +82,7 @@ public static void stopContainers() { public static GenericContainer createDorisContainer() { GenericContainer container = - new GenericContainer<>(DORIS_12_DOCKER_IMAGE) + new GenericContainer<>(DORIS_DOCKER_IMAGE) .withNetwork(Network.newNetwork()) .withNetworkAliases("DorisContainer") .withEnv("FE_SERVERS", "fe1:127.0.0.1:9010") @@ -94,7 +95,7 @@ public static GenericContainer createDorisContainer() { .withPrivilegedMode(true) .withLogConsumer( new Slf4jLogConsumer( - DockerLoggerFactory.getLogger(DORIS_12_DOCKER_IMAGE))); + DockerLoggerFactory.getLogger(DORIS_DOCKER_IMAGE))); container.setPortBindings( Lists.newArrayList( @@ -126,10 +127,10 @@ protected static void initializeJdbcConnection() throws SQLException, MalformedU } private static boolean isBeReady(ResultSet rs, Duration duration) throws SQLException { + LockSupport.parkNanos(duration.toNanos()); if (rs.next()) { - String isAlive = rs.getString(10).trim(); - String totalCap = rs.getString(16).trim(); - LockSupport.parkNanos(duration.toNanos()); + String isAlive = rs.getString("Alive").trim(); + String totalCap = rs.getString("TotalCapacity").trim(); return "true".equalsIgnoreCase(isAlive) && !"0.000".equalsIgnoreCase(totalCap); } return false; diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MySQLDorisE2ECase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MySQLDorisE2ECase.java index 0246e3930..89870305d 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MySQLDorisE2ECase.java +++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MySQLDorisE2ECase.java @@ -103,7 +103,7 @@ public void testMySQL2Doris() throws Exception { Arrays.asList("doris_3", 3)) .collect(Collectors.toSet()); String sql = - "select * from %s.%s union all select * from %s.%s union all select * from %s.%s order by 1"; + "select * from ( select * from %s.%s union all select * from %s.%s union all select * from %s.%s ) res order by 1"; checkResult(expected, sql, 2); // add incremental data @@ -135,7 +135,7 @@ public void testMySQL2Doris() throws Exception { Arrays.asList("doris_3_1", 12)) .collect(Collectors.toSet()); sql = - "select * from %s.%s union all select * from %s.%s union all select * from %s.%s order by 1"; + "select * from ( select * from %s.%s union all select * from %s.%s union all select * from %s.%s ) res order by 1"; checkResult(expected2, sql, 2); // mock schema change