add regression test

gnehil · Jul 29, 2024 · a0ce117 · a0ce117
1 parent be72dcd
commit a0ce117
Show file tree

Hide file tree

Showing 3 changed files with 177 additions and 0 deletions.
diff --git a/regression-test/data/load_p0/ingestion_load/data.parquet b/regression-test/data/load_p0/ingestion_load/data.parquet
diff --git a/regression-test/data/load_p0/ingestion_load/test_ingestion_load.out b/regression-test/data/load_p0/ingestion_load/test_ingestion_load.out
@@ -0,0 +1,13 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+-2067215761	41V4fgCKI3Hkioe	DZGaq24Yqh7SwmbPT6IX23jfC5NKqG7gE9JT4GLwiaQtoO8l6EjhGWQP9X7NHmjdqMbIN5kNeDkffOrlS6roIwj2wXpJ	true	123	-4524	1014223769452772206	-6589390612399245616	29103.387	-4.09385322835896E8	886.394	-699.960	2024-02-22	2024-03-04	2024-01-01T22:13:54	2023-10-02T19:35:10
+-1982022740	fcZ1o6ZXG8UOFh5	iw4Ziys42GRRTFNkVPeQEA9I5EQtBD04xfefDsPCWN0vr1	true	-3	-25143	7122614823242399351	7391807634038366248	23160.604	-9.20283206353984E8	829.880	-387.403	2023-10-16	2024-05-13	2024-07-16T18:27:45	2023-11-03T05:30:21
+-1228926576	1TISaGtB01BiqVt	kq4u	false	-123	15962	3590007830423934951	-1478759439092857810	-7813.757	-6.98785793100899E8	930.743	402.350	2024-07-23	2023-07-30	2023-11-27T17:48:50	2024-03-11T21:09:58
+-575060895	rRfatUJiOO5dq9Y	ETjqrUNUnI5kSmkafjWfRTG8HIp98pLGXagNpXZHqOIZZDRkoGeahOwk9	false	16	-767	6623730208927396375	-3055706367894284822	12540.839	-1.047911096098831E9	-752.454	-241.620	2024-04-10	2024-05-16	2023-12-07T23:38:05	2023-12-11T05:48:36
+-76042627	PcVaKC43qmIzuxY	U3aGxaZumFpqcUsLI	true	44	31151	9085406701767055602	-5846138572199996843	-16845.29	2.44522690225531E8	-784.720	-467.133	2023-10-31	2023-08-29	2023-09-12T10:12:46	2023-10-19T17:02:51
+121048200	KPLWjhhbGXqflJi	rzqOYQH9ySHPwCm5K4GdeuI28G8LLmnpqLmsLMLfyRIvcfrlubQI47wUa8QILhuS38MBkjL	true	42	13182	-6601530758880565531	5619594098883737912	-2782.1506	3.86698722676211E8	478.420	-330.289	2024-06-17	2023-12-26	2024-04-28T03:29:04	2023-08-18T21:05:32
+262860291	m3XgmlbIHYNH1qS	BTJRzVrpM78zJAsHMEGhkF5BiDoc3yJuoV0s209sFcqElZsheBgolBGlFl9X4EfauD64FcFF2Mi4V0dKZfpDgaLLRPfG1SALV7	false	-42	5990	-7504815416577235660	1535659792778122944	1171.9619	1.28834143701229E8	626.721	682.828	2023-11-24	2023-11-18	2024-03-21T11:50:17	2024-03-31T12:59:27
+579428879	KsOC6WGrieGlo7B	SzeA6tRbsiGWJTBDvBQdBjCqjSE6Y	false	-111	32758	4029182463831656671	-3546198025044093789	20338.55	-2.015222388533773E9	61.981	720.310	2023-11-13	2024-07-04	2024-07-19T12:42:28	2024-01-04T10:32:53
+1145092610	xWJUDWAV8Nllo0F	dnZ9RMVdoqxh4kGBvy55zQdChNTVYdlvRZP4aWIkXyErUbM1XmFGQ9vuCD113JKKCyx4crDoY	false	115	-22832	-7242855305248390982	-4240353246453053617	-9074.909	-2.51212400295869E8	-502.410	618.820	2024-06-12	2024-04-18	2023-11-04T09:55:17	2023-11-13T16:30:23
+1736373707	UU14wnLhPkBid41	pmuNqYfOc3JCscf9meT5dYB2i28Pt9iaeXK4QqjVZJdoKFOeZI5bG9RKm1zInTdDMW1N0PKI5Y	true	-105	-20276	360048259532857165	-4602633478165721463	-13230.296	-1.708246954394742E9	757.147	-533.800	2024-01-05	2023-09-08	2023-11-27T05:21:33	2024-02-11T21:35:03
+
diff --git a/regression-test/suites/load_p0/ingestion_load/test_ingestion_load.groovy b/regression-test/suites/load_p0/ingestion_load/test_ingestion_load.groovy
@@ -0,0 +1,164 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import java.nio.file.Files
+import java.nio.file.Paths
+import java.nio.file.StandardCopyOption
+
+suite('test_ingestion_load', 'p0') {
+
+    def testIngestLoadJob = { testTable, loadLabel, String dataFile ->
+
+        sql "TRUNCATE TABLE ${testTable}"
+
+        sql "CLEAN LABEL FROM ${context.dbName}"
+
+        Integer loadId = -1
+        Integer tableId = -1
+        Integer partitionId = -1
+        Integer indexId = -1
+        Integer bucketId = 0
+        Integer schemaHash = -1
+
+        String reqBody =
+                """{
+                    "label": "${loadLabel}",
+                    "tableToPartition": {
+                        "${testTable}": []
+                    },
+                    "properties": {}
+                }"""
+
+        httpTest {
+            endpoint context.config.feHttpAddress
+            uri "/api/ingestion_load/${context.dbName}/_create"
+            op "post"
+            user context.config.feHttpUser
+            password context.config.feHttpPassword
+            body reqBody
+            check { code, resBody ->
+                assert code == 200
+                def resBodyJson = parseJson(resBody)
+                assert resBodyJson instanceof Map
+                assert resBodyJson.code == 0
+                def data = resBodyJson.data
+                loadId = data.loadId
+                def tableMeta = data.tableMeta
+                tableId = tableMeta["${testTable}"].id
+                def index = tableMeta["${testTable}"].indexes[0]
+                indexId = index.indexId
+                schemaHash = index.schemaHash
+                partitionId = tableMeta["${testTable}"].partitionInfo.partitions[0].partitionId
+            }
+        }
+
+        String resultFileName = "V1.${loadLabel}.${tableId}.${partitionId}.${indexId}.${bucketId}.${schemaHash}.parquet"
+        logger.info("resultFileName: " + resultFileName)
+
+        Files.copy(Paths.get(dataFile),
+                Paths.get(context.config.dataPath + "/load_p0/ingestion_load/${resultFileName}"), StandardCopyOption.REPLACE_EXISTING)
+
+        String etlResultFilePath = uploadToHdfs "/load_p0/ingestion_load/${resultFileName}"
+
+        String dppResult = '{\\"isSuccess\\":true,\\"failedReason\\":\\"\\",\\"scannedRows\\":10,\\"fileNumber\\":1,' +
+                '\\"fileSize\\":2441,\\"normalRows\\":10,\\"abnormalRows\\":0,\\"unselectRows\\":0,' +
+                '\\"partialAbnormalRows\\":\\"[]\\",\\"scannedBytes\\":0}'
+
+        String updateStatusReqBody =
+                """{
+                "loadId": ${loadId},
+                "statusInfo": {
+                    "status": "SUCCESS",
+                    "msg": "",
+                    "appId": "",
+                    "dppResult": "${dppResult}",
+                    "filePathToSize": "{\\"${etlResultFilePath}\\": 81758}",
+                    "hadoopProperties": "{\\"fs.defaultFS\\":\\"${getHdfsFs()}\\",\\"hadoop.username\\":\\"${getHdfsUser()}\\",\\"hadoop.password\\":\\"${getHdfsPasswd()}\\"}"
+                }
+            }"""
+
+        httpTest {
+            endpoint context.config.feHttpAddress
+            uri "/api/ingestion_load/${context.dbName}/_update"
+            op "post"
+            user context.config.feHttpUser
+            password context.config.feHttpPassword
+            body updateStatusReqBody
+            check { code, resBody ->
+                {
+                    assert code == 200
+                    def resBodyJson = parseJson(resBody)
+                    assert resBodyJson instanceof Map
+                    assert resBodyJson.code == 0
+                }
+            }
+        }
+
+        max_try_milli_secs = 60000
+        while (max_try_milli_secs) {
+            result = sql "show load where label = '${loadLabel}'"
+            if (result[0][2] == "FINISHED") {
+                sql "sync"
+                qt_select "select * from ${testTable} order by c_int"
+                break
+            } else {
+                sleep(5000) // wait 1 second every time
+                max_try_milli_secs -= 5000
+                if (max_try_milli_secs < 0) {
+                    assertEquals(1, 2)
+                }
+            }
+        }
+
+    }
+
+    if (enableHdfs()) {
+
+        tableName = 'tbl_test_spark_load'
+
+        sql """
+            CREATE TABLE IF NOT EXISTS ${tableName} (
+                c_int int(11) NULL,
+                c_char char(15) NULL,
+                c_varchar varchar(100) NULL,
+                c_bool boolean NULL,
+                c_tinyint tinyint(4) NULL,
+                c_smallint smallint(6) NULL,
+                c_bigint bigint(20) NULL,
+                c_largeint largeint(40) NULL,
+                c_float float NULL,
+                c_double double NULL,
+                c_decimal decimal(6, 3) NULL,
+                c_decimalv3 decimal(6, 3) NULL,
+                c_date date NULL,
+                c_datev2 date NULL,
+                c_datetime datetime NULL,
+                c_datetimev2 datetime NULL
+            )
+            DISTRIBUTED BY HASH(c_int) BUCKETS 1
+            PROPERTIES (
+            "replication_num" = "1"
+            )
+            """
+
+        def label = "test_ingestion_load"
+
+        testIngestLoadJob.call(tableName, label, context.config.dataPath + '/load_p0/ingestion_load/data.parquet')
+
+    }
+
+}