[improve](routine-load) increase routing load max_batch _size max lim…

…it (apache#31846)
liujiwen-up · Mar 8, 2024 · 77af487 · 77af487
1 parent 37ef2e3
commit 77af487
Show file tree

Hide file tree

Showing 5 changed files with 7 additions and 6 deletions.
diff --git a/...l-manual/sql-reference/Data-Manipulation-Statements/Load/CREATE-ROUTINE-LOAD.md b/...l-manual/sql-reference/Data-Manipulation-Statements/Load/CREATE-ROUTINE-LOAD.md
@@ -161,7 +161,7 @@ FROM data_source [data_source_properties]
 
      1. The maximum execution time of each subtask, in seconds. Must be greater than or equal to 1. The default is 10.
      2. The maximum number of lines read by each subtask. Must be greater than or equal to 200000. The default is 200000.
-     3. The maximum number of bytes read by each subtask. The unit is bytes and the range is 100MB to 1GB. The default is 100MB.
+     3. The maximum number of bytes read by each subtask. The unit is bytes and the range is 100MB to 10GB. The default is 100MB.
 
      These three parameters are used to control the execution time and processing volume of a subtask. When either one reaches the threshold, the task ends.
 

diff --git a/...l-manual/sql-reference/Data-Manipulation-Statements/Load/CREATE-ROUTINE-LOAD.md b/...l-manual/sql-reference/Data-Manipulation-Statements/Load/CREATE-ROUTINE-LOAD.md
@@ -158,7 +158,7 @@ FROM data_source [data_source_properties]
 
      1. 每个子任务最大执行时间，单位是秒。必须大于等于 1。默认为10。
      2. 每个子任务最多读取的行数。必须大于等于200000。默认是200000。
-     3. 每个子任务最多读取的字节数。单位是字节，范围是 100MB 到 1GB。默认是 100MB。
+     3. 每个子任务最多读取的字节数。单位是字节，范围是 100MB 到 10GB。默认是 100MB。
 
      这三个参数，用于控制一个子任务的执行时间和处理量。当任意一个达到阈值，则任务结束。
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRoutineLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRoutineLoadStmt.java
@@ -199,7 +199,7 @@ private void checkJobProperties() throws UserException {
             long maxBatchSizeBytes = Util.getLongPropertyOrDefault(
                     jobProperties.get(CreateRoutineLoadStmt.MAX_BATCH_SIZE_PROPERTY),
                     -1, CreateRoutineLoadStmt.MAX_BATCH_SIZE_PRED,
-                    CreateRoutineLoadStmt.MAX_BATCH_SIZE_PROPERTY + " should between 100MB and 1GB");
+                    CreateRoutineLoadStmt.MAX_BATCH_SIZE_PROPERTY + " should between 100MB and 10GB");
             analyzedJobProperties.put(CreateRoutineLoadStmt.MAX_BATCH_SIZE_PROPERTY,
                     String.valueOf(maxBatchSizeBytes));
         }

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java
@@ -196,7 +196,8 @@ public class CreateRoutineLoadStmt extends DdlStmt {
     public static final Predicate<Double> MAX_FILTER_RATIO_PRED = (v) -> v >= 0 && v <= 1;
     public static final Predicate<Long> MAX_BATCH_INTERVAL_PRED = (v) -> v >= 1;
     public static final Predicate<Long> MAX_BATCH_ROWS_PRED = (v) -> v >= 200000;
-    public static final Predicate<Long> MAX_BATCH_SIZE_PRED = (v) -> v >= 100 * 1024 * 1024 && v <= 1024 * 1024 * 1024;
+    public static final Predicate<Long> MAX_BATCH_SIZE_PRED = (v) -> v >= 100 * 1024 * 1024
+                                                            && v <= (long) (1024 * 1024 * 1024) * 10;
     public static final Predicate<Long> EXEC_MEM_LIMIT_PRED = (v) -> v >= 0L;
     public static final Predicate<Long> SEND_BATCH_PARALLELISM_PRED = (v) -> v > 0L;
 
@@ -482,7 +483,7 @@ private void checkJobProperties() throws UserException {
 
         maxBatchSizeBytes = Util.getLongPropertyOrDefault(jobProperties.get(MAX_BATCH_SIZE_PROPERTY),
                 RoutineLoadJob.DEFAULT_MAX_BATCH_SIZE, MAX_BATCH_SIZE_PRED,
-                MAX_BATCH_SIZE_PROPERTY + " should between 100MB and 1GB");
+                MAX_BATCH_SIZE_PROPERTY + " should between 100MB and 10GB");
 
         strictMode = Util.getBooleanPropertyOrDefault(jobProperties.get(LoadStmt.STRICT_MODE),
                 RoutineLoadJob.DEFAULT_STRICT_MODE,

diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/AlterRoutineLoadStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/AlterRoutineLoadStmtTest.java
@@ -220,7 +220,7 @@ public void testUnsupportedProperties() throws MetaNotFoundException {
                 stmt.analyze(analyzer);
                 Assert.fail();
             } catch (AnalysisException e) {
-                Assert.assertTrue(e.getMessage().contains("max_batch_size should between 100MB and 1GB"));
+                Assert.assertTrue(e.getMessage().contains("max_batch_size should between 100MB and 10GB"));
             } catch (UserException e) {
                 Assert.fail();
             }