Skip to content

Commit eefb544

Browse files
sollhuidataroaring
authored andcommitted
[chore](routine-load) increase routine load job default max batch size and rows (#36632)
Most users only care about the size of **max_batch_interval**, but in order to achieve an interval effect, they have to configure **max_batch_rows** and **max_batch_size** according to the characteristics of the data. By adjusting these two default values, users do not need to worry about configuration in most scenarios.
1 parent 5fd059d commit eefb544

File tree

2 files changed

+3
-4
lines changed

2 files changed

+3
-4
lines changed

fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,8 +109,8 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
109109
public static final double DEFAULT_MAX_FILTER_RATIO = 1.0;
110110

111111
public static final long DEFAULT_MAX_INTERVAL_SECOND = 10;
112-
public static final long DEFAULT_MAX_BATCH_ROWS = 200000;
113-
public static final long DEFAULT_MAX_BATCH_SIZE = 100 * 1024 * 1024; // 100MB
112+
public static final long DEFAULT_MAX_BATCH_ROWS = 20000000;
113+
public static final long DEFAULT_MAX_BATCH_SIZE = 1024 * 1024 * 1024; // 1GB
114114
public static final long DEFAULT_EXEC_MEM_LIMIT = 2 * 1024 * 1024 * 1024L;
115115
public static final boolean DEFAULT_STRICT_MODE = false; // default is false
116116
public static final int DEFAULT_SEND_BATCH_PARALLELISM = 1;

fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -355,7 +355,6 @@ public void testGetShowCreateInfo() throws UserException {
355355
11, "localhost:9092", "test_topic", UserIdentity.ADMIN);
356356
Deencapsulation.setField(routineLoadJob, "maxErrorNum", 10);
357357
Deencapsulation.setField(routineLoadJob, "maxBatchRows", 10);
358-
Deencapsulation.setField(routineLoadJob, "maxBatchRows", 10);
359358
String showCreateInfo = routineLoadJob.getShowCreateInfo();
360359
String expect = "CREATE ROUTINE LOAD test_load ON 11\n"
361360
+ "WITH APPEND\n"
@@ -366,7 +365,7 @@ public void testGetShowCreateInfo() throws UserException {
366365
+ "\"max_filter_ratio\" = \"1.0\",\n"
367366
+ "\"max_batch_interval\" = \"10\",\n"
368367
+ "\"max_batch_rows\" = \"10\",\n"
369-
+ "\"max_batch_size\" = \"104857600\",\n"
368+
+ "\"max_batch_size\" = \"1073741824\",\n"
370369
+ "\"format\" = \"csv\",\n"
371370
+ "\"strip_outer_array\" = \"false\",\n"
372371
+ "\"num_as_string\" = \"false\",\n"

0 commit comments

Comments
 (0)