From dcf6be29d8422f3ff678639cc513d293e546593b Mon Sep 17 00:00:00 2001 From: "zky.zhoukeyong" Date: Wed, 12 Jul 2023 20:18:48 +0800 Subject: [PATCH] [CELEBORN-789] Increase default value of flushBuffer's max components ### What changes were proposed in this pull request? Set default value of ```celeborn.worker.push.compositeBuffer.maxComponents``` to 256, to be aligned with 0.2.1-incubating version. ### Why are the changes needed? Default 16 is too small, and causes ~~severe GC~~ and CPU high load. image ### Does this PR introduce _any_ user-facing change? No, it's internal config. ### How was this patch tested? Passes GA. Closes #1707 from waitinfuture/789. Authored-by: zky.zhoukeyong Signed-off-by: zky.zhoukeyong --- .../main/scala/org/apache/celeborn/common/CelebornConf.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala index d0ea3afa7..17fe8e8d7 100644 --- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala +++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala @@ -2600,7 +2600,7 @@ object CelebornConf extends Logging { "When set to 1, Netty's direct memory is close to disk buffer, but performance " + "might decrease due to frequent memory copy during compaction.") .intConf - .createWithDefault(16) + .createWithDefault(128) val WORKER_FETCH_HEARTBEAT_ENABLED: ConfigEntry[Boolean] = buildConf("celeborn.worker.fetch.heartbeat.enabled")