diff --git a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala index a10d58daa..8d0577e5d 100644 --- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala +++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala @@ -4705,7 +4705,7 @@ object CelebornConf extends Logging { val CLIENT_BATCH_HANDLE_CHANGE_PARTITION_BUCKETS: ConfigEntry[Int] = buildConf("celeborn.client.shuffle.batchHandleChangePartition.partitionBuckets") .categories("client") - .doc("Max number of change partition requests which can be concurrently processed ") + .doc("Max number of change partition requests which can be concurrently processed.") .version("0.5.0") .intConf .createWithDefault(256) @@ -4841,7 +4841,7 @@ object CelebornConf extends Logging { buildConf("celeborn.client.shuffle.dynamicResourceEnabled") .categories("client") .version("0.6.0") - .doc("When enabled, the ChangePartitionManager will obtain candidate workers from the availableWorkers pool “ +" + + .doc("When enabled, the ChangePartitionManager will obtain candidate workers from the availableWorkers pool " + "during heartbeats when worker resource change.") .booleanConf .createWithDefault(false) @@ -4851,8 +4851,8 @@ object CelebornConf extends Logging { .categories("client") .version("0.6.0") .doc("The ChangePartitionManager will check whether (unavailable workers / shuffle allocated workers) " + - "is more than the factor before obtaining candidate workers from the requestSlots RPC response" + - "when ${CLIENT_SHUFFLE_DYNAMIC_RESOURCE_ENABLED.key} set true") + "is more than the factor before obtaining candidate workers from the requestSlots RPC response " + + s"when `${CLIENT_SHUFFLE_DYNAMIC_RESOURCE_ENABLED.key}` set true") .doubleConf .checkValue(v => v >= 0.0 && v <= 1.0, "Should be in [0.0, 1.0].") .createWithDefault(0.5) diff --git a/docs/configuration/client.md b/docs/configuration/client.md index fd1160ff3..fb7d353f4 100644 --- a/docs/configuration/client.md +++ b/docs/configuration/client.md @@ -85,7 +85,7 @@ license: | | celeborn.client.rpc.reserveSlots.askTimeout | <value of celeborn.rpc.askTimeout> | false | Timeout for LifecycleManager request reserve slots. | 0.3.0 | | | celeborn.client.rpc.shared.threads | 16 | false | Number of shared rpc threads in LifecycleManager. | 0.3.2 | | | celeborn.client.shuffle.batchHandleChangePartition.interval | 100ms | false | Interval for LifecycleManager to schedule handling change partition requests in batch. | 0.3.0 | celeborn.shuffle.batchHandleChangePartition.interval | -| celeborn.client.shuffle.batchHandleChangePartition.partitionBuckets | 256 | false | Max number of change partition requests which can be concurrently processed | 0.5.0 | | +| celeborn.client.shuffle.batchHandleChangePartition.partitionBuckets | 256 | false | Max number of change partition requests which can be concurrently processed. | 0.5.0 | | | celeborn.client.shuffle.batchHandleChangePartition.threads | 8 | false | Threads number for LifecycleManager to handle change partition request in batch. | 0.3.0 | celeborn.shuffle.batchHandleChangePartition.threads | | celeborn.client.shuffle.batchHandleCommitPartition.interval | 5s | false | Interval for LifecycleManager to schedule handling commit partition requests in batch. | 0.3.0 | celeborn.shuffle.batchHandleCommitPartition.interval | | celeborn.client.shuffle.batchHandleCommitPartition.threads | 8 | false | Threads number for LifecycleManager to handle commit partition request in batch. | 0.3.0 | celeborn.shuffle.batchHandleCommitPartition.threads | @@ -95,8 +95,8 @@ license: | | celeborn.client.shuffle.compression.codec | LZ4 | false | The codec used to compress shuffle data. By default, Celeborn provides three codecs: `lz4`, `zstd`, `none`. `none` means that shuffle compression is disabled. Since Flink version 1.17, zstd is supported for Flink shuffle client. | 0.3.0 | celeborn.shuffle.compression.codec,remote-shuffle.job.compression.codec | | celeborn.client.shuffle.compression.zstd.level | 1 | false | Compression level for Zstd compression codec, its value should be an integer between -5 and 22. Increasing the compression level will result in better compression at the expense of more CPU and memory. | 0.3.0 | celeborn.shuffle.compression.zstd.level | | celeborn.client.shuffle.decompression.lz4.xxhash.instance | <undefined> | false | Decompression XXHash instance for Lz4. Available options: JNI, JAVASAFE, JAVAUNSAFE. | 0.3.2 | | -| celeborn.client.shuffle.dynamicResourceEnabled | false | false | When enabled, the ChangePartitionManager will obtain candidate workers from the availableWorkers pool “ +during heartbeats when worker resource change. | 0.6.0 | | -| celeborn.client.shuffle.dynamicResourceFactor | 0.5 | false | The ChangePartitionManager will check whether (unavailable workers / shuffle allocated workers) is more than the factor before obtaining candidate workers from the requestSlots RPC responsewhen ${CLIENT_SHUFFLE_DYNAMIC_RESOURCE_ENABLED.key} set true | 0.6.0 | | +| celeborn.client.shuffle.dynamicResourceEnabled | false | false | When enabled, the ChangePartitionManager will obtain candidate workers from the availableWorkers pool during heartbeats when worker resource change. | 0.6.0 | | +| celeborn.client.shuffle.dynamicResourceFactor | 0.5 | false | The ChangePartitionManager will check whether (unavailable workers / shuffle allocated workers) is more than the factor before obtaining candidate workers from the requestSlots RPC response when `celeborn.client.shuffle.dynamicResourceEnabled` set true | 0.6.0 | | | celeborn.client.shuffle.expired.checkInterval | 60s | false | Interval for client to check expired shuffles. | 0.3.0 | celeborn.shuffle.expired.checkInterval | | celeborn.client.shuffle.manager.port | 0 | false | Port used by the LifecycleManager on the Driver. | 0.3.0 | celeborn.shuffle.manager.port | | celeborn.client.shuffle.mapPartition.split.enabled | false | false | whether to enable shuffle partition split. Currently, this only applies to MapPartition. | 0.3.1 | |