apache/celeborn
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[CELEBORN-304] Missing setIfMissing celeborn.$module.io.serverThreads (#1238)

Browse Source
This commit is contained in:
jiaoqingbo 2023-02-15 15:49:08 +08:00 committed by GitHub
parent 75c83093f2
commit bd9e0ddc1f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
client-spark/spark-3/src/main/scala/org/apache/spark/sql/execution/columnar/RssCompressibleColumnBuilder.scala
View File

@ -44,7 +44,7 @@ trait RssCompressibleColumnBuilder[T <: AtomicType]
// The various compression schemes, while saving memory use, cause all of the data within
// the row to become unaligned, thus causing crashes. Until a way of fixing the compression
// is found to also allow aligned accesses this must be disabled for SPARC.
// is found to also allow aligned accesses this must be disabled for SPARK.
protected def isWorthCompressing(encoder: Encoder[T]) = {
RssCompressibleColumnBuilder.unaligned && encoder.compressionRatio < 0.8

4
client/src/main/java/org/apache/celeborn/client/ShuffleClient.java
View File

@ -199,7 +199,7 @@ public abstract class ShuffleClient {
// Write data to a specific map partition, input data's type is Bytebuf.
// data's type is Bytebuf to avoid copy between application and netty
// closecallback will do some clean opertions like memory release.
// closecallback will do some clean operations like memory release.
public abstract int pushDataToLocation(
String applicationId,
int shuffleId,
@ -209,7 +209,7 @@ public abstract class ShuffleClient {
ByteBuf data,
PartitionLocation location,
BooleanSupplier closeCallBack)
throws IOException;;
throws IOException;
public abstract Optional<PartitionLocation> regionStart(
String applicationId,

2
common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
View File

@ -497,7 +497,7 @@ object Utils extends Logging {
// assuming we have all the machine's cores).
// NB: Only set if serverThreads/clientThreads not already set.
val numThreads = defaultNumThreads(numUsableCores)
conf.setIfMissing(s"celeborn.$module.io.clientThreads", numThreads.toString)
conf.setIfMissing(s"celeborn.$module.io.serverThreads", numThreads.toString)
conf.setIfMissing(s"celeborn.$module.io.clientThreads", numThreads.toString)
// TODO remove after releasing 0.2.0
conf.setIfMissing(s"rss.$module.io.serverThreads", numThreads.toString)