apache/celeborn
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Feature]Update spark2 patch (#89)

Browse Source
This commit is contained in:
Ethan Feng 2022-04-08 21:46:30 +08:00 committed by GitHub
parent db84bce438
commit 2ea136fada
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
assets/spark-patch/RSS_RDA_spark2.patch
View File

@ -24,8 +24,6 @@ diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scal
- if (!conf.get(config.SHUFFLE_SERVICE_ENABLED) && !testing) {
- throw new SparkException("Dynamic allocation of executors requires the external " +
- "shuffle service. You may enable this through spark.shuffle.service.enabled.")
+ val isRss = "org.apache.spark.shuffle.rss.RssShuffleManager" ==
+ conf.get("spark.shuffle.manager", "sort")
+ if (!conf.get(config.SHUFFLE_SERVICE_ENABLED) &&
+ !Utils.isRssEnabled(conf) && !testing) {
+ throw new SparkException("Dynamic allocation of executors requires the external or remote " +
@ -86,7 +84,7 @@ diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/ma
}
+
+ def isRssEnabled(conf: SparkConf): Boolean =
+ conf.get("spark.shuffle.manager") == "org.apache.spark.shuffle.rss.RssShuffleManager"
+ conf.get("spark.shuffle.manager", "sort") == "org.apache.spark.shuffle.rss.RssShuffleManager"
+
}