From 2ea136fadac01c8d115c792ce26fdde1d2ca1811 Mon Sep 17 00:00:00 2001 From: Ethan Feng Date: Fri, 8 Apr 2022 21:46:30 +0800 Subject: [PATCH] [Feature]Update spark2 patch (#89) --- assets/spark-patch/RSS_RDA_spark2.patch | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/assets/spark-patch/RSS_RDA_spark2.patch b/assets/spark-patch/RSS_RDA_spark2.patch index 2ff8106db..505d529c2 100644 --- a/assets/spark-patch/RSS_RDA_spark2.patch +++ b/assets/spark-patch/RSS_RDA_spark2.patch @@ -24,8 +24,6 @@ diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scal - if (!conf.get(config.SHUFFLE_SERVICE_ENABLED) && !testing) { - throw new SparkException("Dynamic allocation of executors requires the external " + - "shuffle service. You may enable this through spark.shuffle.service.enabled.") -+ val isRss = "org.apache.spark.shuffle.rss.RssShuffleManager" == -+ conf.get("spark.shuffle.manager", "sort") + if (!conf.get(config.SHUFFLE_SERVICE_ENABLED) && + !Utils.isRssEnabled(conf) && !testing) { + throw new SparkException("Dynamic allocation of executors requires the external or remote " + @@ -86,7 +84,7 @@ diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/ma } + + def isRssEnabled(conf: SparkConf): Boolean = -+ conf.get("spark.shuffle.manager") == "org.apache.spark.shuffle.rss.RssShuffleManager" ++ conf.get("spark.shuffle.manager", "sort") == "org.apache.spark.shuffle.rss.RssShuffleManager" + }