apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

[KYUUBI-169]Let spark prefer heap buffer by default while using netty rpc #172

Browse Source 
fix #169 set spark.shuffle.io.preferDirectBufs = false by default (#172)
This commit is contained in:
Kent Yao 2019-03-21 16:33:47 +08:00 committed by GitHub
parent ba9b424110
commit 7790bbd9df
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
kyuubi-server/src/main/scala/org/apache/spark/KyuubiSparkUtil.scala
View File

@ -84,6 +84,10 @@ object KyuubiSparkUtil extends Logging {
val MULTIPLE_CONTEXTS: String = SPARK_PREFIX + DRIVER_PREFIX + "allowMultipleContexts"
val MULTIPLE_CONTEXTS_DEFAULT = "true"
// Shuffle
val PREFER_DIRECTBUF: String = SPARK_PREFIX + "shuffle." + "io.preferDirectBufs"
val PREFER_DIRECTBUF_DEFAULT = "false"
// Spark SQL
val CATALOG_IMPL: String = SPARK_PREFIX + SQL_PREFIX + "catalogImplementation"
val CATALOG_IMPL_DEFAULT = "hive"
@ -295,6 +299,7 @@ object KyuubiSparkUtil extends Logging {
// Set missing Kyuubi configs to SparkConf
KyuubiConf.getAllDefaults.foreach(kv => conf.setIfMissing(kv._1, kv._2))
conf.setIfMissing(PREFER_DIRECTBUF, PREFER_DIRECTBUF_DEFAULT)
conf.setIfMissing(SPARK_LOCAL_DIR, conf.get(KyuubiConf.BACKEND_SESSION_LOCAL_DIR.key))
conf.setIfMissing(GC_INTERVAL, GC_INTERVAL_DEFAULT)
if (UserGroupInformation.isSecurityEnabled) {