apache/kyuubi
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

[KYUUBI-201]Add some user friendly spark configurations (#202)

Browse Source 
fix #201
This commit is contained in:
Kent Yao 2019-06-25 18:40:03 +08:00 committed by GitHub
parent f4c08c6f52
commit abc58c4ee3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 16 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
kyuubi-server/src/main/scala/org/apache/spark/KyuubiSparkUtil.scala
View File

@ -87,6 +87,7 @@ object KyuubiSparkUtil extends Logging {
// Shuffle
val PREFER_DIRECTBUF: String = SPARK_PREFIX + "shuffle." + "io.preferDirectBufs"
val PREFER_DIRECTBUF_DEFAULT = "false"
val MAX_REMOTE_BLOCK_SIZE_FETCH_TO_MEM: String = SPARK_PREFIX + "maxRemoteBlockSizeFetchToMem"
// Spark SQL
val CATALOG_IMPL: String = SPARK_PREFIX + SQL_PREFIX + "catalogImplementation"
@ -97,6 +98,13 @@ object KyuubiSparkUtil extends Logging {
val ORC_IMPL_DEFAULT = "native"
val ORC_VECTORIZED_READER_ENABLED: String =
SPARK_PREFIX + SQL_PREFIX + "orc.enableVectorizedReader"
val IGNORE_CORRUPT_FILES: String = SPARK_PREFIX + SQL_PREFIX + "files.ignoreCorruptFiles"
val IGNORE_MISSING_FILES: String = SPARK_PREFIX + SQL_PREFIX + "files.ignoreMissingFiles"
val HIVE_VERIFY_PARTITION_PATH: String =
SPARK_PREFIX + SQL_PREFIX + HIVE_PREFIX + "verifyPartitionPath"
val ENABLE_FALL_BACK_TO_HDFS_FOR_STATS: String =
SPARK_PREFIX + SQL_PREFIX + "statistics.fallBackToHdfs"
val CROSS_JOINS_ENABLED: String = SPARK_PREFIX + SQL_PREFIX + "crossJoin.enabled"
val DEPLOY_MODE: String = SPARK_PREFIX + "submit.deployMode"
val DEPLOY_MODE_DEFAULT = "client"
@ -295,6 +303,7 @@ object KyuubiSparkUtil extends Logging {
KyuubiConf.getAllDefaults.foreach(kv => conf.setIfMissing(kv._1, kv._2))
conf.setIfMissing(PREFER_DIRECTBUF, PREFER_DIRECTBUF_DEFAULT)
conf.setIfMissing(MAX_REMOTE_BLOCK_SIZE_FETCH_TO_MEM, "2047m")
conf.setIfMissing(SPARK_LOCAL_DIR, conf.get(KyuubiConf.BACKEND_SESSION_LOCAL_DIR.key))
conf.setIfMissing(GC_INTERVAL, GC_INTERVAL_DEFAULT)
if (UserGroupInformation.isSecurityEnabled) {
@ -333,6 +342,13 @@ object KyuubiSparkUtil extends Logging {
}
conf.set(SPARK_YARN_DIST_JARS, distJars)
// Spark SQL Configurations
conf.setIfMissing(IGNORE_CORRUPT_FILES, "true")
conf.setIfMissing(IGNORE_MISSING_FILES, "true")
conf.setIfMissing(HIVE_VERIFY_PARTITION_PATH, "true")
conf.setIfMissing(ENABLE_FALL_BACK_TO_HDFS_FOR_STATS, "true")
conf.setIfMissing(CROSS_JOINS_ENABLED, "true")
}
@tailrec