[KYUUBI #5522] [BATCH] Ignore main class for PySpark batch job submission
### _Why are the changes needed?_ For PySpark batch jobs, the main class is not required for spark submission and should be ignored. - Spark submit doc: https://spark.apache.org/docs/latest/submitting-applications.html - Also reported in : https://github.com/apache/kyuubi/pull/5498/files#diff-a215909f2588cb42eb307d1bdaa73eef9e05259bb0a99099f29859235bec19bbR30 ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request ### _Was this patch authored or co-authored using generative AI tooling?_ No. Closes #5522 from bowenliang123/pyspark-submit-ignore. Closes #5522 c28ef86da [liangbowen] ignore main class for pyspark batch submission Authored-by: liangbowen <liangbowen@gf.com.cn> Signed-off-by: Bowen Liang <liangbowen@gf.com.cn>
This commit is contained in:
parent
cb93d6fd1a
commit
bf008de1e1
@ -84,22 +84,21 @@ class BatchJobSubmission(
|
||||
|
||||
@VisibleForTesting
|
||||
private[kyuubi] val builder: ProcBuilder = {
|
||||
Option(batchType).map(_.toUpperCase(Locale.ROOT)) match {
|
||||
case Some("SPARK") | Some("PYSPARK") =>
|
||||
new SparkBatchProcessBuilder(
|
||||
session.user,
|
||||
session.sessionConf,
|
||||
batchId,
|
||||
batchName,
|
||||
Option(resource),
|
||||
className,
|
||||
batchConf,
|
||||
batchArgs,
|
||||
getOperationLog)
|
||||
|
||||
case _ =>
|
||||
throw new UnsupportedOperationException(s"Batch type $batchType unsupported")
|
||||
val mainClass = Option(batchType).map(_.toUpperCase(Locale.ROOT)) match {
|
||||
case Some("SPARK") => className
|
||||
case Some("PYSPARK") => null
|
||||
case _ => throw new UnsupportedOperationException(s"Batch type $batchType unsupported")
|
||||
}
|
||||
new SparkBatchProcessBuilder(
|
||||
session.user,
|
||||
session.sessionConf,
|
||||
batchId,
|
||||
batchName,
|
||||
Option(resource),
|
||||
mainClass,
|
||||
batchConf,
|
||||
batchArgs,
|
||||
getOperationLog)
|
||||
}
|
||||
|
||||
override def currentApplicationInfo(): Option[ApplicationInfo] = {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user