[KYUUBI #5862] Use TestContainerForAll for testing JDBC engine with testcontainers

# 🔍 Description
## Issue References 🔗

As described.

## Describe Your Solution 🔧

- replacing the usage of `ForAllTestContainer` with `TestContainerForAll`, simplifying the lifecycle for starting / stopping the containers and fetching the configs from the containers
- use `testcontainers-scala-postgresql` for testing with PostgreSQL containers
- add version 16 for PostgreSQL image tag

## Types of changes 🔖

- [ ] Bugfix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing functionality to change)

## Test Plan 🧪

#### Behavior Without This Pull Request ⚰️
No behaviour changes.

#### Behavior With This Pull Request 🎉
No behaviour changes.

#### Related Unit Tests
JDBC Engine IT.

---

# Checklists
## 📝 Author Self Checklist

- [x] My code follows the [style guidelines](https://kyuubi.readthedocs.io/en/master/contributing/code/style.html) of this project
- [x] I have performed a self-review
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my feature works
- [x] New and existing unit tests pass locally with my changes
- [x] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html)

## 📝 Committer Pre-Merge Checklist

- [ ] Pull request title is okay.
- [ ] No license issues.
- [ ] Milestone correctly set?
- [ ] Test coverage is ok
- [ ] Assignees are selected.
- [ ] Minimum number of approvals
- [ ] No changes are requested

**Be nice. Be informative.**

Closes #5862 from bowenliang123/jdbc-container.

Closes #5862

29e85121c [Bowen Liang] TestContainerForAll

Authored-by: Bowen Liang <liangbowen@gf.com.cn>
Signed-off-by: liangbowen <liangbowen@gf.com.cn>

externals/kyuubi-jdbc-engine/pom.xml

@@ -64,6 +64,12 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>com.dimafeng</groupId>
+            <artifactId>testcontainers-scala-postgresql_${scala.binary.version}</artifactId>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.kyuubi</groupId>
             <artifactId>${hive.jdbc.artifact}</artifactId>

externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/WithJdbcServerContainer.scala

@@ -16,8 +16,8 @@
  */
 package org.apache.kyuubi.engine.jdbc
 
-import com.dimafeng.testcontainers.ForAllTestContainer
+import com.dimafeng.testcontainers.scalatest.TestContainerForAll
 
 import org.apache.kyuubi.KyuubiFunSuite
 
-trait WithJdbcServerContainer extends KyuubiFunSuite with ForAllTestContainer {}
+trait WithJdbcServerContainer extends KyuubiFunSuite with TestContainerForAll {}

externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisContainer.scala

@@ -35,7 +35,7 @@ trait WithDorisContainer extends WithJdbcServerContainer {
 
   private val DORIS_BE_SERVICE_NAME = "doris-be"
 
-  override val container: DockerComposeContainer =
+  override val containerDef: DockerComposeContainer.Def =
     DockerComposeContainer
       .Def(
         composeFiles = new File(Utils.getContextOrKyuubiClassLoader
@@ -51,17 +51,11 @@ trait WithDorisContainer extends WithJdbcServerContainer {
             DORIS_BE_PORT,
             waitStrategy =
               new DockerHealthcheckWaitStrategy().withStartupTimeout(Duration.ofMinutes(5)))))
-      .createContainer()
 
-  protected def feUrl: String = {
+  protected def feUrl: String = withContainers { container =>
     val feHost: String = container.getServiceHost(DORIS_FE_SERVICE_NAME, DORIS_FE_PORT)
     val fePort: Int = container.getServicePort(DORIS_FE_SERVICE_NAME, DORIS_FE_PORT)
     val url = s"$feHost:$fePort"
     url
   }
-
-  override def afterAll(): Unit = {
-    super.afterAll()
-    container.close()
-  }
 }

externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/mysql/WithMySQLEngine.scala

@@ -27,7 +27,7 @@ trait WithMySQLEngine extends WithJdbcEngine with TestContainerForAll {
 
   private val mysqlDockerImage = "mysql:8.0.32"
 
-  override val containerDef = MySQLContainer.Def(
+  override val containerDef: MySQLContainer.Def = MySQLContainer.Def(
     dockerImageName = DockerImageName.parse(mysqlDockerImage),
     username = "root",
     password = "kyuubi")
@@ -36,8 +36,8 @@ trait WithMySQLEngine extends WithJdbcEngine with TestContainerForAll {
     Map(
       ENGINE_SHARE_LEVEL.key -> "SERVER",
       ENGINE_JDBC_CONNECTION_URL.key -> mysqlContainer.jdbcUrl,
-      ENGINE_JDBC_CONNECTION_USER.key -> "root",
-      ENGINE_JDBC_CONNECTION_PASSWORD.key -> "kyuubi",
+      ENGINE_JDBC_CONNECTION_USER.key -> mysqlContainer.username,
+      ENGINE_JDBC_CONNECTION_PASSWORD.key -> mysqlContainer.password,
       ENGINE_TYPE.key -> "jdbc",
       ENGINE_JDBC_SHORT_NAME.key -> "mysql",
       ENGINE_JDBC_DRIVER_CLASS.key -> "com.mysql.cj.jdbc.Driver")

externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/phoenix/WithPhoenixContainer.scala

@@ -16,7 +16,7 @@
  */
 package org.apache.kyuubi.engine.jdbc.phoenix
 
-import com.dimafeng.testcontainers.{GenericContainer, SingleContainer}
+import com.dimafeng.testcontainers.GenericContainer
 import org.testcontainers.containers.wait.strategy.Wait
 
 import org.apache.kyuubi.engine.jdbc.WithJdbcServerContainer
@@ -27,21 +27,15 @@ trait WithPhoenixContainer extends WithJdbcServerContainer {
 
   private val phoenixDockerImage = "iteblog/hbase-phoenix-docker:1.0"
 
-  override val container: SingleContainer[_] = GenericContainer(
+  override val containerDef: GenericContainer.Def[GenericContainer] = GenericContainer.Def(
     dockerImage = phoenixDockerImage,
     exposedPorts = Seq(PHOENIX_PORT),
     waitStrategy = Wait.forListeningPort)
 
-  protected def queryServerUrl: String = {
+  protected def queryServerUrl: String = withContainers { container =>
     val queryServerHost: String = container.host
     val queryServerPort: Int = container.mappedPort(PHOENIX_PORT)
     val url = s"$queryServerHost:$queryServerPort"
     url
   }
-
-  override def afterAll(): Unit = {
-    super.afterAll()
-    container.close()
-  }
-
 }

externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/postgresql/WithPostgreSQLContainer.scala

@@ -16,34 +16,18 @@
  */
 package org.apache.kyuubi.engine.jdbc.postgresql
 
-import com.dimafeng.testcontainers.{GenericContainer, SingleContainer}
-import org.testcontainers.containers.wait.strategy.Wait
+import com.dimafeng.testcontainers.PostgreSQLContainer
+import org.testcontainers.utility.DockerImageName
 
 import org.apache.kyuubi.engine.jdbc.WithJdbcServerContainer
 
 trait WithPostgreSQLContainer extends WithJdbcServerContainer {
 
-  private val POSTGRESQL_PORT = 5432
-
-  private val postgreSQLDockerImage = "postgres"
-
-  override val container: SingleContainer[_] = GenericContainer(
-    dockerImage = postgreSQLDockerImage,
-    exposedPorts = Seq(POSTGRESQL_PORT),
-    env = Map[String, String](
-      "POSTGRES_PASSWORD" -> "postgres"),
-    waitStrategy = Wait.forListeningPort)
-
-  protected def queryUrl: String = {
-    val queryServerHost: String = container.host
-    val queryServerPort: Int = container.mappedPort(POSTGRESQL_PORT)
-    val url = s"$queryServerHost:$queryServerPort"
-    url
-  }
-
-  override def afterAll(): Unit = {
-    super.afterAll()
-    container.close()
-  }
+  private val postgreSQLDockerImage = "postgres:16.1"
 
+  override val containerDef: PostgreSQLContainer.Def = PostgreSQLContainer.Def(
+    dockerImageName = DockerImageName.parse(postgreSQLDockerImage),
+    databaseName = "postgres",
+    username = "kyuubi",
+    password = "postgres")
 }

externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/postgresql/WithPostgreSQLEngine.scala

@@ -21,13 +21,15 @@ import org.apache.kyuubi.engine.jdbc.WithJdbcEngine
 
 trait WithPostgreSQLEngine extends WithJdbcEngine with WithPostgreSQLContainer {
 
-  override def withKyuubiConf: Map[String, String] = Map(
-    ENGINE_SHARE_LEVEL.key -> "SERVER",
-    ENGINE_JDBC_CONNECTION_URL.key -> s"jdbc:postgresql://$queryUrl/postgres",
-    ENGINE_JDBC_CONNECTION_USER.key -> "postgres",
-    ENGINE_JDBC_CONNECTION_PASSWORD.key -> "postgres",
-    ENGINE_TYPE.key -> "jdbc",
-    ENGINE_JDBC_SHORT_NAME.key -> "postgresql",
-    ENGINE_JDBC_DRIVER_CLASS.key -> "org.postgresql.Driver")
+  override def withKyuubiConf: Map[String, String] = withContainers { container =>
+    Map(
+      ENGINE_SHARE_LEVEL.key -> "SERVER",
+      ENGINE_JDBC_CONNECTION_URL.key -> container.jdbcUrl,
+      ENGINE_JDBC_CONNECTION_USER.key -> container.username,
+      ENGINE_JDBC_CONNECTION_PASSWORD.key -> container.password,
+      ENGINE_TYPE.key -> "jdbc",
+      ENGINE_JDBC_SHORT_NAME.key -> "postgresql",
+      ENGINE_JDBC_DRIVER_CLASS.key -> container.driverClassName)
+  }
 
 }

integration-tests/kyuubi-jdbc-it/pom.xml

@@ -90,6 +90,12 @@
             <artifactId>mysql-connector-j</artifactId>
             <scope>test</scope>
         </dependency>
+
+        <dependency>
+            <groupId>com.dimafeng</groupId>
+            <artifactId>testcontainers-scala-postgresql_${scala.binary.version}</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

pom.xml

@@ -574,6 +574,12 @@
                 <version>${testcontainers-scala.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>com.dimafeng</groupId>
+                <artifactId>testcontainers-scala-postgresql_${scala.binary.version}</artifactId>
+                <version>${testcontainers-scala.version}</version>
+            </dependency>
+
             <dependency>
                 <groupId>com.dimafeng</groupId>
                 <artifactId>testcontainers-scala-trino_${scala.binary.version}</artifactId>