diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/Serverable.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/Serverable.scala
index ab66a64c0..670210264 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/Serverable.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/Serverable.scala
@@ -31,20 +31,21 @@ abstract class Serverable(name: String) extends CompositeService(name) {
 
   def connectionUrl: String = frontendService.connectionUrl
 
-  override def initialize(conf: KyuubiConf): Unit = {
+  override def initialize(conf: KyuubiConf): Unit = synchronized {
     addService(backendService)
     addService(frontendService)
     super.initialize(conf)
   }
 
-  override def start(): Unit = {
-    super.start()
-    started.set(true)
+  override def start(): Unit = synchronized {
+    if (!started.getAndSet(true)) {
+      super.start()
+    }
   }
 
   protected def stopServer(): Unit
 
-  override def stop(): Unit = {
+  override def stop(): Unit = synchronized {
     try {
       stopServer()
     } catch {
diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala
index eb42f4659..fe2be01ae 100644
--- a/kyuubi-common/src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala
+++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala
@@ -35,13 +35,13 @@ trait KerberizedTestHelper extends KyuubiFunSuite {
     this.getClass.getProtectionDomain.getCodeSource.getLocation.getPath, "kyuubi-kdc").toFile
   val kdcConf = MiniKdc.createConf()
   val hostName = "localhost"
-  kdcConf.setProperty(MiniKdc.INSTANCE, "KyuubiKrbServer")
-  kdcConf.setProperty(MiniKdc.ORG_NAME, "KYUUBI")
+  kdcConf.setProperty(MiniKdc.INSTANCE, this.getClass.getSimpleName)
+  kdcConf.setProperty(MiniKdc.ORG_NAME, this.getClass.getSimpleName)
   kdcConf.setProperty(MiniKdc.ORG_DOMAIN, "COM")
   kdcConf.setProperty(MiniKdc.KDC_BIND_ADDRESS, hostName)
-  if (logger.isDebugEnabled) {
-    kdcConf.setProperty(MiniKdc.DEBUG, "true")
-  }
+  kdcConf.setProperty(MiniKdc.KDC_PORT, "0")
+  kdcConf.setProperty(MiniKdc.DEBUG, "true")
+
   private var kdc: MiniKdc = _
 
   eventually(timeout(60.seconds), interval(1.second)) {
@@ -59,9 +59,7 @@ trait KerberizedTestHelper extends KyuubiFunSuite {
   }
 
   private val keytabFile = new File(baseDir, "kyuubi-test.keytab")
-
   protected val testKeytab: String = keytabFile.getAbsolutePath
-
   protected var testPrincipal = s"client/$hostName"
   kdc.createPrincipal(keytabFile, testPrincipal)
 
@@ -93,7 +91,7 @@ trait KerberizedTestHelper extends KyuubiFunSuite {
 
     kdc.getKrb5conf.delete()
     Files.write(krb5confStr, kdc.getKrb5conf, StandardCharsets.UTF_8)
-    debug(s"krb5.conf file content: $krb5confStr")
+    info(s"krb5.conf file content: $krb5confStr")
   }
 
   private def addedKrb5Config(key: String, value: String): String = {
@@ -101,6 +99,7 @@ trait KerberizedTestHelper extends KyuubiFunSuite {
   }
 
   rewriteKrb5Conf()
+
   testPrincipal = testPrincipal + "@" + kdc.getRealm
 
   info(s"KerberizedTest Principal: $testPrincipal")
diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/KinitAuxiliaryServiceSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/KinitAuxiliaryServiceSuite.scala
index b35f67b6d..7cc4ded37 100644
--- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/KinitAuxiliaryServiceSuite.scala
+++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/KinitAuxiliaryServiceSuite.scala
@@ -51,7 +51,6 @@ class KinitAuxiliaryServiceSuite extends KerberizedTestHelper {
       assert(service.getServiceState === ServiceState.STARTED)
       service.stop()
       assert(service.getServiceState === ServiceState.STOPPED)
-
     }
   }
 
diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/ServerableSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/ServerableSuite.scala
index 86941e89a..474f3d505 100644
--- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/ServerableSuite.scala
+++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/ServerableSuite.scala
@@ -32,23 +32,24 @@ class ServerableSuite extends KyuubiFunSuite {
     assert(serverable.getServiceState === ServiceState.LATENT)
     intercept[IllegalStateException](serverable.start())
 
+    val serverable1 = new NoopServer()
     val conf = KyuubiConf().set(KyuubiConf.FRONTEND_BIND_PORT, 0)
-    serverable.initialize(conf)
-    serverable.stop()
-    assert(serverable.getStartTime === 0)
-    assert(serverable.getConf === conf)
-    assert(serverable.connectionUrl.nonEmpty)
-    assert(serverable.getServiceState === ServiceState.INITIALIZED)
-    serverable.start()
-    assert(serverable.getStartTime !== 0)
-    assert(serverable.getConf === conf)
-    assert(serverable.getServiceState === ServiceState.STARTED)
-    serverable.stop()
-    assert(serverable.getStartTime !== 0)
-    assert(serverable.getConf === conf)
-    assert(serverable.connectionUrl.nonEmpty)
-    assert(serverable.getServiceState === ServiceState.STOPPED)
-    serverable.stop()
+    serverable1.initialize(conf)
+    serverable1.stop()
+    assert(serverable1.getStartTime === 0)
+    assert(serverable1.getConf === conf)
+    assert(serverable1.connectionUrl.nonEmpty)
+    assert(serverable1.getServiceState === ServiceState.INITIALIZED)
+    serverable1.start()
+    assert(serverable1.getStartTime !== 0)
+    assert(serverable1.getConf === conf)
+    assert(serverable1.getServiceState === ServiceState.STARTED)
+    serverable1.stop()
+    assert(serverable1.getStartTime !== 0)
+    assert(serverable1.getConf === conf)
+    assert(serverable1.connectionUrl.nonEmpty)
+    assert(serverable1.getServiceState === ServiceState.STOPPED)
+    serverable1.stop()
   }
 
   test("invalid port") {
diff --git a/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/ServiceDiscoverySuite.scala b/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/ServiceDiscoverySuite.scala
index 2a323a182..b40078545 100644
--- a/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/ServiceDiscoverySuite.scala
+++ b/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/ServiceDiscoverySuite.scala
@@ -23,13 +23,13 @@ import javax.security.auth.login.Configuration
 
 import scala.collection.JavaConverters._
 
-import org.apache.kyuubi.{KerberizedTestHelper, KYUUBI_VERSION, KyuubiFunSuite}
+import org.apache.kyuubi.{KerberizedTestHelper, KYUUBI_VERSION}
 import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.ha.HighAvailabilityConf._
 import org.apache.kyuubi.ha.server.EmbeddedZkServer
 import org.apache.kyuubi.service.{NoopServer, Serverable, ServiceState}
 
-class ServiceDiscoverySuite extends KyuubiFunSuite with KerberizedTestHelper {
+class ServiceDiscoverySuite extends KerberizedTestHelper {
   val zkServer = new EmbeddedZkServer()
   val conf: KyuubiConf = KyuubiConf()
 
diff --git a/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/ZooKeeperACLProviderSuite.scala b/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/ZooKeeperACLProviderSuite.scala
index 79df06fbc..982f25c1e 100644
--- a/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/ZooKeeperACLProviderSuite.scala
+++ b/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/ZooKeeperACLProviderSuite.scala
@@ -19,9 +19,9 @@ package org.apache.kyuubi.ha.client
 
 import org.apache.zookeeper.ZooDefs
 
-import org.apache.kyuubi.{KerberizedTestHelper, KyuubiFunSuite}
+import org.apache.kyuubi.KerberizedTestHelper
 
-class ZooKeeperACLProviderSuite extends KyuubiFunSuite with KerberizedTestHelper {
+class ZooKeeperACLProviderSuite extends KerberizedTestHelper {
 
   test("acl for zookeeper") {
     val provider = new ZooKeeperACLProvider()