diff --git a/kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeper.scala b/kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeper.scala
index 3301c2aa6..24aaba2bd 100644
--- a/kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeper.scala
+++ b/kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeper.scala
@@ -18,7 +18,7 @@
 package org.apache.kyuubi.zookeeper
 
 import java.io.File
-import java.net.{InetAddress, InetSocketAddress}
+import java.net.InetSocketAddress
 
 import org.apache.zookeeper.server.{NIOServerCnxnFactory, ZooKeeperServer}
 
@@ -34,6 +34,7 @@ class EmbeddedZookeeper extends AbstractService("EmbeddedZookeeper") {
   private var dataDirectory: File = _
   // TODO: Is it right in prod?
   private val deleteDataDirectoryOnClose = true
+  private var host: String = _
 
   override def initialize(conf: KyuubiConf): Unit = synchronized {
     dataDirectory = new File(conf.get(ZK_DATA_DIR))
@@ -42,15 +43,15 @@ class EmbeddedZookeeper extends AbstractService("EmbeddedZookeeper") {
     val maxClientCnxns = conf.get(ZK_MAX_CLIENT_CONNECTIONS)
     val minSessionTimeout = conf.get(ZK_MIN_SESSION_TIMEOUT)
     val maxSessionTimeout = conf.get(ZK_MAX_SESSION_TIMEOUT)
-    val hostname = conf.get(ZK_CLIENT_PORT_ADDRESS).map(InetAddress.getByName)
-      .getOrElse(findLocalInetAddress).getCanonicalHostName
+    host = conf.get(ZK_CLIENT_PORT_ADDRESS)
+      .getOrElse(findLocalInetAddress.getCanonicalHostName)
 
     zks = new ZooKeeperServer(dataDirectory, dataDirectory, tickTime)
     zks.setMinSessionTimeout(minSessionTimeout)
     zks.setMaxSessionTimeout(maxSessionTimeout)
 
     serverFactory = new NIOServerCnxnFactory
-    serverFactory.configure(new InetSocketAddress(hostname, clientPort), maxClientCnxns)
+    serverFactory.configure(new InetSocketAddress(host, clientPort), maxClientCnxns)
 
     super.initialize(conf)
   }
@@ -74,7 +75,6 @@ class EmbeddedZookeeper extends AbstractService("EmbeddedZookeeper") {
 
   def getConnectString: String = synchronized {
     assert(zks != null, s"$getName is in $getServiceState")
-    s"${serverFactory.getLocalAddress.getHostName}:${serverFactory.getLocalPort}"
+    s"${host}:${serverFactory.getLocalPort}"
   }
-
 }
diff --git a/kyuubi-zookeeper/src/test/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeperSuite.scala b/kyuubi-zookeeper/src/test/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeperSuite.scala
index a0815931d..837e0c355 100644
--- a/kyuubi-zookeeper/src/test/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeperSuite.scala
+++ b/kyuubi-zookeeper/src/test/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeperSuite.scala
@@ -24,6 +24,7 @@ import org.apache.curator.retry.ExponentialBackoffRetry
 import org.apache.kyuubi.KyuubiFunSuite
 import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.service.ServiceState._
+import org.apache.kyuubi.zookeeper.ZookeeperConf.{ZK_CLIENT_PORT, ZK_CLIENT_PORT_ADDRESS}
 
 class EmbeddedZookeeperSuite extends KyuubiFunSuite {
 
@@ -62,4 +63,20 @@ class EmbeddedZookeeperSuite extends KyuubiFunSuite {
     assert(zkClient.getState === CuratorFrameworkState.STARTED)
     assert(zkClient.getZookeeperClient.blockUntilConnectedOrTimedOut())
   }
+
+  test("use zookeeper.embedded.client.port.address cover default hostname") {
+    var zkServer = new EmbeddedZookeeper()
+    // cover default hostname
+    var conf = KyuubiConf()
+      .set(ZK_CLIENT_PORT, 0)
+      .set(ZK_CLIENT_PORT_ADDRESS, "localhost")
+    zkServer.initialize(conf)
+    assert(zkServer.getConnectString.contains("localhost"))
+    zkServer = new EmbeddedZookeeper()
+    conf = KyuubiConf()
+      .set(ZK_CLIENT_PORT, 0)
+      .set(ZK_CLIENT_PORT_ADDRESS, "127.0.0.1")
+    zkServer.initialize(conf)
+    assert(zkServer.getConnectString.contains("127.0.0.1"))
+  }
 }