diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/FrontendService.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/FrontendService.scala
index 218857147..a2f348d6b 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/FrontendService.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/FrontendService.scala
@@ -103,7 +103,12 @@ class FrontendService private (name: String, be: BackendService, oomHook: Runnab
     super.initialize(conf)
   }
 
-  def connectionUrl: String = s"${serverAddr.getCanonicalHostName}:$portNum"
+  def connectionUrl: String = {
+    getServiceState match {
+      case s @ ServiceState.LATENT => throw new IllegalStateException(s"Illegal Service State: $s")
+      case _ => s"${serverAddr.getCanonicalHostName}:$portNum"
+    }
+  }
 
   override def start(): Unit = synchronized {
     super.start()
diff --git a/kyuubi-main/src/test/scala/org/apache/kyuubi/server/KyuubiServerSuite.scala b/kyuubi-main/src/test/scala/org/apache/kyuubi/server/KyuubiServerSuite.scala
index 4c890ebe7..2882162d1 100644
--- a/kyuubi-main/src/test/scala/org/apache/kyuubi/server/KyuubiServerSuite.scala
+++ b/kyuubi-main/src/test/scala/org/apache/kyuubi/server/KyuubiServerSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.kyuubi.server
 
-import org.apache.kyuubi.KyuubiFunSuite
+import org.apache.kyuubi.{KyuubiException, KyuubiFunSuite}
 import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.service.ServiceState._
 
@@ -28,11 +28,15 @@ class KyuubiServerSuite extends KyuubiFunSuite {
     val conf = KyuubiConf().set(KyuubiConf.FRONTEND_BIND_PORT, 0)
     assert(server.getServices.isEmpty)
     assert(server.getServiceState === LATENT)
+    val e = intercept[IllegalStateException](server.connectionUrl)
+    assert(e.getMessage === "Illegal Service State: LATENT")
+
     server.initialize(conf)
     assert(server.getServiceState === INITIALIZED)
     val backendService = server.getServices.head.asInstanceOf[KyuubiBackendService]
     assert(backendService.getServiceState == INITIALIZED)
     assert(backendService.getServices.forall(_.getServiceState === INITIALIZED))
+    assert(server.connectionUrl.split(":").length === 2)
 
     server.start()
     assert(server.getServiceState === STARTED)
@@ -45,4 +49,10 @@ class KyuubiServerSuite extends KyuubiFunSuite {
     assert(backendService.getServices.forall(_.getServiceState === STOPPED))
   }
 
+  test("invalid port") {
+    val conf = KyuubiConf().set(KyuubiConf.FRONTEND_BIND_PORT, 100)
+    val e = intercept[KyuubiException](new KyuubiServer().initialize(conf))
+    assert(e.getMessage === "Failed to initialize frontend service")
+    assert(e.getCause.getMessage === "Invalid Port number")
+  }
 }