diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index 5e9010215..5ca5c78a8 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -193,15 +193,15 @@ case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging {
     cloned
   }
 
-  private lazy val serverOnlyPrefixes = get(KyuubiConf.SERVER_ONLY_PREFIXES)
-  private lazy val serverOnlyPrefixConfigKeys = settings.keys().asScala
+  private lazy val serverOnlyPrefixes: Set[String] = get(KyuubiConf.SERVER_ONLY_PREFIXES)
+  private lazy val serverOnlyPrefixConfigKeys: Set[String] = settings.keys().asScala
     // for ConfigEntry, respect the serverOnly flag and exclude it here
     .filter(key => getConfigEntry(key) == null)
     .filter { key =>
       serverOnlyPrefixes.exists { prefix =>
         key.startsWith(prefix)
       }
-    }
+    }.toSet
 
   def getUserDefaults(user: String): KyuubiConf = {
     val cloned = KyuubiConf(false)
diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala
index 4b1a45fbc..45c9b3c36 100644
--- a/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala
+++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala
@@ -231,9 +231,10 @@ class KyuubiConfSuite extends KyuubiFunSuite {
       Some("/var/run/secrets/kubernetes.io/token.ns2"))
   }
 
-  test("KYUUBI #7053 - Support to exclude server only configs with prefixes") {
+  test("KYUUBI #7055 - Support to exclude server only configs with prefixes") {
     val kyuubiConf = KyuubiConf(false)
     kyuubiConf.set("kyuubi.backend.server.event.kafka.broker", "localhost:9092")
     assert(kyuubiConf.getUserDefaults("kyuubi").getAll.size == 0)
+    assert(kyuubiConf.getUserDefaults("user").getAll.size == 0)
   }
 }