From a0b04d0036d00bbc665e0c24565388aa9946abbd Mon Sep 17 00:00:00 2001 From: Sanskar Modi Date: Mon, 12 Aug 2024 15:04:43 +0800 Subject: [PATCH] [CELEBORN-1550] Add support of providing custom dynamic store backend implementation ### What changes were proposed in this pull request? Adding support of providing custom dynamic store backend implementation, users can now pass there own implementation for dynamic config store backend. This change also keep the backwards compatibility of supporting short names for backend like "FS" and "DB" ### Why are the changes needed? Currently celeborn only supports File and DB based backend while there can be other ways of managing these configs. ### Does this PR introduce _any_ user-facing change? NO, user facing behaviour will be same. ### How was this patch tested? Existing UTs verifies that this change is working for "FS" and "DB" implementation. Closes #2670 from s0nskar/dynamic_config. Authored-by: Sanskar Modi Signed-off-by: zky.zhoukeyong --- .../apache/celeborn/common/CelebornConf.scala | 6 ++--- .../apache/celeborn/common/util/Utils.scala | 13 ++++++++++ docs/configuration/master.md | 2 +- docs/configuration/worker.md | 2 +- docs/developers/configuration.md | 5 ++-- .../common/service/config/ConfigService.java | 7 ++++++ .../service/config/DbConfigServiceImpl.java | 5 ++++ .../config/DynamicConfigServiceFactory.java | 25 ++++++++++++++----- .../service/config/FsConfigServiceImpl.java | 5 ++++ 9 files changed, 57 insertions(+), 13 deletions(-) diff --git a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala index 55173484a..c73331a67 100644 --- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala +++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala @@ -5253,12 +5253,12 @@ object CelebornConf extends Logging { val DYNAMIC_CONFIG_STORE_BACKEND: OptionalConfigEntry[String] = buildConf("celeborn.dynamicConfig.store.backend") .categories("master", "worker") - .doc("Store backend for dynamic config service. Available options: FS, DB. " + + .doc("Store backend for dynamic config service. The backend can be specified in two ways:" + + " - Using short names: Default available options are FS, DB." + + " - Using the fully qualified class name of the backend implementation." + "If not provided, it means that dynamic configuration is disabled.") .version("0.4.0") .stringConf - .transform(_.toUpperCase(Locale.ROOT)) - .checkValues(Set("FS", "DB")) .createOptional val DYNAMIC_CONFIG_REFRESH_INTERVAL: ConfigEntry[Long] = diff --git a/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala b/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala index 44a7d5c2e..02650dc1b 100644 --- a/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala +++ b/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala @@ -586,6 +586,19 @@ object Utils extends Logging { } } + def instantiateDynamicConfigStoreBackend[T](className: String, conf: CelebornConf): T = { + try { + DynConstructors.builder().impl(className, classOf[CelebornConf]) + .build[T]() + .newInstance(conf) + } catch { + case e: Throwable => + throw new CelebornException( + s"Failed to instantiate dynamic config store backend $className.", + e) + } + } + def getCodeSourceLocation(clazz: Class[_]): String = { new File(clazz.getProtectionDomain.getCodeSource.getLocation.toURI).getPath } diff --git a/docs/configuration/master.md b/docs/configuration/master.md index b7d36c4b3..537904f31 100644 --- a/docs/configuration/master.md +++ b/docs/configuration/master.md @@ -21,7 +21,7 @@ license: | | --- | ------- | --------- | ----------- | ----- | ---------- | | celeborn.cluster.name | default | false | Celeborn cluster name. | 0.5.0 | | | celeborn.dynamicConfig.refresh.interval | 120s | false | Interval for refreshing the corresponding dynamic config periodically. | 0.4.0 | | -| celeborn.dynamicConfig.store.backend | <undefined> | false | Store backend for dynamic config service. Available options: FS, DB. If not provided, it means that dynamic configuration is disabled. | 0.4.0 | | +| celeborn.dynamicConfig.store.backend | <undefined> | false | Store backend for dynamic config service. The backend can be specified in two ways: - Using short names: Default available options are FS, DB. - Using the fully qualified class name of the backend implementation.If not provided, it means that dynamic configuration is disabled. | 0.4.0 | | | celeborn.dynamicConfig.store.db.fetch.pageSize | 1000 | false | The page size for db store to query configurations. | 0.5.0 | | | celeborn.dynamicConfig.store.db.hikari.connectionTimeout | 30s | false | The connection timeout that a client will wait for a connection from the pool for db store backend. | 0.5.0 | | | celeborn.dynamicConfig.store.db.hikari.driverClassName | | false | The jdbc driver class name of db store backend. | 0.5.0 | | diff --git a/docs/configuration/worker.md b/docs/configuration/worker.md index 1605f1f33..75f861feb 100644 --- a/docs/configuration/worker.md +++ b/docs/configuration/worker.md @@ -21,7 +21,7 @@ license: | | --- | ------- | --------- | ----------- | ----- | ---------- | | celeborn.cluster.name | default | false | Celeborn cluster name. | 0.5.0 | | | celeborn.dynamicConfig.refresh.interval | 120s | false | Interval for refreshing the corresponding dynamic config periodically. | 0.4.0 | | -| celeborn.dynamicConfig.store.backend | <undefined> | false | Store backend for dynamic config service. Available options: FS, DB. If not provided, it means that dynamic configuration is disabled. | 0.4.0 | | +| celeborn.dynamicConfig.store.backend | <undefined> | false | Store backend for dynamic config service. The backend can be specified in two ways: - Using short names: Default available options are FS, DB. - Using the fully qualified class name of the backend implementation.If not provided, it means that dynamic configuration is disabled. | 0.4.0 | | | celeborn.dynamicConfig.store.db.fetch.pageSize | 1000 | false | The page size for db store to query configurations. | 0.5.0 | | | celeborn.dynamicConfig.store.db.hikari.connectionTimeout | 30s | false | The connection timeout that a client will wait for a connection from the pool for db store backend. | 0.5.0 | | | celeborn.dynamicConfig.store.db.hikari.driverClassName | | false | The jdbc driver class name of db store backend. | 0.5.0 | | diff --git a/docs/developers/configuration.md b/docs/developers/configuration.md index 84f48fb8a..533cfa988 100644 --- a/docs/developers/configuration.md +++ b/docs/developers/configuration.md @@ -46,8 +46,9 @@ If tenant-user-level configuration is missing, it will fall back to the tenant-l ## Config Service The config service provides a configuration management service with a local cache for both static and dynamic configurations. Moreover, `ConfigService` is a pluggable service interface whose implementation can vary based on different storage backends. The storage backend for `ConfigService` is specified by the -configuration key `celeborn.dynamicConfig.store.backend`, and it currently supports both filesystem (`FS`) and database (`DB`) as storage backends. If no -storage backend is specified, this indicates that the config service is disabled. +configuration key `celeborn.dynamicConfig.store.backend`, and it currently supports filesystem (`FS`) and database (`DB`) as storage backends by default. +Additionally, users can provide their own implementation by extending the `ConfigService` interface and using the fully qualified class name of the implementation +as storage backend. If no storage backend is specified, this indicates that the config service is disabled. ### FileSystem Config Service The filesystem config service enables the use of dynamic configuration files, the location of which is set by the configuration key `celeborn.dynamicConfig.store.fs.path`. diff --git a/service/src/main/java/org/apache/celeborn/server/common/service/config/ConfigService.java b/service/src/main/java/org/apache/celeborn/server/common/service/config/ConfigService.java index 1d8531654..94e3d6dd1 100644 --- a/service/src/main/java/org/apache/celeborn/server/common/service/config/ConfigService.java +++ b/service/src/main/java/org/apache/celeborn/server/common/service/config/ConfigService.java @@ -110,4 +110,11 @@ public interface ConfigService { /** Shutdowns configuration management service. */ void shutdown(); + + /** + * Retrieves the name of the configuration service. + * + * @return The name of the configuration service + */ + String getName(); } diff --git a/service/src/main/java/org/apache/celeborn/server/common/service/config/DbConfigServiceImpl.java b/service/src/main/java/org/apache/celeborn/server/common/service/config/DbConfigServiceImpl.java index 7ef0fcd5f..993b669df 100644 --- a/service/src/main/java/org/apache/celeborn/server/common/service/config/DbConfigServiceImpl.java +++ b/service/src/main/java/org/apache/celeborn/server/common/service/config/DbConfigServiceImpl.java @@ -57,6 +57,11 @@ public class DbConfigServiceImpl extends BaseConfigServiceImpl implements Config Function.identity()))); } + @Override + public String getName() { + return "DB"; + } + @VisibleForTesting public IServiceManager getServiceManager() { return iServiceManager; diff --git a/service/src/main/java/org/apache/celeborn/server/common/service/config/DynamicConfigServiceFactory.java b/service/src/main/java/org/apache/celeborn/server/common/service/config/DynamicConfigServiceFactory.java index 8ee3c38c5..ede3860b9 100644 --- a/service/src/main/java/org/apache/celeborn/server/common/service/config/DynamicConfigServiceFactory.java +++ b/service/src/main/java/org/apache/celeborn/server/common/service/config/DynamicConfigServiceFactory.java @@ -18,12 +18,24 @@ package org.apache.celeborn.server.common.service.config; import java.io.IOException; +import java.util.HashMap; +import java.util.Locale; import org.apache.celeborn.common.CelebornConf; +import org.apache.celeborn.common.util.Utils; public class DynamicConfigServiceFactory { private static volatile ConfigService _INSTANCE; + // short names for dynamic config store backends + private static final HashMap dynamicConfigStoreBackendShortNames = + new HashMap() { + { + put("FS", FsConfigServiceImpl.class.getName()); + put("DB", DbConfigServiceImpl.class.getName()); + } + }; + public static ConfigService getConfigService(CelebornConf celebornConf) throws IOException { if (celebornConf.dynamicConfigStoreBackend().isEmpty()) { return null; @@ -32,12 +44,13 @@ public class DynamicConfigServiceFactory { if (_INSTANCE == null) { synchronized (DynamicConfigServiceFactory.class) { if (_INSTANCE == null) { - String configStoreBackend = celebornConf.dynamicConfigStoreBackend().get(); - if ("FS".equals(configStoreBackend)) { - _INSTANCE = new FsConfigServiceImpl(celebornConf); - } else { - _INSTANCE = new DbConfigServiceImpl(celebornConf); - } + String configStoreBackendName = celebornConf.dynamicConfigStoreBackend().get(); + String configStoreBackendClass = + dynamicConfigStoreBackendShortNames.getOrDefault( + configStoreBackendName.toUpperCase(Locale.ROOT), configStoreBackendName); + + _INSTANCE = + Utils.instantiateDynamicConfigStoreBackend(configStoreBackendClass, celebornConf); } } } diff --git a/service/src/main/java/org/apache/celeborn/server/common/service/config/FsConfigServiceImpl.java b/service/src/main/java/org/apache/celeborn/server/common/service/config/FsConfigServiceImpl.java index 3085d34bf..965417443 100644 --- a/service/src/main/java/org/apache/celeborn/server/common/service/config/FsConfigServiceImpl.java +++ b/service/src/main/java/org/apache/celeborn/server/common/service/config/FsConfigServiceImpl.java @@ -104,4 +104,9 @@ public class FsConfigServiceImpl extends BaseConfigServiceImpl implements Config } return configFile; } + + @Override + public String getName() { + return "FS"; + } }