From 634343e2001fd3f004e245c56cbe9d45f2db1daf Mon Sep 17 00:00:00 2001 From: sychen Date: Thu, 22 May 2025 17:20:58 -0700 Subject: [PATCH] [CELEBORN-2007] Reduce PartitionLocation memory usage ### What changes were proposed in this pull request? ### Why are the changes needed? Driver may have a large number of `PartitionLocation` objects, reducing some unnecessary fields of `PartitionLocation` can reduce the memory pressure of Driver. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? GA Closes #3274 from cxzl25/CELEBORN-2007. Authored-by: sychen Signed-off-by: Wang, Fei --- .../celeborn/common/protocol/PartitionLocation.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/common/src/main/java/org/apache/celeborn/common/protocol/PartitionLocation.java b/common/src/main/java/org/apache/celeborn/common/protocol/PartitionLocation.java index 8e76f3488..46316d8bd 100644 --- a/common/src/main/java/org/apache/celeborn/common/protocol/PartitionLocation.java +++ b/common/src/main/java/org/apache/celeborn/common/protocol/PartitionLocation.java @@ -78,8 +78,6 @@ public class PartitionLocation implements Serializable { this.peer = loc.peer; this.storageInfo = loc.storageInfo; this.mapIdBitMap = loc.mapIdBitMap; - this._hostPushPort = host + ":" + pushPort; - this._hostFetchPort = host + ":" + fetchPort; } public PartitionLocation( @@ -152,8 +150,6 @@ public class PartitionLocation implements Serializable { this.peer = peer; this.storageInfo = hint; this.mapIdBitMap = mapIdBitMap; - this._hostPushPort = host + ":" + pushPort; - this._hostFetchPort = host + ":" + fetchPort; } public int getId() { @@ -210,10 +206,16 @@ public class PartitionLocation implements Serializable { } public String hostAndFetchPort() { + if (_hostFetchPort == null) { + _hostFetchPort = host + ":" + fetchPort; + } return _hostFetchPort; } public String hostAndPushPort() { + if (_hostPushPort == null) { + _hostPushPort = host + ":" + pushPort; + } return _hostPushPort; }