apache/celeborn
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[CELEBORN-1693] Fix storageFetcherPool concurrent problem

Browse Source 
### What changes were proposed in this pull request?

Fix storageFetcherPool concurrent problem.

There may be duplicate thread pools created as multi-thread race condition.

![image](https://github.com/user-attachments/assets/ba4b0964-700e-4502-933a-b6c7cb93f32d)

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
No need.

Closes #2886 from reswqa/storageFetcherPool.

Authored-by: Weijie Guo <reswqa@163.com>
Signed-off-by: SteNicholas <programgeek@163.com>
This commit is contained in:
Weijie Guo 2024-11-07 13:51:39 +08:00 committed by SteNicholas
parent f1bda46de4
commit 4e7df13af7
No known key found for this signature in database
GPG Key ID: 1FC79E01BA3B84D5
4 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
worker/src/main/java/org/apache/celeborn/service/deploy/worker/storage/CreditStreamManager.java
View File

@ -44,7 +44,8 @@ public class CreditStreamManager {
private final AtomicLong nextStreamId;
private final ConcurrentHashMap<Long, StreamState> streams;
private final ConcurrentHashMap<FileInfo, MapPartitionData> activeMapPartitions;
private final HashMap<String, ExecutorService> storageFetcherPool = new HashMap<>();
private final ConcurrentHashMap<String, ExecutorService> storageFetcherPool =
JavaUtils.newConcurrentHashMap();
private int minReadBuffers;
private int maxReadBuffers;
private int threadsPerMountPoint;

3
worker/src/main/java/org/apache/celeborn/service/deploy/worker/storage/MapPartitionData.java
View File

@ -19,7 +19,6 @@ package org.apache.celeborn.service.deploy.worker.storage;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.util.HashMap;
import java.util.List;
import java.util.PriorityQueue;
import java.util.concurrent.ConcurrentHashMap;
@ -68,7 +67,7 @@ public class MapPartitionData implements MemoryManager.ReadBufferTargetChangeLis
public MapPartitionData(
int minReadBuffers,
int maxReadBuffers,
HashMap<String, ExecutorService> storageFetcherPool,
ConcurrentHashMap<String, ExecutorService> storageFetcherPool,
int threadsPerMountPoint,
DiskFileInfo diskFileInfo,
Consumer<Long> recycleStream,

4
worker/src/main/java/org/apache/celeborn/service/deploy/worker/storage/segment/SegmentMapPartitionData.java
View File

@ -19,7 +19,7 @@
package org.apache.celeborn.service.deploy.worker.storage.segment;
import java.io.IOException;
import java.util.HashMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.function.Consumer;
@ -38,7 +38,7 @@ public class SegmentMapPartitionData extends MapPartitionData {
public SegmentMapPartitionData(
int minReadBuffers,
int maxReadBuffers,
HashMap<String, ExecutorService> storageFetcherPool,
ConcurrentHashMap<String, ExecutorService> storageFetcherPool,
int threadsPerMountPoint,
DiskFileInfo fileInfo,
Consumer<Long> recycleStream,

1
worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/CreditStreamManagerSuiteJ.java
View File

@ -79,6 +79,7 @@ public class CreditStreamManagerSuiteJ {
new DiskFileInfo(
createTemporaryFileWithIndexFile(), new UserIdentifier("default", "default"), conf);
MapFileMeta mapFileMeta = new MapFileMeta(1024, 10);
mapFileMeta.setMountPoint("/tmp");
diskFileInfo.replaceFileMeta(mapFileMeta);
Consumer<Long> streamIdConsumer = streamId -> Assert.assertTrue(streamId > 0);