apache/celeborn
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[CELEBORN-815] Remove unused ShuffleClient.readPartition

Browse Source 
### What changes were proposed in this pull request?
As title.

### Why are the changes needed?

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
Passes GA.

Closes #1739 from AngersZhuuuu/CELEBORN-815.

Authored-by: Angerszhuuuu <angers.zhu@gmail.com>
Signed-off-by: zky.zhoukeyong <zky.zhoukeyong@alibaba-inc.com>
This commit is contained in:
Angerszhuuuu 2023-07-20 20:49:29 +08:00 committed by zky.zhoukeyong
parent 5c7848d531
commit be05ae37fe
5 changed files with 3 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
client/src/main/java/org/apache/celeborn/client/ShuffleClient.java
View File

@ -146,9 +146,6 @@ public abstract class ShuffleClient {
int shuffleId, int partitionId, int attemptNumber, int startMapIndex, int endMapIndex)
throws IOException;
public abstract CelebornInputStream readPartition(
int shuffleId, int partitionId, int attemptNumber) throws IOException;
public abstract boolean cleanupShuffle(int shuffleId);
public abstract void shutdown();

6
client/src/main/java/org/apache/celeborn/client/ShuffleClientImpl.java
View File

@ -1508,12 +1508,6 @@ public class ShuffleClientImpl extends ShuffleClient {
return true;
}
@Override
public CelebornInputStream readPartition(int shuffleId, int partitionId, int attemptNumber)
throws IOException {
return readPartition(shuffleId, partitionId, attemptNumber, 0, Integer.MAX_VALUE);
}
protected ReduceFileGroups loadFileGroupInternal(int shuffleId) {
{
long getReducerFileGroupStartTime = System.nanoTime();

5
client/src/test/java/org/apache/celeborn/client/DummyShuffleClient.java
View File

@ -116,11 +116,6 @@ public class DummyShuffleClient extends ShuffleClient {
return null;
}
@Override
public CelebornInputStream readPartition(int shuffleId, int partitionId, int attemptNumber) {
return null;
}
@Override
public boolean cleanupShuffle(int shuffleId) {
return false;

4
client/src/test/scala/org/apache/celeborn/client/WithShuffleClientSuite.scala
View File

@ -141,11 +141,11 @@ trait WithShuffleClientSuite extends CelebornFunSuite {
Assert.assertEquals(shuffleClient.getReduceFileGroupsMap.size(), 0)
// reduce normal empty CelebornInputStream
var stream = shuffleClient.readPartition(shuffleId, 1, 1)
var stream = shuffleClient.readPartition(shuffleId, 1, 1, 0, Integer.MAX_VALUE)
Assert.assertEquals(stream.read(), -1)
// reduce normal null partition for CelebornInputStream
stream = shuffleClient.readPartition(shuffleId, 3, 1)
stream = shuffleClient.readPartition(shuffleId, 3, 1, 0, Integer.MAX_VALUE)
Assert.assertEquals(stream.read(), -1)
}

2
worker/src/test/scala/org/apache/celeborn/service/deploy/cluster/ReadWriteTestBase.scala
View File

@ -93,7 +93,7 @@ trait ReadWriteTestBase extends AnyFunSuite
shuffleClient.mapperEnd(1, 0, 0, 1)
val inputStream = shuffleClient.readPartition(1, 0, 0)
val inputStream = shuffleClient.readPartition(1, 0, 0, 0, Integer.MAX_VALUE)
val outputStream = new ByteArrayOutputStream()
var b = inputStream.read()