Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1302,7 +1302,7 @@ public TezConfiguration(boolean loadDefaults) {
TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS =
TEZ_AM_PREFIX + "container.reuse.locality.delay-allocation-millis";
public static final long
TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS_DEFAULT = 250l;
TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS_DEFAULT = 250L;

/**
* Int value. The minimum amount of time to hold on to a container that is idle. Only active when
Expand All @@ -1313,7 +1313,7 @@ public TezConfiguration(boolean loadDefaults) {
public static final String TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS =
TEZ_AM_PREFIX + "container.idle.release-timeout-min.millis";
public static final long
TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS_DEFAULT = 5000l;
TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS_DEFAULT = 5000L;

/**
* Int value. The maximum amount of time to hold on to a container if no task can be
Expand All @@ -1330,7 +1330,7 @@ public TezConfiguration(boolean loadDefaults) {
public static final String TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS =
TEZ_AM_PREFIX + "container.idle.release-timeout-max.millis";
public static final long
TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS_DEFAULT = 10000l;
TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS_DEFAULT = 10000L;

/**
* Int value. The minimum number of containers that will be held in session mode. Not active in
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,7 @@ public void testGetDAGSessionTimeout() {
// set to 1 month - * 1000 guaranteed to cross positive integer boundary
conf.setInt(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS,
24 * 60 * 60 * 30);
assertEquals(86400l*1000*30,
assertEquals(86400L*1000*30,
TezCommonUtils.getDAGSessionTimeout(conf));

// set to negative val
Expand Down
4 changes: 2 additions & 2 deletions tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java
Original file line number Diff line number Diff line change
Expand Up @@ -376,8 +376,8 @@ public void testRecreateDAG() {
TezConfiguration tezConf = new TezConfiguration();
DAGPlan firstPlan = dag.createDag(tezConf, null, null, null, false);
for (int i = 0; i < 3; ++i) {
DAGPlan dagPlan = dag.createDag(tezConf, null, null, null, false);
assertEquals(dagPlan, firstPlan);
DAGPlan dagPlan = dag.createDag(tezConf, null, null, null, false);
assertEquals(dagPlan, firstPlan);
}
}

Expand Down
26 changes: 13 additions & 13 deletions tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,9 @@
// based on TestDAGLocationHint
public class TestDAGPlan {

@SuppressWarnings("checkstyle:VisibilityModifier")
@TempDir
public Path tempFolder;
Path tempFolder;

@Test
@Timeout(value = 5000, unit = TimeUnit.MILLISECONDS)
Expand All @@ -88,17 +89,16 @@ public void testBasicJobPlanSerde() throws IOException {
.build())
.build())
.build();
File file = tempFolder.resolve("jobPlan").toFile();
FileOutputStream outStream = null;
try {
outStream = new FileOutputStream(file);
job.writeTo(outStream);
}
finally {
if(outStream != null){
outStream.close();
}
}
File file = tempFolder.resolve("jobPlan").toFile();
FileOutputStream outStream = null;
try {
outStream = new FileOutputStream(file);
job.writeTo(outStream);
} finally {
if (outStream != null) {
outStream.close();
}
}

DAGPlan inJob;
FileInputStream inputStream;
Expand All @@ -110,7 +110,7 @@ public void testBasicJobPlanSerde() throws IOException {
outStream.close();
}

assertEquals(job, inJob);
assertEquals(job, inJob);
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -359,7 +359,7 @@ public void testGetDagStatusWithTimeout() throws Exception {
long diff;

TezConfiguration tezConf = new TezConfiguration();
tezConf.setLong(TezConfiguration.TEZ_DAG_STATUS_POLLINTERVAL_MS, 800l);
tezConf.setLong(TezConfiguration.TEZ_DAG_STATUS_POLLINTERVAL_MS, 800L);

DAGClientImplForTest dagClient = new DAGClientImplForTest(mockAppId, dagIdStr, tezConf, null);
DAGClientRPCImplForTest dagClientRpc =
Expand All @@ -376,10 +376,10 @@ public void testGetDagStatusWithTimeout() throws Exception {
dagClient.setRmDagStatus(rmDagStatus);

startTime = System.currentTimeMillis();
dagStatus = dagClient.getDAGStatus(EnumSet.noneOf(StatusGetOpts.class), 2000l);
dagStatus = dagClient.getDAGStatus(EnumSet.noneOf(StatusGetOpts.class), 2000L);
endTime = System.currentTimeMillis();
diff = endTime - startTime;
assertTrue(diff > 1500l && diff < 2500l);
assertTrue(diff > 1500L && diff < 2500L);
// One at start. Second and Third within the sleep. Fourth at final refresh.
assertEquals(0, dagClientRpc.numGetStatusViaAmInvocations); // No AM available, so no invocations to AM
assertEquals(4, dagClient.numGetStatusViaRmInvocations);
Expand All @@ -394,10 +394,10 @@ public void testGetDagStatusWithTimeout() throws Exception {
dagClientRpc.setAMProxy(createMockProxy(DAGStatusStateProto.DAG_RUNNING, -1));

startTime = System.currentTimeMillis();
dagStatus = dagClient.getDAGStatus(EnumSet.noneOf(StatusGetOpts.class), 2000l);
dagStatus = dagClient.getDAGStatus(EnumSet.noneOf(StatusGetOpts.class), 2000L);
endTime = System.currentTimeMillis();
diff = endTime - startTime;
assertTrue(diff > 1500l && diff < 2500l);
assertTrue(diff > 1500L && diff < 2500L);
// Directly from AM
assertEquals(0, dagClient.numGetStatusViaRmInvocations);
// Directly from AM - one refresh. One with timeout.
Expand All @@ -411,13 +411,13 @@ public void testGetDagStatusWithTimeout() throws Exception {
rmDagStatus =
new DAGStatus(constructDagStatusProto(DAGStatusStateProto.DAG_RUNNING), DagStatusSource.RM);
dagClient.setRmDagStatus(rmDagStatus);
dagClientRpc.setAMProxy(createMockProxy(DAGStatusStateProto.DAG_SUCCEEDED, 1000l));
dagClientRpc.setAMProxy(createMockProxy(DAGStatusStateProto.DAG_SUCCEEDED, 1000L));

startTime = System.currentTimeMillis();
dagStatus = dagClient.getDAGStatus(EnumSet.noneOf(StatusGetOpts.class), 2000l);
dagStatus = dagClient.getDAGStatus(EnumSet.noneOf(StatusGetOpts.class), 2000L);
endTime = System.currentTimeMillis();
diff = endTime - startTime;
assertTrue(diff > 500l && diff < 1500l);
assertTrue(diff > 500L && diff < 1500L);
// Directly from AM
assertEquals(0, dagClient.numGetStatusViaRmInvocations);
// Directly from AM - previous request cached, so single invocation only.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -958,7 +958,7 @@ public DAGStatusBuilder getDAGStatus(Set<StatusGetOpts> statusOptions) {

public DAGStatusBuilder getDAGStatus(Set<StatusGetOpts> statusOptions,
long timeoutMillis) throws TezException {
long timeoutNanos = timeoutMillis * 1000l * 1000l;
long timeoutNanos = timeoutMillis * 1000L * 1000L;
if (timeoutMillis < 0) {
// Return only on SUCCESS
timeoutNanos = Long.MAX_VALUE;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ public void shutdown() throws Exception {
}
if (eventHandlingThread != null) {
eventHandlingThread.interrupt();
eventHandlingThread.join(2000l);
eventHandlingThread.join(2000L);
}
if (taskExecutorService != null) {
taskExecutorService.shutdownNow();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -744,7 +744,7 @@ public void serviceStop() throws InterruptedException {
}
LOG.info("Shutting down AppCallbackExecutor");
appCallbackExecutor.shutdownNow();
appCallbackExecutor.awaitTermination(1000l, TimeUnit.MILLISECONDS);
appCallbackExecutor.awaitTermination(1000L, TimeUnit.MILLISECONDS);
}

// TaskSchedulerAppCallback methods with schedulerId, where relevant
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -392,7 +392,7 @@ public void shutdown() throws InterruptedException {
try {
delayedContainerManager.shutdown();
// Wait for contianers to be released.
delayedContainerManager.join(2000l);
delayedContainerManager.join(2000L);
synchronized (this) {
if (shouldUnregister.get()) {
AppFinalStatus status = getContext().getFinalAppStatus();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,9 @@
import org.mockito.MockitoAnnotations;

public class TestDAGClientAMProtocolBlockingPBServerImpl {
@SuppressWarnings("checkstyle:VisibilityModifier")
@TempDir
public java.nio.file.Path tmpFolder;
java.nio.file.Path tmpFolder;

@Captor
private ArgumentCaptor<Map<String, LocalResource>> localResourcesCaptor;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ public void start() throws Exception {
public void shutdown() throws Exception {
if (eventHandlingThread != null) {
eventHandlingThread.interrupt();
eventHandlingThread.join(2000l);
eventHandlingThread.join(2000L);
}
if (executorService != null) {
executorService.shutdownNow();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1811,7 +1811,7 @@ public void testGetDAGStatusWithWait() throws TezException {
assertEquals(5, dag.getSuccessfulVertices());

long dagStatusStartTime = System.currentTimeMillis();
DAGStatusBuilder dagStatus = dag.getDAGStatus(EnumSet.noneOf(StatusGetOpts.class), 2000l);
DAGStatusBuilder dagStatus = dag.getDAGStatus(EnumSet.noneOf(StatusGetOpts.class), 2000L);
long dagStatusEndTime = System.currentTimeMillis();
long diff = dagStatusEndTime - dagStatusStartTime;
assertTrue(diff >= 0 && diff < 2500);
Expand Down Expand Up @@ -1882,7 +1882,7 @@ public void runTestGetDAGStatusReturnOnDagFinished(DAGStatusBuilder.State testSt
}

// Sleep for 2 seconds. Then mark the last vertex is successful.
Thread.sleep(2000l);
Thread.sleep(2000L);
if (testState == State.SUCCEEDED) {
dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
TezVertexID.getInstance(dagId, 5), VertexState.SUCCEEDED));
Expand Down Expand Up @@ -2370,7 +2370,7 @@ public void run() {
}
try {
dagStatusStartTime = System.currentTimeMillis();
dagStatus = dag.getDAGStatus(EnumSet.noneOf(StatusGetOpts.class), 10000l);
dagStatus = dag.getDAGStatus(EnumSet.noneOf(StatusGetOpts.class), 10000L);
dagStatusEndTime = System.currentTimeMillis();
} catch (TezException e) {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1053,7 +1053,7 @@ public void testProgressTimeStampUpdate() throws Exception {

TaskHeartbeatHandler mockHeartbeatHandler = mock(TaskHeartbeatHandler.class);
Clock mockClock = mock(Clock.class);
when(mockClock.getTime()).thenReturn(50l);
when(mockClock.getTime()).thenReturn(50L);
TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
taListener, taskConf, mockClock,
mockHeartbeatHandler, appCtx, false,
Expand All @@ -1067,7 +1067,7 @@ public void testProgressTimeStampUpdate() throws Exception {
assertEquals(taImpl.getState(), TaskAttemptState.RUNNING, "Task attempt is not in the RUNNING state");
verify(mockHeartbeatHandler).register(taskAttemptID);

when(mockClock.getTime()).thenReturn(100l);
when(mockClock.getTime()).thenReturn(100L);
taImpl.handle(new TaskAttemptEventStatusUpdate(
taskAttemptID, new TaskStatusUpdateEvent(null, 0.1f, null, false)));
verify(eventHandler, atLeast(1)).handle(arg.capture());
Expand All @@ -1080,7 +1080,7 @@ public void testProgressTimeStampUpdate() throws Exception {
} else {
assertEquals(taImpl.getInternalState(), TaskAttemptStateInternal.RUNNING, "Task Attempt's internal state should be RUNNING!");
}
when(mockClock.getTime()).thenReturn(200l);
when(mockClock.getTime()).thenReturn(200L);
taImpl.handle(new TaskAttemptEventStatusUpdate(
taskAttemptID, new TaskStatusUpdateEvent(null, 0.1f, null, false)));
verify(eventHandler, atLeast(1)).handle(arg.capture());
Expand Down Expand Up @@ -1265,26 +1265,26 @@ public void testNoProgressFail() throws Exception {
assertEquals(TaskAttemptState.RUNNING, taImpl.getState(), "Task attempt is not in the RUNNING state");
verify(mockHeartbeatHandler).register(taskAttemptID);

when(mockClock.getTime()).thenReturn(100l);
when(mockClock.getTime()).thenReturn(100L);
taImpl.handle(new TaskAttemptEventStatusUpdate(
taskAttemptID, new TaskStatusUpdateEvent(null, 0.1f, null, true)));
// invocations and time updated
assertEquals(100l, taImpl.lastNotifyProgressTimestamp);
when(mockClock.getTime()).thenReturn(150l);
assertEquals(100L, taImpl.lastNotifyProgressTimestamp);
when(mockClock.getTime()).thenReturn(150L);
taImpl.handle(new TaskAttemptEventStatusUpdate(
taskAttemptID, new TaskStatusUpdateEvent(null, 0.1f, null, true)));
// invocations and time updated
assertEquals(150l, taImpl.lastNotifyProgressTimestamp);
when(mockClock.getTime()).thenReturn(200l);
assertEquals(150L, taImpl.lastNotifyProgressTimestamp);
when(mockClock.getTime()).thenReturn(200L);
taImpl.handle(new TaskAttemptEventStatusUpdate(
taskAttemptID, new TaskStatusUpdateEvent(null, 0.1f, null, false)));
// invocations and time not updated
assertEquals(150l, taImpl.lastNotifyProgressTimestamp);
when(mockClock.getTime()).thenReturn(250l);
assertEquals(150L, taImpl.lastNotifyProgressTimestamp);
when(mockClock.getTime()).thenReturn(250L);
taImpl.handle(new TaskAttemptEventStatusUpdate(
taskAttemptID, new TaskStatusUpdateEvent(null, 0.1f, null, false)));
// invocations and time not updated
assertEquals(150l, taImpl.lastNotifyProgressTimestamp);
assertEquals(150L, taImpl.lastNotifyProgressTimestamp);
// failed event sent to self
verify(eventHandler, atLeast(1)).handle(arg.capture());
TaskAttemptEventAttemptFailed fEvent = (TaskAttemptEventAttemptFailed) arg.getValue();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public void testBasicStats() {

TezVertexID tezVertexID = TezVertexID.getInstance(
TezDAGID.getInstance(
ApplicationId.newInstance(100l, 1), 1), 1);
ApplicationId.newInstance(100L, 1), 1), 1);
TezTaskID tezTaskID1 = TezTaskID.getInstance(tezVertexID, 1);
TezTaskID tezTaskID2 = TezTaskID.getInstance(tezVertexID, 2);
TezTaskID tezTaskID3 = TezTaskID.getInstance(tezVertexID, 3);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ public void testDelayedReuseContainerBecomesAvailable()
conf.setBoolean(
TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, false);
conf.setLong(
TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 3000l);
TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 3000L);
conf.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, 0);
conf.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, 0);

Expand Down Expand Up @@ -237,7 +237,7 @@ public void testDelayedReuseContainerBecomesAvailable()

long currentTs = System.currentTimeMillis();
Throwable exception = null;
while (System.currentTimeMillis() < currentTs + 5000l) {
while (System.currentTimeMillis() < currentTs + 5000L) {
try {
verify(taskSchedulerManager,
times(1)).containerBeingReleased(eq(0), eq(containerHost2.getId()));
Expand All @@ -261,7 +261,7 @@ public void testDelayedReuseContainerNotAvailable()
conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true);
conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, false);
conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, false);
conf.setLong(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 1000l);
conf.setLong(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 1000L);
conf.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, 0);
conf.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, 0);

Expand Down Expand Up @@ -710,9 +710,9 @@ public void testReuseNonLocalRequest()
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true);
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, true);
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, true);
tezConf.setLong(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 100l);
tezConf.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, 1000l);
tezConf.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, 1000l);
tezConf.setLong(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 100L);
tezConf.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, 1000L);
tezConf.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, 1000L);

CapturingEventHandler eventHandler = new CapturingEventHandler();
DAG dag = mock(DAG.class);
Expand Down Expand Up @@ -833,11 +833,11 @@ public void testReuseAcrossVertices()
Configuration tezConf = new Configuration();
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true);
tezConf.setLong(
TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 1l);
TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 1L);
tezConf.setLong(
TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, 20l);
TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, 20L);
tezConf.setLong(
TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, 30l);
TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, 30L);
tezConf.setInt(
TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, 1);

Expand Down Expand Up @@ -941,7 +941,7 @@ public void testReuseAcrossVertices()

LOG.info("Sleeping to ensure that the container has been idled longer " +
"than TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS ");
Thread.sleep(50l);
Thread.sleep(50L);
// container should not get released due to min held containers
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));

Expand Down
Loading