1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
|
package core;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.file.Files;
import java.nio.file.Path;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import events.implementations.VSProcessCrashEvent;
import events.implementations.VSProcessRecoverEvent;
import simulator.VSSimulator;
import simulator.VSSimulatorVisualization;
import simulator.builder.SimulationBuilder;
import testing.HeadlessLoader;
class VSTaskManagerCrashRecoveryIntegrationTest {
private static final long ADVANCE_STEP_MS = 1L;
private static final String RAFT_REPLAY = "saved-simulations/raft.dat";
private VSSimulator simulatorToStop;
private VSSimulator loadedSimulatorToStop;
@AfterEach
void tearDown() {
stopSimulatorThread(simulatorToStop);
stopSimulatorThread(loadedSimulatorToStop);
}
@Test
@DisplayName("Runtime supports recover then later crash of a different process")
void runtimeSupportsRecoverThenCrashAnotherProcess() throws Exception {
VSSimulator simulator = new SimulationBuilder()
.withProcesses(3)
.withDuration(1000)
.getSimulator();
simulatorToStop = simulator;
VSSimulatorVisualization visualization = simulator.getSimulatorCanvas();
VSInternalProcess process0 = visualization.getProcess(0);
VSInternalProcess process1 = visualization.getProcess(1);
VSTaskManager taskManager = visualization.getTaskManager();
taskManager.addTask(new VSTask(5, process0, new VSProcessCrashEvent(), VSTask.GLOBAL));
taskManager.addTask(new VSTask(10, process0, new VSProcessRecoverEvent(), VSTask.GLOBAL));
taskManager.addTask(new VSTask(15, process1, new VSProcessCrashEvent(), VSTask.GLOBAL));
runUntil(visualization, 20);
assertFalse(process0.isCrashed(), "process 0 should have recovered");
assertTrue(process1.isCrashed(), "process 1 should crash after process 0 recovers");
}
@Test
@DisplayName("Saved replay preserves recover then later crash of another process")
void savedReplayPreservesRecoverThenCrashAnotherProcess() throws Exception {
Path datFile = Files.createTempFile("crash-recover-replay", ".dat");
VSSimulator builtSimulator = new SimulationBuilder()
.withProcesses(3)
.withDuration(1000)
.addCrashEvent(0, 5)
.addRecoveryEvent(0, 10)
.addCrashEvent(1, 15)
.save(datFile.toString())
.getSimulator();
simulatorToStop = builtSimulator;
HeadlessLoader.LoadedSimulation loaded =
HeadlessLoader.load(datFile.toString(), builtSimulator.getPrefs());
loadedSimulatorToStop = loaded.getSimulator();
VSSimulatorVisualization visualization = loaded.getVisualization();
assertFalse(visualization.getProcess(0).isCrashed(),
"process 0 should start alive after replay load");
assertFalse(visualization.getProcess(1).isCrashed(),
"process 1 should start alive after replay load");
runUntil(visualization, 4);
assertFalse(visualization.getProcess(0).isCrashed(),
"process 0 should stay alive before its crash point");
assertFalse(visualization.getProcess(1).isCrashed(),
"process 1 should stay alive before process 0 crashes");
runUntil(visualization, 6);
assertTrue(visualization.getProcess(0).isCrashed(),
"process 0 should crash immediately after its scheduled crash point");
assertFalse(visualization.getProcess(1).isCrashed(),
"process 1 should still be alive while process 0 is crashed");
runUntil(visualization, 10);
assertTrue(visualization.getProcess(0).isCrashed(),
"process 0 should remain crashed before its recover point");
assertFalse(visualization.getProcess(1).isCrashed(),
"process 1 should still be alive before its later crash");
runUntil(visualization, 11);
assertFalse(visualization.getProcess(0).isCrashed(),
"process 0 should recover immediately after its scheduled recover point");
assertFalse(visualization.getProcess(1).isCrashed(),
"process 1 should still be alive before its crash point");
runUntil(visualization, 15);
assertFalse(visualization.getProcess(0).isCrashed(),
"process 0 should stay recovered before process 1 crashes");
assertFalse(visualization.getProcess(1).isCrashed(),
"process 1 should stay alive before its later crash point");
runUntil(visualization, 16);
assertFalse(visualization.getProcess(0).isCrashed(),
"process 0 should remain recovered after replay load");
assertTrue(visualization.getProcess(1).isCrashed(),
"process 1 should crash immediately after its later replay point");
}
@Test
@DisplayName("Loaded raft replay keeps crash and recover events visible in Event view collections")
void loadedRaftReplayKeepsCrashAndRecoverEventsVisible() throws Exception {
HeadlessLoader.LoadedSimulation loaded =
HeadlessLoader.load(RAFT_REPLAY, prefs.VSDefaultPrefs.init());
loadedSimulatorToStop = loaded.getSimulator();
VSSimulatorVisualization visualization = loaded.getVisualization();
VSTaskManager taskManager = visualization.getTaskManager();
VSInternalProcess process0 = visualization.getProcess(0);
VSInternalProcess process2 = visualization.getProcess(2);
assertEquals(6, taskManager.getGlobalTasks().size(),
"loaded raft replay should expose all saved global tasks");
assertEquals(3, taskManager.getProcessGlobalTasks(process0).size(),
"process 0 should expose its activation, crash, and recover events");
assertEquals(2, taskManager.getProcessGlobalTasks(process2).size(),
"process 2 should expose its activation and later crash event");
runUntil(visualization, 12001);
assertEquals(3, taskManager.getProcessGlobalTasks(process0).size(),
"process 0 recover event should remain visible after it executes");
runUntil(visualization, 20001);
assertEquals(2, taskManager.getProcessGlobalTasks(process2).size(),
"process 2 later crash event should remain visible after it executes");
}
@Test
@DisplayName("Live GUI-style event injection supports recover and later crash")
void liveEventInjectionSupportsRecoverAndLaterCrash() throws Exception {
VSSimulator simulator = new SimulationBuilder()
.withProcesses(3)
.withDuration(1000)
.getSimulator();
simulatorToStop = simulator;
VSSimulatorVisualization visualization = simulator.getSimulatorCanvas();
VSInternalProcess process0 = visualization.getProcess(0);
VSInternalProcess process1 = visualization.getProcess(1);
VSTaskManager taskManager = visualization.getTaskManager();
runUntil(visualization, 5);
taskManager.addTask(new VSTask(process0.getGlobalTime(), process0,
new VSProcessCrashEvent(), VSTask.GLOBAL));
runUntil(visualization, 6);
assertTrue(process0.isCrashed(), "process 0 should crash from live event");
taskManager.addTask(new VSTask(process0.getGlobalTime(), process0,
new VSProcessRecoverEvent(), VSTask.GLOBAL));
runUntil(visualization, 7);
assertFalse(process0.isCrashed(), "process 0 should recover from live event");
taskManager.addTask(new VSTask(process1.getGlobalTime(), process1,
new VSProcessCrashEvent(), VSTask.GLOBAL));
runUntil(visualization, 8);
assertTrue(process1.isCrashed(), "process 1 should crash after process 0 recovers");
}
private void runUntil(VSSimulatorVisualization visualization, long targetTime)
throws Exception {
setBooleanField(visualization, "isPaused", false);
setBooleanField(visualization, "hasFinished", false);
setDoubleField(visualization, "clockSpeed", 1.0d);
Method updateSimulator = VSSimulatorVisualization.class.getDeclaredMethod(
"updateSimulator", long.class, long.class);
updateSimulator.setAccessible(true);
long wallTime = visualization.getTime();
while (visualization.getTime() < targetTime) {
long nextWallTime = wallTime + ADVANCE_STEP_MS;
updateSimulator.invoke(visualization, nextWallTime, wallTime);
wallTime = nextWallTime;
}
}
private void setBooleanField(Object target, String fieldName, boolean value)
throws Exception {
Field field = target.getClass().getDeclaredField(fieldName);
field.setAccessible(true);
field.setBoolean(target, value);
}
private void setDoubleField(Object target, String fieldName, double value)
throws Exception {
Field field = target.getClass().getDeclaredField(fieldName);
field.setAccessible(true);
field.setDouble(target, value);
}
private void stopSimulatorThread(VSSimulator simulator) {
if (simulator != null) {
simulator.getSimulatorCanvas().stopThread();
}
}
}
|
