1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.procedure2;
20
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.OutputStream;
24 import java.util.concurrent.atomic.AtomicBoolean;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.fs.FileSystem;
29 import org.apache.hadoop.fs.FileStatus;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
32 import org.apache.hadoop.hbase.ProcedureInfo;
33 import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
34 import org.apache.hadoop.hbase.testclassification.SmallTests;
35 import org.apache.hadoop.hbase.util.Bytes;
36 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
37 import org.apache.hadoop.hbase.util.Threads;
38
39 import org.junit.After;
40 import org.junit.Before;
41 import org.junit.Test;
42 import org.junit.experimental.categories.Category;
43
44 import static org.junit.Assert.assertEquals;
45 import static org.junit.Assert.assertFalse;
46 import static org.junit.Assert.assertTrue;
47
48 @Category(SmallTests.class)
49 public class TestProcedureRecovery {
50 private static final Log LOG = LogFactory.getLog(TestProcedureRecovery.class);
51
52 private static final int PROCEDURE_EXECUTOR_SLOTS = 1;
53
54 private static ProcedureExecutor<Void> procExecutor;
55 private static ProcedureStore procStore;
56 private static int procSleepInterval;
57
58 private HBaseCommonTestingUtility htu;
59 private FileSystem fs;
60 private Path testDir;
61 private Path logDir;
62
63 @Before
64 public void setUp() throws IOException {
65 htu = new HBaseCommonTestingUtility();
66 testDir = htu.getDataTestDir();
67 fs = testDir.getFileSystem(htu.getConfiguration());
68 assertTrue(testDir.depth() > 1);
69
70 logDir = new Path(testDir, "proc-logs");
71 procStore = ProcedureTestingUtility.createStore(htu.getConfiguration(), fs, logDir);
72 procExecutor = new ProcedureExecutor(htu.getConfiguration(), null, procStore);
73 procExecutor.testing = new ProcedureExecutor.Testing();
74 procStore.start(PROCEDURE_EXECUTOR_SLOTS);
75 procExecutor.start(PROCEDURE_EXECUTOR_SLOTS);
76 procSleepInterval = 0;
77 }
78
79 @After
80 public void tearDown() throws IOException {
81 procExecutor.stop();
82 procStore.stop(false);
83 fs.delete(logDir, true);
84 }
85
86 private void restart() throws Exception {
87 dumpLogDirState();
88 ProcedureTestingUtility.restart(procExecutor);
89 dumpLogDirState();
90 }
91
92 public static class TestSingleStepProcedure extends SequentialProcedure<Void> {
93 private int step = 0;
94
95 public TestSingleStepProcedure() { }
96
97 @Override
98 protected Procedure[] execute(Void env) {
99 LOG.debug("execute procedure " + this + " step=" + step);
100 step++;
101 setResult(Bytes.toBytes(step));
102 return null;
103 }
104
105 @Override
106 protected void rollback(Void env) { }
107
108 @Override
109 protected boolean abort(Void env) { return true; }
110 }
111
112 public static class BaseTestStepProcedure extends SequentialProcedure<Void> {
113 private AtomicBoolean abort = new AtomicBoolean(false);
114 private int step = 0;
115
116 @Override
117 protected Procedure[] execute(Void env) {
118 LOG.debug("execute procedure " + this + " step=" + step);
119 ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);
120 step++;
121 Threads.sleepWithoutInterrupt(procSleepInterval);
122 if (isAborted()) {
123 setFailure(new RemoteProcedureException(getClass().getName(),
124 new ProcedureAbortedException(
125 "got an abort at " + getClass().getName() + " step=" + step)));
126 return null;
127 }
128 return null;
129 }
130
131 @Override
132 protected void rollback(Void env) {
133 LOG.debug("rollback procedure " + this + " step=" + step);
134 ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);
135 step++;
136 }
137
138 @Override
139 protected boolean abort(Void env) {
140 abort.set(true);
141 return true;
142 }
143
144 private boolean isAborted() {
145 boolean aborted = abort.get();
146 BaseTestStepProcedure proc = this;
147 while (proc.hasParent() && !aborted) {
148 proc = (BaseTestStepProcedure)procExecutor.getProcedure(proc.getParentProcId());
149 aborted = proc.isAborted();
150 }
151 return aborted;
152 }
153 }
154
155 public static class TestMultiStepProcedure extends BaseTestStepProcedure {
156 public TestMultiStepProcedure() { }
157
158 @Override
159 public Procedure[] execute(Void env) {
160 super.execute(env);
161 return isFailed() ? null : new Procedure[] { new Step1Procedure() };
162 }
163
164 public static class Step1Procedure extends BaseTestStepProcedure {
165 public Step1Procedure() { }
166
167 @Override
168 protected Procedure[] execute(Void env) {
169 super.execute(env);
170 return isFailed() ? null : new Procedure[] { new Step2Procedure() };
171 }
172 }
173
174 public static class Step2Procedure extends BaseTestStepProcedure {
175 public Step2Procedure() { }
176 }
177 }
178
179 @Test
180 public void testNoopLoad() throws Exception {
181 restart();
182 }
183
184 @Test(timeout=30000)
185 public void testSingleStepProcRecovery() throws Exception {
186 Procedure proc = new TestSingleStepProcedure();
187 procExecutor.testing.killBeforeStoreUpdate = true;
188 long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
189 assertFalse(procExecutor.isRunning());
190 procExecutor.testing.killBeforeStoreUpdate = false;
191
192
193 long restartTs = EnvironmentEdgeManager.currentTime();
194 restart();
195 waitProcedure(procId);
196 ProcedureInfo result = procExecutor.getResult(procId);
197 assertTrue(result.getLastUpdate() > restartTs);
198 ProcedureTestingUtility.assertProcNotFailed(result);
199 assertEquals(1, Bytes.toInt(result.getResult()));
200 long resultTs = result.getLastUpdate();
201
202
203 restart();
204 result = procExecutor.getResult(procId);
205 ProcedureTestingUtility.assertProcNotFailed(result);
206 assertEquals(resultTs, result.getLastUpdate());
207 assertEquals(1, Bytes.toInt(result.getResult()));
208 }
209
210 @Test(timeout=30000)
211 public void testMultiStepProcRecovery() throws Exception {
212
213 Procedure proc = new TestMultiStepProcedure();
214 long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
215 assertFalse(procExecutor.isRunning());
216
217
218 restart();
219 waitProcedure(procId);
220 ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
221 assertFalse(procExecutor.isRunning());
222
223
224 restart();
225 waitProcedure(procId);
226 ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
227 assertFalse(procExecutor.isRunning());
228
229
230 restart();
231 waitProcedure(procId);
232 assertTrue(procExecutor.isRunning());
233
234
235 ProcedureInfo result = procExecutor.getResult(procId);
236 ProcedureTestingUtility.assertProcNotFailed(result);
237 }
238
239 @Test(timeout=30000)
240 public void testMultiStepRollbackRecovery() throws Exception {
241
242 Procedure proc = new TestMultiStepProcedure();
243 long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
244 assertFalse(procExecutor.isRunning());
245
246
247 restart();
248 waitProcedure(procId);
249 ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
250 assertFalse(procExecutor.isRunning());
251
252
253 restart();
254 waitProcedure(procId);
255 ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
256 assertFalse(procExecutor.isRunning());
257
258
259 procSleepInterval = 2500;
260 restart();
261 assertTrue(procExecutor.abort(procId));
262 waitProcedure(procId);
263 assertFalse(procExecutor.isRunning());
264
265
266 restart();
267 waitProcedure(procId);
268 ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
269 assertFalse(procExecutor.isRunning());
270
271
272 restart();
273 waitProcedure(procId);
274 ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
275 assertFalse(procExecutor.isRunning());
276
277
278 restart();
279 waitProcedure(procId);
280
281
282 ProcedureInfo result = procExecutor.getResult(procId);
283 ProcedureTestingUtility.assertIsAbortException(result);
284 }
285
286 public static class TestStateMachineProcedure
287 extends StateMachineProcedure<Void, TestStateMachineProcedure.State> {
288 enum State { STATE_1, STATE_2, STATE_3, DONE }
289
290 public TestStateMachineProcedure() {}
291
292 private AtomicBoolean aborted = new AtomicBoolean(false);
293 private int iResult = 0;
294
295 @Override
296 protected StateMachineProcedure.Flow executeFromState(Void env, State state) {
297 switch (state) {
298 case STATE_1:
299 LOG.info("execute step 1 " + this);
300 setNextState(State.STATE_2);
301 iResult += 3;
302 break;
303 case STATE_2:
304 LOG.info("execute step 2 " + this);
305 setNextState(State.STATE_3);
306 iResult += 5;
307 break;
308 case STATE_3:
309 LOG.info("execute step 3 " + this);
310 Threads.sleepWithoutInterrupt(procSleepInterval);
311 if (aborted.get()) {
312 LOG.info("aborted step 3 " + this);
313 setAbortFailure("test", "aborted");
314 break;
315 }
316 setNextState(State.DONE);
317 iResult += 7;
318 setResult(Bytes.toBytes(iResult));
319 return Flow.NO_MORE_STATE;
320 default:
321 throw new UnsupportedOperationException();
322 }
323 return Flow.HAS_MORE_STATE;
324 }
325
326 @Override
327 protected void rollbackState(Void env, final State state) {
328 switch (state) {
329 case STATE_1:
330 LOG.info("rollback step 1 " + this);
331 break;
332 case STATE_2:
333 LOG.info("rollback step 2 " + this);
334 break;
335 case STATE_3:
336 LOG.info("rollback step 3 " + this);
337 break;
338 default:
339 throw new UnsupportedOperationException();
340 }
341 }
342
343 @Override
344 protected State getState(final int stateId) {
345 return State.values()[stateId];
346 }
347
348 @Override
349 protected int getStateId(final State state) {
350 return state.ordinal();
351 }
352
353 @Override
354 protected State getInitialState() {
355 return State.STATE_1;
356 }
357
358 @Override
359 protected boolean abort(Void env) {
360 aborted.set(true);
361 return true;
362 }
363
364 @Override
365 protected void serializeStateData(final OutputStream stream) throws IOException {
366 super.serializeStateData(stream);
367 stream.write(Bytes.toBytes(iResult));
368 }
369
370 @Override
371 protected void deserializeStateData(final InputStream stream) throws IOException {
372 super.deserializeStateData(stream);
373 byte[] data = new byte[4];
374 stream.read(data);
375 iResult = Bytes.toInt(data);
376 }
377 }
378
379 @Test(timeout=30000)
380 public void testStateMachineRecovery() throws Exception {
381 ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, true);
382 ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, true);
383
384
385 Procedure proc = new TestStateMachineProcedure();
386 long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
387 assertFalse(procExecutor.isRunning());
388
389
390 restart();
391 waitProcedure(procId);
392 ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
393 assertFalse(procExecutor.isRunning());
394
395
396 restart();
397 waitProcedure(procId);
398 ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
399 assertFalse(procExecutor.isRunning());
400
401
402 restart();
403 waitProcedure(procId);
404 assertTrue(procExecutor.isRunning());
405
406
407 ProcedureInfo result = procExecutor.getResult(procId);
408 ProcedureTestingUtility.assertProcNotFailed(result);
409 assertEquals(15, Bytes.toInt(result.getResult()));
410 }
411
412 @Test(timeout=30000)
413 public void testStateMachineRollbackRecovery() throws Exception {
414 ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, true);
415 ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, true);
416
417
418 Procedure proc = new TestStateMachineProcedure();
419 long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
420 ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
421 assertFalse(procExecutor.isRunning());
422
423
424 restart();
425 waitProcedure(procId);
426 ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
427 assertFalse(procExecutor.isRunning());
428
429
430 restart();
431 waitProcedure(procId);
432 ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
433 assertFalse(procExecutor.isRunning());
434
435
436 procSleepInterval = 2500;
437 restart();
438 assertTrue(procExecutor.abort(procId));
439 waitProcedure(procId);
440 ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
441 assertFalse(procExecutor.isRunning());
442
443
444 restart();
445 waitProcedure(procId);
446 assertFalse(procExecutor.isRunning());
447 ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
448
449
450 restart();
451 waitProcedure(procId);
452 assertFalse(procExecutor.isRunning());
453 ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
454
455
456 restart();
457 waitProcedure(procId);
458 assertTrue(procExecutor.isRunning());
459
460
461 ProcedureInfo result = procExecutor.getResult(procId);
462 ProcedureTestingUtility.assertIsAbortException(result);
463 }
464
465 private void waitProcedure(final long procId) {
466 ProcedureTestingUtility.waitProcedure(procExecutor, procId);
467 dumpLogDirState();
468 }
469
470 private void dumpLogDirState() {
471 try {
472 FileStatus[] files = fs.listStatus(logDir);
473 if (files != null && files.length > 0) {
474 for (FileStatus file: files) {
475 assertTrue(file.toString(), file.isFile());
476 LOG.debug("log file " + file.getPath() + " size=" + file.getLen());
477 }
478 } else {
479 LOG.debug("no files under: " + logDir);
480 }
481 } catch (IOException e) {
482 LOG.warn("Unable to dump " + logDir, e);
483 }
484 }
485 }