View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.procedure2;
20  
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.io.OutputStream;
24  import java.util.concurrent.atomic.AtomicBoolean;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.FileStatus;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
32  import org.apache.hadoop.hbase.ProcedureInfo;
33  import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
34  import org.apache.hadoop.hbase.testclassification.SmallTests;
35  import org.apache.hadoop.hbase.util.Bytes;
36  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
37  import org.apache.hadoop.hbase.util.Threads;
38  
39  import org.junit.After;
40  import org.junit.Before;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  
44  import static org.junit.Assert.assertEquals;
45  import static org.junit.Assert.assertFalse;
46  import static org.junit.Assert.assertTrue;
47  
48  @Category(SmallTests.class)
49  public class TestProcedureRecovery {
50    private static final Log LOG = LogFactory.getLog(TestProcedureRecovery.class);
51  
52    private static final int PROCEDURE_EXECUTOR_SLOTS = 1;
53  
54    private static ProcedureExecutor<Void> procExecutor;
55    private static ProcedureStore procStore;
56    private static int procSleepInterval;
57  
58    private HBaseCommonTestingUtility htu;
59    private FileSystem fs;
60    private Path testDir;
61    private Path logDir;
62  
63    @Before
64    public void setUp() throws IOException {
65      htu = new HBaseCommonTestingUtility();
66      testDir = htu.getDataTestDir();
67      fs = testDir.getFileSystem(htu.getConfiguration());
68      assertTrue(testDir.depth() > 1);
69  
70      logDir = new Path(testDir, "proc-logs");
71      procStore = ProcedureTestingUtility.createStore(htu.getConfiguration(), fs, logDir);
72      procExecutor = new ProcedureExecutor(htu.getConfiguration(), null, procStore);
73      procExecutor.testing = new ProcedureExecutor.Testing();
74      procStore.start(PROCEDURE_EXECUTOR_SLOTS);
75      procExecutor.start(PROCEDURE_EXECUTOR_SLOTS);
76      procSleepInterval = 0;
77    }
78  
79    @After
80    public void tearDown() throws IOException {
81      procExecutor.stop();
82      procStore.stop(false);
83      fs.delete(logDir, true);
84    }
85  
86    private void restart() throws Exception {
87      dumpLogDirState();
88      ProcedureTestingUtility.restart(procExecutor);
89      dumpLogDirState();
90    }
91  
92    public static class TestSingleStepProcedure extends SequentialProcedure<Void> {
93      private int step = 0;
94  
95      public TestSingleStepProcedure() { }
96  
97      @Override
98      protected Procedure[] execute(Void env) {
99        LOG.debug("execute procedure " + this + " step=" + step);
100       step++;
101       setResult(Bytes.toBytes(step));
102       return null;
103     }
104 
105     @Override
106     protected void rollback(Void env) { }
107 
108     @Override
109     protected boolean abort(Void env) { return true; }
110   }
111 
112   public static class BaseTestStepProcedure extends SequentialProcedure<Void> {
113     private AtomicBoolean abort = new AtomicBoolean(false);
114     private int step = 0;
115 
116     @Override
117     protected Procedure[] execute(Void env) {
118       LOG.debug("execute procedure " + this + " step=" + step);
119       ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);
120       step++;
121       Threads.sleepWithoutInterrupt(procSleepInterval);
122       if (isAborted()) {
123         setFailure(new RemoteProcedureException(getClass().getName(),
124           new ProcedureAbortedException(
125             "got an abort at " + getClass().getName() + " step=" + step)));
126         return null;
127       }
128       return null;
129     }
130 
131     @Override
132     protected void rollback(Void env) {
133       LOG.debug("rollback procedure " + this + " step=" + step);
134       ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);
135       step++;
136     }
137 
138     @Override
139     protected boolean abort(Void env) {
140       abort.set(true);
141       return true;
142     }
143 
144     private boolean isAborted() {
145       boolean aborted = abort.get();
146       BaseTestStepProcedure proc = this;
147       while (proc.hasParent() && !aborted) {
148         proc = (BaseTestStepProcedure)procExecutor.getProcedure(proc.getParentProcId());
149         aborted = proc.isAborted();
150       }
151       return aborted;
152     }
153   }
154 
155   public static class TestMultiStepProcedure extends BaseTestStepProcedure {
156     public TestMultiStepProcedure() { }
157 
158     @Override
159     public Procedure[] execute(Void env) {
160       super.execute(env);
161       return isFailed() ? null : new Procedure[] { new Step1Procedure() };
162     }
163 
164     public static class Step1Procedure extends BaseTestStepProcedure {
165       public Step1Procedure() { }
166 
167       @Override
168       protected Procedure[] execute(Void env) {
169         super.execute(env);
170         return isFailed() ? null : new Procedure[] { new Step2Procedure() };
171       }
172     }
173 
174     public static class Step2Procedure extends BaseTestStepProcedure {
175       public Step2Procedure() { }
176     }
177   }
178 
179   @Test
180   public void testNoopLoad() throws Exception {
181     restart();
182   }
183 
184   @Test(timeout=30000)
185   public void testSingleStepProcRecovery() throws Exception {
186     Procedure proc = new TestSingleStepProcedure();
187     procExecutor.testing.killBeforeStoreUpdate = true;
188     long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
189     assertFalse(procExecutor.isRunning());
190     procExecutor.testing.killBeforeStoreUpdate = false;
191 
192     // Restart and verify that the procedures restart
193     long restartTs = EnvironmentEdgeManager.currentTime();
194     restart();
195     waitProcedure(procId);
196     ProcedureInfo result = procExecutor.getResult(procId);
197     assertTrue(result.getLastUpdate() > restartTs);
198     ProcedureTestingUtility.assertProcNotFailed(result);
199     assertEquals(1, Bytes.toInt(result.getResult()));
200     long resultTs = result.getLastUpdate();
201 
202     // Verify that after another restart the result is still there
203     restart();
204     result = procExecutor.getResult(procId);
205     ProcedureTestingUtility.assertProcNotFailed(result);
206     assertEquals(resultTs, result.getLastUpdate());
207     assertEquals(1, Bytes.toInt(result.getResult()));
208   }
209 
210   @Test(timeout=30000)
211   public void testMultiStepProcRecovery() throws Exception {
212     // Step 0 - kill
213     Procedure proc = new TestMultiStepProcedure();
214     long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
215     assertFalse(procExecutor.isRunning());
216 
217     // Step 0 exec && Step 1 - kill
218     restart();
219     waitProcedure(procId);
220     ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
221     assertFalse(procExecutor.isRunning());
222 
223     // Step 1 exec && step 2 - kill
224     restart();
225     waitProcedure(procId);
226     ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
227     assertFalse(procExecutor.isRunning());
228 
229     // Step 2 exec
230     restart();
231     waitProcedure(procId);
232     assertTrue(procExecutor.isRunning());
233 
234     // The procedure is completed
235     ProcedureInfo result = procExecutor.getResult(procId);
236     ProcedureTestingUtility.assertProcNotFailed(result);
237   }
238 
239   @Test(timeout=30000)
240   public void testMultiStepRollbackRecovery() throws Exception {
241     // Step 0 - kill
242     Procedure proc = new TestMultiStepProcedure();
243     long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
244     assertFalse(procExecutor.isRunning());
245 
246     // Step 0 exec && Step 1 - kill
247     restart();
248     waitProcedure(procId);
249     ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
250     assertFalse(procExecutor.isRunning());
251 
252     // Step 1 exec && step 2 - kill
253     restart();
254     waitProcedure(procId);
255     ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
256     assertFalse(procExecutor.isRunning());
257 
258     // Step 2 exec - rollback - kill
259     procSleepInterval = 2500;
260     restart();
261     assertTrue(procExecutor.abort(procId));
262     waitProcedure(procId);
263     assertFalse(procExecutor.isRunning());
264 
265     // rollback - kill
266     restart();
267     waitProcedure(procId);
268     ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
269     assertFalse(procExecutor.isRunning());
270 
271     // rollback - complete
272     restart();
273     waitProcedure(procId);
274     ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
275     assertFalse(procExecutor.isRunning());
276 
277     // Restart the executor and get the result
278     restart();
279     waitProcedure(procId);
280 
281     // The procedure is completed
282     ProcedureInfo result = procExecutor.getResult(procId);
283     ProcedureTestingUtility.assertIsAbortException(result);
284   }
285 
286   public static class TestStateMachineProcedure
287       extends StateMachineProcedure<Void, TestStateMachineProcedure.State> {
288     enum State { STATE_1, STATE_2, STATE_3, DONE }
289 
290     public TestStateMachineProcedure() {}
291 
292     private AtomicBoolean aborted = new AtomicBoolean(false);
293     private int iResult = 0;
294 
295     @Override
296     protected StateMachineProcedure.Flow executeFromState(Void env, State state) {
297       switch (state) {
298         case STATE_1:
299           LOG.info("execute step 1 " + this);
300           setNextState(State.STATE_2);
301           iResult += 3;
302           break;
303         case STATE_2:
304           LOG.info("execute step 2 " + this);
305           setNextState(State.STATE_3);
306           iResult += 5;
307           break;
308         case STATE_3:
309           LOG.info("execute step 3 " + this);
310           Threads.sleepWithoutInterrupt(procSleepInterval);
311           if (aborted.get()) {
312             LOG.info("aborted step 3 " + this);
313             setAbortFailure("test", "aborted");
314             break;
315           }
316           setNextState(State.DONE);
317           iResult += 7;
318           setResult(Bytes.toBytes(iResult));
319           return Flow.NO_MORE_STATE;
320         default:
321           throw new UnsupportedOperationException();
322       }
323       return Flow.HAS_MORE_STATE;
324     }
325 
326     @Override
327     protected void rollbackState(Void env, final State state) {
328       switch (state) {
329         case STATE_1:
330           LOG.info("rollback step 1 " + this);
331           break;
332         case STATE_2:
333           LOG.info("rollback step 2 " + this);
334           break;
335         case STATE_3:
336           LOG.info("rollback step 3 " + this);
337           break;
338         default:
339           throw new UnsupportedOperationException();
340       }
341     }
342 
343     @Override
344     protected State getState(final int stateId) {
345       return State.values()[stateId];
346     }
347 
348     @Override
349     protected int getStateId(final State state) {
350       return state.ordinal();
351     }
352 
353     @Override
354     protected State getInitialState() {
355       return State.STATE_1;
356     }
357 
358     @Override
359     protected boolean abort(Void env) {
360       aborted.set(true);
361       return true;
362     }
363 
364     @Override
365     protected void serializeStateData(final OutputStream stream) throws IOException {
366       super.serializeStateData(stream);
367       stream.write(Bytes.toBytes(iResult));
368     }
369 
370     @Override
371     protected void deserializeStateData(final InputStream stream) throws IOException {
372       super.deserializeStateData(stream);
373       byte[] data = new byte[4];
374       stream.read(data);
375       iResult = Bytes.toInt(data);
376     }
377   }
378 
379   @Test(timeout=30000)
380   public void testStateMachineRecovery() throws Exception {
381     ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, true);
382     ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, true);
383 
384     // Step 1 - kill
385     Procedure proc = new TestStateMachineProcedure();
386     long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
387     assertFalse(procExecutor.isRunning());
388 
389     // Step 1 exec && Step 2 - kill
390     restart();
391     waitProcedure(procId);
392     ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
393     assertFalse(procExecutor.isRunning());
394 
395     // Step 2 exec && step 3 - kill
396     restart();
397     waitProcedure(procId);
398     ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
399     assertFalse(procExecutor.isRunning());
400 
401     // Step 3 exec
402     restart();
403     waitProcedure(procId);
404     assertTrue(procExecutor.isRunning());
405 
406     // The procedure is completed
407     ProcedureInfo result = procExecutor.getResult(procId);
408     ProcedureTestingUtility.assertProcNotFailed(result);
409     assertEquals(15, Bytes.toInt(result.getResult()));
410   }
411 
412   @Test(timeout=30000)
413   public void testStateMachineRollbackRecovery() throws Exception {
414     ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, true);
415     ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, true);
416 
417     // Step 1 - kill
418     Procedure proc = new TestStateMachineProcedure();
419     long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
420     ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
421     assertFalse(procExecutor.isRunning());
422 
423     // Step 1 exec && Step 2 - kill
424     restart();
425     waitProcedure(procId);
426     ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
427     assertFalse(procExecutor.isRunning());
428 
429     // Step 2 exec && step 3 - kill
430     restart();
431     waitProcedure(procId);
432     ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
433     assertFalse(procExecutor.isRunning());
434 
435     // Step 3 exec - rollback step 3 - kill
436     procSleepInterval = 2500;
437     restart();
438     assertTrue(procExecutor.abort(procId));
439     waitProcedure(procId);
440     ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
441     assertFalse(procExecutor.isRunning());
442 
443     // Rollback step 3 - rollback step 2 - kill
444     restart();
445     waitProcedure(procId);
446     assertFalse(procExecutor.isRunning());
447     ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
448 
449     // Rollback step 2 - step 1 - kill
450     restart();
451     waitProcedure(procId);
452     assertFalse(procExecutor.isRunning());
453     ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
454 
455     // Rollback step 1 - complete
456     restart();
457     waitProcedure(procId);
458     assertTrue(procExecutor.isRunning());
459 
460     // The procedure is completed
461     ProcedureInfo result = procExecutor.getResult(procId);
462     ProcedureTestingUtility.assertIsAbortException(result);
463   }
464 
465   private void waitProcedure(final long procId) {
466     ProcedureTestingUtility.waitProcedure(procExecutor, procId);
467     dumpLogDirState();
468   }
469 
470   private void dumpLogDirState() {
471     try {
472       FileStatus[] files = fs.listStatus(logDir);
473       if (files != null && files.length > 0) {
474         for (FileStatus file: files) {
475           assertTrue(file.toString(), file.isFile());
476           LOG.debug("log file " + file.getPath() + " size=" + file.getLen());
477         }
478       } else {
479         LOG.debug("no files under: " + logDir);
480       }
481     } catch (IOException e) {
482       LOG.warn("Unable to dump " + logDir, e);
483     }
484   }
485 }