1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21
22 import static org.junit.Assert.assertTrue;
23
24 import java.io.IOException;
25 import java.util.ArrayList;
26 import java.util.List;
27 import java.util.concurrent.CountDownLatch;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.Cell;
35 import org.apache.hadoop.hbase.CellScanner;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HTableDescriptor;
39 import org.apache.hadoop.hbase.Server;
40 import org.apache.hadoop.hbase.TableName;
41 import org.apache.hadoop.hbase.client.Durability;
42 import org.apache.hadoop.hbase.client.Put;
43 import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
44 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
45 import org.apache.hadoop.hbase.testclassification.MediumTests;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
48 import org.apache.hadoop.hbase.util.Threads;
49 import org.apache.hadoop.hbase.wal.WAL;
50 import org.apache.hadoop.hbase.wal.WALKey;
51 import org.apache.hadoop.hbase.wal.WALProvider.Writer;
52 import org.junit.After;
53 import org.junit.Before;
54 import org.junit.Ignore;
55 import org.junit.Rule;
56 import org.junit.Test;
57 import org.junit.experimental.categories.Category;
58 import org.junit.rules.TestName;
59 import org.mockito.Mockito;
60
61
62
63
64
65 @Category({MediumTests.class})
66 public class TestWALLockup {
67 private static final Log LOG = LogFactory.getLog(TestWALLockup.class);
68 @Rule public TestName name = new TestName();
69
70 private static final String COLUMN_FAMILY = "MyCF";
71 private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
72
73 HRegion region = null;
74
75 private static HBaseTestingUtility TEST_UTIL;
76 private static Configuration CONF ;
77 private String dir;
78
79
80 protected TableName tableName;
81
82 @Before
83 public void setup() throws IOException {
84 TEST_UTIL = HBaseTestingUtility.createLocalHTU();
85 CONF = TEST_UTIL.getConfiguration();
86
87 CONF.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
88 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
89 tableName = TableName.valueOf(name.getMethodName());
90 }
91
92 @After
93 public void tearDown() throws Exception {
94 EnvironmentEdgeManagerTestHelper.reset();
95 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
96 TEST_UTIL.cleanupTestDir();
97 }
98
99 String getName() {
100 return name.getMethodName();
101 }
102
103
104
105
106
107
108
109
110 @Ignore @Test (timeout=30000)
111 public void testLockupWhenSyncInMiddleOfZigZagSetup() throws IOException {
112
113 class DodgyFSLog extends FSHLog {
114
115 volatile boolean throwException = false;
116
117
118 CountDownLatch latch = new CountDownLatch(1);
119
120 public DodgyFSLog(FileSystem fs, Path root, String logDir, Configuration conf)
121 throws IOException {
122 super(fs, root, logDir, conf);
123 }
124
125 @Override
126 protected void afterCreatingZigZagLatch() {
127
128
129
130
131 if (throwException) {
132 try {
133 LOG.info("LATCHED");
134 this.latch.await();
135 } catch (InterruptedException e) {
136
137 e.printStackTrace();
138 }
139 }
140 }
141
142 @Override
143 protected void beforeWaitOnSafePoint() {
144 if (throwException) {
145 LOG.info("COUNTDOWN");
146
147
148
149 while (this.latch.getCount() <= 0) Threads.sleep(1);
150 this.latch.countDown();
151 }
152 }
153
154 @Override
155 protected Writer createWriterInstance(Path path) throws IOException {
156 final Writer w = super.createWriterInstance(path);
157 return new Writer() {
158 @Override
159 public void close() throws IOException {
160 w.close();
161 }
162
163 @Override
164 public void sync() throws IOException {
165 if (throwException) {
166 throw new IOException("FAKE! Failed to replace a bad datanode...SYNC");
167 }
168 w.sync();
169 }
170
171 @Override
172 public void append(Entry entry) throws IOException {
173 if (throwException) {
174 throw new IOException("FAKE! Failed to replace a bad datanode...APPEND");
175 }
176 w.append(entry);
177 }
178
179 @Override
180 public long getLength() throws IOException {
181 return w.getLength();
182 }
183 };
184 }
185 }
186
187
188 Server server = Mockito.mock(Server.class);
189 Mockito.when(server.getConfiguration()).thenReturn(CONF);
190 Mockito.when(server.isStopped()).thenReturn(false);
191 Mockito.when(server.isAborted()).thenReturn(false);
192 RegionServerServices services = Mockito.mock(RegionServerServices.class);
193
194
195 FileSystem fs = FileSystem.get(CONF);
196 Path rootDir = new Path(dir + getName());
197 DodgyFSLog dodgyWAL = new DodgyFSLog(fs, rootDir, getName(), CONF);
198 Path originalWAL = dodgyWAL.getCurrentFileName();
199
200 LogRoller logRoller = new LogRoller(server, services);
201 logRoller.addWAL(dodgyWAL);
202
203 logRoller.start();
204
205 HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME);
206 final HRegion region = initHRegion(tableName, null, null, dodgyWAL);
207 byte [] bytes = Bytes.toBytes(getName());
208 try {
209
210
211
212 Put put = new Put(bytes);
213 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("1"), bytes);
214 WALKey key = new WALKey(region.getRegionInfo().getEncodedNameAsBytes(), htd.getTableName());
215 WALEdit edit = new WALEdit();
216 List<Cell> cells = new ArrayList<Cell>();
217 for (CellScanner cs = put.cellScanner(); cs.advance();) {
218 edit.add(cs.current());
219 cells.add(cs.current());
220 }
221
222
223 for (int i = 0; i < 1000; i++) {
224 dodgyWAL.append(htd, region.getRegionInfo(), key, edit, region.getSequenceId(), true,
225 cells);
226 }
227
228 dodgyWAL.throwException = true;
229
230 dodgyWAL.append(htd, region.getRegionInfo(), key, edit, region.getSequenceId(), true, cells);
231 boolean exception = false;
232 try {
233 dodgyWAL.sync();
234 } catch (Exception e) {
235 exception = true;
236 }
237 assertTrue("Did not get sync exception", exception);
238
239
240
241
242 Thread t = new Thread ("flusher") {
243 public void run() {
244 try {
245 region.flush(false);
246 } catch (IOException e) {
247
248 e.printStackTrace();
249 }
250 };
251 };
252 t.setDaemon(true);
253 t.start();
254
255 while (!region.writestate.flushing) Threads.sleep(1);
256
257 assertTrue(originalWAL != dodgyWAL.getCurrentFileName());
258
259 dodgyWAL.throwException = false;
260 region.put(put);
261 } finally {
262
263 Mockito.when(server.isStopped()).thenReturn(true);
264 if (logRoller != null) logRoller.interrupt();
265 if (region != null) region.close();
266 if (dodgyWAL != null) dodgyWAL.close();
267 }
268 }
269
270
271
272
273
274 public HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey, WAL wal)
275 throws IOException {
276 return TEST_UTIL.createLocalHRegion(tableName.getName(), startKey, stopKey,
277 getName(), CONF, false, Durability.SYNC_WAL,
278 wal, COLUMN_FAMILY_BYTES);
279 }
280 }