1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.wal;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.assertTrue;
25
26 import java.io.IOException;
27 import java.util.HashSet;
28 import java.util.Random;
29 import java.util.Set;
30 import java.util.concurrent.atomic.AtomicLong;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileStatus;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.HBaseTestingUtility;
39 import org.apache.hadoop.hbase.HColumnDescriptor;
40 import org.apache.hadoop.hbase.HConstants;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.KeyValue;
44 import org.apache.hadoop.hbase.testclassification.MediumTests;
45 import org.apache.hadoop.hbase.ServerName;
46 import org.apache.hadoop.hbase.TableName;
47 import org.apache.hadoop.hbase.util.Bytes;
48 import org.apache.hadoop.hbase.util.FSUtils;
49 import org.junit.After;
50 import org.junit.AfterClass;
51 import org.junit.Before;
52 import org.junit.BeforeClass;
53 import org.junit.Rule;
54 import org.junit.Test;
55 import org.junit.experimental.categories.Category;
56 import org.junit.rules.TestName;
57
58
59 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
60
61 @Category(MediumTests.class)
62 public class TestDefaultWALProvider {
63 protected static final Log LOG = LogFactory.getLog(TestDefaultWALProvider.class);
64
65 protected static Configuration conf;
66 protected static FileSystem fs;
67 protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
68
69 @Rule
70 public final TestName currentTest = new TestName();
71
72 @Before
73 public void setUp() throws Exception {
74 FileStatus[] entries = fs.listStatus(new Path("/"));
75 for (FileStatus dir : entries) {
76 fs.delete(dir.getPath(), true);
77 }
78 }
79
80 @After
81 public void tearDown() throws Exception {
82 }
83
84 @BeforeClass
85 public static void setUpBeforeClass() throws Exception {
86
87 TEST_UTIL.getConfiguration().setInt("dfs.blocksize", 1024 * 1024);
88
89 TEST_UTIL.getConfiguration().setInt("dfs.namenode.heartbeat.recheck-interval", 5000);
90 TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1);
91 TEST_UTIL.getConfiguration().setInt("dfs.client.socket-timeout", 5000);
92
93
94 TEST_UTIL.getConfiguration()
95 .setInt("hbase.ipc.client.connect.max.retries", 1);
96 TEST_UTIL.getConfiguration().setInt(
97 "dfs.client.block.recovery.retries", 1);
98 TEST_UTIL.getConfiguration().setInt(
99 "hbase.ipc.client.connection.maxidletime", 500);
100 TEST_UTIL.startMiniDFSCluster(3);
101
102
103 TEST_UTIL.createRootDir();
104 conf = TEST_UTIL.getConfiguration();
105 fs = TEST_UTIL.getDFSCluster().getFileSystem();
106 }
107
108 @AfterClass
109 public static void tearDownAfterClass() throws Exception {
110 TEST_UTIL.shutdownMiniCluster();
111 }
112
113 static String getName() {
114 return "TestDefaultWALProvider";
115 }
116
117 @Test
118 public void testGetServerNameFromWALDirectoryName() throws IOException {
119 ServerName sn = ServerName.valueOf("hn", 450, 1398);
120 String hl = FSUtils.getRootDir(conf) + "/" +
121 DefaultWALProvider.getWALDirectoryName(sn.toString());
122
123
124 assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, null));
125 assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf,
126 FSUtils.getRootDir(conf).toUri().toString()));
127 assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, ""));
128 assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, " "));
129 assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, hl));
130 assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, hl + "qdf"));
131 assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, "sfqf" + hl + "qdf"));
132
133 final String wals = "/WALs/";
134 ServerName parsed = DefaultWALProvider.getServerNameFromWALDirectoryName(conf,
135 FSUtils.getRootDir(conf).toUri().toString() + wals + sn +
136 "/localhost%2C32984%2C1343316388997.1343316390417");
137 assertEquals("standard", sn, parsed);
138
139 parsed = DefaultWALProvider.getServerNameFromWALDirectoryName(conf, hl + "/qdf");
140 assertEquals("subdir", sn, parsed);
141
142 parsed = DefaultWALProvider.getServerNameFromWALDirectoryName(conf,
143 FSUtils.getRootDir(conf).toUri().toString() + wals + sn +
144 "-splitting/localhost%3A57020.1340474893931");
145 assertEquals("split", sn, parsed);
146 }
147
148
149 protected void addEdits(WAL log, HRegionInfo hri, HTableDescriptor htd,
150 int times, AtomicLong sequenceId) throws IOException {
151 final byte[] row = Bytes.toBytes("row");
152 for (int i = 0; i < times; i++) {
153 long timestamp = System.currentTimeMillis();
154 WALEdit cols = new WALEdit();
155 cols.add(new KeyValue(row, row, row, timestamp, row));
156 log.append(htd, hri, getWalKey(hri.getEncodedNameAsBytes(), htd.getTableName(), timestamp),
157 cols, sequenceId, true, null);
158 }
159 log.sync();
160 }
161
162
163
164
165 WALKey getWalKey(final byte[] info, final TableName tableName, final long timestamp) {
166 return new WALKey(info, tableName, timestamp);
167 }
168
169
170
171
172
173
174 protected void flushRegion(WAL wal, byte[] regionEncodedName, Set<byte[]> flushedFamilyNames) {
175 wal.startCacheFlush(regionEncodedName, flushedFamilyNames);
176 wal.completeCacheFlush(regionEncodedName);
177 }
178
179 private static final byte[] UNSPECIFIED_REGION = new byte[]{};
180
181 @Test
182 public void testLogCleaning() throws Exception {
183 LOG.info("testLogCleaning");
184 final HTableDescriptor htd =
185 new HTableDescriptor(TableName.valueOf("testLogCleaning")).addFamily(new HColumnDescriptor(
186 "row"));
187 final HTableDescriptor htd2 =
188 new HTableDescriptor(TableName.valueOf("testLogCleaning2"))
189 .addFamily(new HColumnDescriptor("row"));
190 final Configuration localConf = new Configuration(conf);
191 localConf.set(WALFactory.WAL_PROVIDER, DefaultWALProvider.class.getName());
192 final WALFactory wals = new WALFactory(localConf, null, currentTest.getMethodName());
193 final AtomicLong sequenceId = new AtomicLong(1);
194 try {
195 HRegionInfo hri = new HRegionInfo(htd.getTableName(),
196 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
197 HRegionInfo hri2 = new HRegionInfo(htd2.getTableName(),
198 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
199
200 final WAL log = wals.getWAL(UNSPECIFIED_REGION);
201
202
203
204 addEdits(log, hri, htd, 1, sequenceId);
205 log.rollWriter();
206 assertEquals(1, DefaultWALProvider.getNumRolledLogFiles(log));
207
208
209 addEdits(log, hri, htd, 2, sequenceId);
210 log.rollWriter();
211 assertEquals(2, DefaultWALProvider.getNumRolledLogFiles(log));
212
213
214 addEdits(log, hri, htd, 1, sequenceId);
215 addEdits(log, hri2, htd2, 1, sequenceId);
216 addEdits(log, hri, htd, 1, sequenceId);
217 addEdits(log, hri2, htd2, 1, sequenceId);
218 log.rollWriter();
219 assertEquals(3, DefaultWALProvider.getNumRolledLogFiles(log));
220
221
222
223 addEdits(log, hri2, htd2, 1, sequenceId);
224 log.startCacheFlush(hri.getEncodedNameAsBytes(), htd.getFamiliesKeys());
225 log.completeCacheFlush(hri.getEncodedNameAsBytes());
226 log.rollWriter();
227 assertEquals(2, DefaultWALProvider.getNumRolledLogFiles(log));
228
229
230
231
232 addEdits(log, hri2, htd2, 1, sequenceId);
233 log.startCacheFlush(hri2.getEncodedNameAsBytes(), htd2.getFamiliesKeys());
234 log.completeCacheFlush(hri2.getEncodedNameAsBytes());
235 log.rollWriter();
236 assertEquals(0, DefaultWALProvider.getNumRolledLogFiles(log));
237 } finally {
238 if (wals != null) {
239 wals.close();
240 }
241 }
242 }
243
244
245
246
247
248
249
250
251
252
253
254
255
256 @Test
257 public void testWALArchiving() throws IOException {
258 LOG.debug("testWALArchiving");
259 HTableDescriptor table1 =
260 new HTableDescriptor(TableName.valueOf("t1")).addFamily(new HColumnDescriptor("row"));
261 HTableDescriptor table2 =
262 new HTableDescriptor(TableName.valueOf("t2")).addFamily(new HColumnDescriptor("row"));
263 final Configuration localConf = new Configuration(conf);
264 localConf.set(WALFactory.WAL_PROVIDER, DefaultWALProvider.class.getName());
265 final WALFactory wals = new WALFactory(localConf, null, currentTest.getMethodName());
266 try {
267 final WAL wal = wals.getWAL(UNSPECIFIED_REGION);
268 assertEquals(0, DefaultWALProvider.getNumRolledLogFiles(wal));
269 HRegionInfo hri1 =
270 new HRegionInfo(table1.getTableName(), HConstants.EMPTY_START_ROW,
271 HConstants.EMPTY_END_ROW);
272 HRegionInfo hri2 =
273 new HRegionInfo(table2.getTableName(), HConstants.EMPTY_START_ROW,
274 HConstants.EMPTY_END_ROW);
275
276 hri1.setSplit(false);
277 hri2.setSplit(false);
278
279 final AtomicLong sequenceId1 = new AtomicLong(1);
280 final AtomicLong sequenceId2 = new AtomicLong(1);
281
282 addEdits(wal, hri1, table1, 1, sequenceId1);
283 wal.rollWriter();
284
285 assertEquals(1, DefaultWALProvider.getNumRolledLogFiles(wal));
286
287 addEdits(wal, hri1, table1, 1, sequenceId1);
288 wal.rollWriter();
289
290 assertEquals(2, DefaultWALProvider.getNumRolledLogFiles(wal));
291
292 addEdits(wal, hri1, table1, 3, sequenceId1);
293 flushRegion(wal, hri1.getEncodedNameAsBytes(), table1.getFamiliesKeys());
294
295 wal.rollWriter();
296 assertEquals(0, DefaultWALProvider.getNumRolledLogFiles(wal));
297
298 addEdits(wal, hri2, table2, 1, sequenceId2);
299 wal.rollWriter();
300 assertEquals(1, DefaultWALProvider.getNumRolledLogFiles(wal));
301
302 addEdits(wal, hri1, table1, 2, sequenceId1);
303 wal.rollWriter();
304 assertEquals(2, DefaultWALProvider.getNumRolledLogFiles(wal));
305
306 addEdits(wal, hri2, table2, 2, sequenceId2);
307 flushRegion(wal, hri1.getEncodedNameAsBytes(), table2.getFamiliesKeys());
308
309
310
311
312
313 wal.rollWriter();
314 assertEquals(2, DefaultWALProvider.getNumRolledLogFiles(wal));
315
316 addEdits(wal, hri2, table2, 2, sequenceId2);
317 flushRegion(wal, hri2.getEncodedNameAsBytes(), table2.getFamiliesKeys());
318 wal.rollWriter();
319 assertEquals(0, DefaultWALProvider.getNumRolledLogFiles(wal));
320 } finally {
321 if (wals != null) {
322 wals.close();
323 }
324 }
325 }
326
327
328
329
330
331 @Test
332 public void testConcurrentWrites() throws Exception {
333
334
335 int errCode = WALPerformanceEvaluation.
336 innerMain(new Configuration(TEST_UTIL.getConfiguration()),
337 new String [] {"-threads", "3", "-verify", "-noclosefs", "-iterations", "3000"});
338 assertEquals(0, errCode);
339 }
340
341
342
343
344 @Test
345 public void setMembershipDedups() throws IOException {
346 final Configuration localConf = new Configuration(conf);
347 localConf.set(WALFactory.WAL_PROVIDER, DefaultWALProvider.class.getName());
348 final WALFactory wals = new WALFactory(localConf, null, currentTest.getMethodName());
349 try {
350 final Set<WAL> seen = new HashSet<WAL>(1);
351 final Random random = new Random();
352 assertTrue("first attempt to add WAL from default provider should work.",
353 seen.add(wals.getWAL(Bytes.toBytes(random.nextInt()))));
354 for (int i = 0; i < 1000; i++) {
355 assertFalse("default wal provider is only supposed to return a single wal, which should " +
356 "compare as .equals itself.", seen.add(wals.getWAL(Bytes.toBytes(random.nextInt()))));
357 }
358 } finally {
359 wals.close();
360 }
361 }
362 }