1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import java.util.List;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.fs.FileSystem;
27 import org.apache.hadoop.fs.Path;
28 import org.apache.hadoop.hbase.TableName;
29 import org.apache.hadoop.hbase.HBaseTestingUtility;
30 import org.apache.hadoop.hbase.HColumnDescriptor;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.HRegionInfo;
33 import org.apache.hadoop.hbase.HTableDescriptor;
34 import org.apache.hadoop.hbase.testclassification.LargeTests;
35 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
36 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
37 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.junit.After;
40 import org.junit.AfterClass;
41 import org.junit.Assert;
42 import org.junit.Assume;
43 import org.junit.Before;
44 import org.junit.BeforeClass;
45 import org.junit.Test;
46 import org.junit.experimental.categories.Category;
47
48
49
50
51 @Category(LargeTests.class)
52 public class TestSnapshotCloneIndependence {
53 private static final Log LOG = LogFactory.getLog(TestSnapshotCloneIndependence.class);
54
55
56 private static final boolean WINDOWS = System.getProperty("os.name").startsWith("Windows");
57
58 protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
59
60 protected static final int NUM_RS = 2;
61 private static final String STRING_TABLE_NAME = "test";
62 private static final String TEST_FAM_STR = "fam";
63 protected static final byte[] TEST_FAM = Bytes.toBytes(TEST_FAM_STR);
64 protected static final TableName TABLE_NAME = TableName.valueOf(STRING_TABLE_NAME);
65 private static final int CLEANER_INTERVAL = 10;
66
67
68
69
70
71 @BeforeClass
72 public static void setupCluster() throws Exception {
73 setupConf(UTIL.getConfiguration());
74 UTIL.startMiniCluster(NUM_RS);
75 }
76
77 protected static void setupConf(Configuration conf) {
78
79 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
80
81 conf.setInt("hbase.regionsever.info.port", -1);
82
83 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
84
85
86 conf.setInt("hbase.hstore.compaction.min", 10);
87 conf.setInt("hbase.hstore.compactionThreshold", 10);
88
89 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
90 conf.setInt("hbase.regionserver.msginterval", 100);
91 conf.setBoolean("hbase.master.enabletable.roundrobin", true);
92
93 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
94 ConstantSizeRegionSplitPolicy.class.getName());
95
96 conf.setInt("hbase.master.cleaner.interval", CLEANER_INTERVAL);
97 conf.setInt("hbase.master.hfilecleaner.plugins.snapshot.period", CLEANER_INTERVAL);
98
99
100
101 conf.setInt("hbase.master.hfilecleaner.ttl", CLEANER_INTERVAL);
102 }
103
104 @Before
105 public void setup() throws Exception {
106 createTable(TABLE_NAME, TEST_FAM);
107 }
108
109 @After
110 public void tearDown() throws Exception {
111 UTIL.deleteTable(TABLE_NAME);
112 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
113 SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
114 }
115
116 @AfterClass
117 public static void cleanupTest() throws Exception {
118 try {
119 UTIL.shutdownMiniCluster();
120 } catch (Exception e) {
121 LOG.warn("failure shutting down cluster", e);
122 }
123 }
124
125
126
127
128
129 @Test (timeout=300000)
130 public void testOnlineSnapshotAppendIndependent() throws Exception {
131 runTestSnapshotAppendIndependent(true);
132 }
133
134
135
136
137
138 @Test (timeout=300000)
139 public void testOfflineSnapshotAppendIndependent() throws Exception {
140 runTestSnapshotAppendIndependent(false);
141 }
142
143
144
145
146
147 @Test (timeout=300000)
148 public void testOnlineSnapshotMetadataChangesIndependent() throws Exception {
149 runTestSnapshotMetadataChangesIndependent(true);
150 }
151
152
153
154
155
156 @Test (timeout=300000)
157 public void testOfflineSnapshotMetadataChangesIndependent() throws Exception {
158 runTestSnapshotMetadataChangesIndependent(false);
159 }
160
161
162
163
164
165 @Test (timeout=300000)
166 public void testOfflineSnapshotRegionOperationsIndependent() throws Exception {
167 runTestRegionOperationsIndependent(false);
168 }
169
170
171
172
173
174 @Test (timeout=300000)
175 public void testOnlineSnapshotRegionOperationsIndependent() throws Exception {
176 runTestRegionOperationsIndependent(true);
177 }
178
179 @Test (timeout=300000)
180 public void testOfflineSnapshotDeleteIndependent() throws Exception {
181
182
183 Assume.assumeTrue(!WINDOWS);
184
185 runTestSnapshotDeleteIndependent(false);
186 }
187
188 @Test (timeout=300000)
189 public void testOnlineSnapshotDeleteIndependent() throws Exception {
190 runTestSnapshotDeleteIndependent(true);
191 }
192
193 private static void waitOnSplit(final HTable t, int originalCount) throws Exception {
194 for (int i = 0; i < 200; i++) {
195 try {
196 Thread.sleep(50);
197 } catch (InterruptedException e) {
198
199 Thread.currentThread().interrupt();
200 }
201 if (t.getAllRegionLocations().size() > originalCount) {
202 return;
203 }
204 }
205 throw new Exception("Split did not increase the number of regions");
206 }
207
208
209
210
211
212
213 private void runTestSnapshotAppendIndependent(boolean online) throws Exception {
214 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
215 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
216
217 Admin admin = UTIL.getHBaseAdmin();
218 final long startTime = System.currentTimeMillis();
219 final TableName localTableName =
220 TableName.valueOf(STRING_TABLE_NAME + startTime);
221
222 try (Table original = createTable(localTableName, TEST_FAM)) {
223 loadData(original, TEST_FAM);
224 final int origTableRowCount = countRows(original);
225
226
227 final String snapshotNameAsString = "snapshot_" + localTableName;
228 byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
229
230 SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
231 snapshotNameAsString, rootDir, fs, online);
232
233 if (!online) {
234 admin.enableTable(localTableName);
235 }
236 TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
237 admin.cloneSnapshot(snapshotName, cloneTableName);
238
239 try (Table clonedTable = new HTable(UTIL.getConfiguration(), cloneTableName)){
240 final int clonedTableRowCount = countRows(clonedTable);
241
242 Assert.assertEquals(
243 "The line counts of original and cloned tables do not match after clone. ",
244 origTableRowCount, clonedTableRowCount);
245
246
247 final String rowKey = "new-row-" + System.currentTimeMillis();
248
249 Put p = new Put(Bytes.toBytes(rowKey));
250 p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
251 original.put(p);
252
253
254 Assert.assertEquals("The row count of the original table was not modified by the put",
255 origTableRowCount + 1, countRows(original));
256 Assert.assertEquals(
257 "The row count of the cloned table changed as a result of addition to the original",
258 clonedTableRowCount, countRows(clonedTable));
259
260 p = new Put(Bytes.toBytes(rowKey));
261 p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
262 clonedTable.put(p);
263
264
265 Assert.assertEquals(
266 "The row count of the original table was modified by the put to the clone",
267 origTableRowCount + 1, countRows(original));
268 Assert.assertEquals("The row count of the cloned table was not modified by the put",
269 clonedTableRowCount + 1, countRows(clonedTable));
270 }
271 }
272 }
273
274
275
276
277
278 private void runTestRegionOperationsIndependent(boolean online) throws Exception {
279 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
280 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
281
282
283 Admin admin = UTIL.getHBaseAdmin();
284 final long startTime = System.currentTimeMillis();
285 final TableName localTableName =
286 TableName.valueOf(STRING_TABLE_NAME + startTime);
287 Table original = createTable(localTableName, TEST_FAM);
288 loadData(original, TEST_FAM);
289 final int loadedTableCount = countRows(original);
290 System.out.println("Original table has: " + loadedTableCount + " rows");
291
292 final String snapshotNameAsString = "snapshot_" + localTableName;
293
294
295 SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
296 snapshotNameAsString, rootDir, fs, online);
297
298 if (!online) {
299 admin.enableTable(localTableName);
300 }
301
302 TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
303
304
305 byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
306 admin.cloneSnapshot(snapshotName, cloneTableName);
307
308
309 ((HTable)original).clearRegionCache();
310 List<HRegionInfo> originalTableHRegions = admin.getTableRegions(localTableName);
311
312 final int originalRegionCount = originalTableHRegions.size();
313 final int cloneTableRegionCount = admin.getTableRegions(cloneTableName).size();
314 Assert.assertEquals(
315 "The number of regions in the cloned table is different than in the original table.",
316 originalRegionCount, cloneTableRegionCount);
317
318
319 admin.splitRegion(originalTableHRegions.get(0).getRegionName());
320 waitOnSplit((HTable)original, originalRegionCount);
321
322
323 final int cloneTableRegionCount2 = admin.getTableRegions(cloneTableName).size();
324 Assert.assertEquals(
325 "The number of regions in the cloned table changed though none of its regions were split.",
326 cloneTableRegionCount, cloneTableRegionCount2);
327 }
328
329
330
331
332
333
334 private void runTestSnapshotMetadataChangesIndependent(boolean online) throws Exception {
335 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
336 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
337
338
339 Admin admin = UTIL.getHBaseAdmin();
340 final long startTime = System.currentTimeMillis();
341 final TableName localTableName =
342 TableName.valueOf(STRING_TABLE_NAME + startTime);
343 Table original = createTable(localTableName, TEST_FAM);
344 loadData(original, TEST_FAM);
345
346 final String snapshotNameAsString = "snapshot_" + localTableName;
347
348
349 SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
350 snapshotNameAsString, rootDir, fs, online);
351
352 if (!online) {
353 admin.enableTable(localTableName);
354 }
355 TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
356
357
358 byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
359 admin.cloneSnapshot(snapshotName, cloneTableName);
360
361
362 byte[] TEST_FAM_2 = Bytes.toBytes("fam2");
363 HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM_2);
364
365 admin.disableTable(localTableName);
366 admin.addColumn(localTableName, hcd);
367
368
369 admin.enableTable(localTableName);
370
371
372
373
374 HTableDescriptor originalTableDescriptor = original.getTableDescriptor();
375 HTableDescriptor clonedTableDescriptor = admin.getTableDescriptor(cloneTableName);
376
377 Assert.assertTrue("The original family was not found. There is something wrong. ",
378 originalTableDescriptor.hasFamily(TEST_FAM));
379 Assert.assertTrue("The original family was not found in the clone. There is something wrong. ",
380 clonedTableDescriptor.hasFamily(TEST_FAM));
381
382 Assert.assertTrue("The new family was not found. ",
383 originalTableDescriptor.hasFamily(TEST_FAM_2));
384 Assert.assertTrue("The new family was not found. ",
385 !clonedTableDescriptor.hasFamily(TEST_FAM_2));
386 }
387
388
389
390
391
392
393 private void runTestSnapshotDeleteIndependent(boolean online) throws Exception {
394 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
395 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
396
397 final Admin admin = UTIL.getHBaseAdmin();
398 final long startTime = System.currentTimeMillis();
399 final TableName localTableName =
400 TableName.valueOf(STRING_TABLE_NAME + startTime);
401
402 try (Table original = createTable(localTableName, TEST_FAM)) {
403 loadData(original, TEST_FAM);
404 }
405
406
407 final String snapshotNameAsString = "snapshot_" + localTableName;
408 byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
409
410 SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
411 snapshotNameAsString, rootDir, fs, online);
412
413 if (!online) {
414 admin.enableTable(localTableName);
415 }
416 TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
417 admin.cloneSnapshot(snapshotName, cloneTableName);
418
419
420 admin.majorCompact(localTableName);
421
422
423 admin.deleteSnapshot(snapshotName);
424
425
426 Thread.sleep(10000);
427
428 try (Table original = UTIL.getConnection().getTable(localTableName)) {
429 try (Table clonedTable = UTIL.getConnection().getTable(cloneTableName)) {
430
431 final int origTableRowCount = countRows(original);
432 final int clonedTableRowCount = countRows(clonedTable);
433 Assert.assertEquals(origTableRowCount, clonedTableRowCount);
434 }
435 }
436 }
437
438 protected Table createTable(final TableName table, byte[] family) throws Exception {
439 return UTIL.createTable(table, family);
440 }
441
442 protected void loadData(final Table table, byte[]... families) throws Exception {
443 UTIL.loadTable(table, families);
444 }
445
446 protected int countRows(final Table table, final byte[]... families) throws Exception {
447 return UTIL.countRows(table, families);
448 }
449 }