1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.backup;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNull;
25 import static org.junit.Assert.assertTrue;
26
27 import java.io.IOException;
28 import java.util.ArrayList;
29 import java.util.Arrays;
30 import java.util.Collections;
31 import java.util.HashMap;
32 import java.util.Iterator;
33 import java.util.List;
34 import java.util.Set;
35 import java.util.TreeSet;
36
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.hbase.HBaseTestingUtility;
39 import org.apache.hadoop.hbase.MiniHBaseCluster;
40 import org.apache.hadoop.hbase.TableName;
41 import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
42 import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
43 import org.apache.hadoop.hbase.client.Admin;
44 import org.apache.hadoop.hbase.client.Connection;
45 import org.apache.hadoop.hbase.testclassification.MediumTests;
46 import org.junit.After;
47 import org.junit.AfterClass;
48 import org.junit.Before;
49 import org.junit.BeforeClass;
50 import org.junit.Test;
51 import org.junit.experimental.categories.Category;
52
53
54
55
56
57 @Category(MediumTests.class)
58 public class TestBackupSystemTable {
59
60 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
61 protected static Configuration conf = UTIL.getConfiguration();
62 protected static MiniHBaseCluster cluster;
63 protected static Connection conn;
64 protected BackupSystemTable table;
65
66 @BeforeClass
67 public static void setUp() throws Exception {
68 cluster = UTIL.startMiniCluster();
69 conn = UTIL.getConnection();
70 waitForSystemTable();
71 }
72
73 static void waitForSystemTable() throws Exception
74 {
75 try(Admin admin = UTIL.getHBaseAdmin();) {
76 while (!admin.tableExists(BackupSystemTable.getTableName())
77 || !admin.isTableAvailable(BackupSystemTable.getTableName())) {
78 Thread.sleep(1000);
79 }
80 }
81 }
82
83 @Before
84 public void before() throws IOException {
85 table = new BackupSystemTable(conn);
86 }
87
88 @After
89 public void after() {
90 if (table != null) {
91 table.close();
92 }
93
94 }
95
96 @Test
97 public void testUpdateReadDeleteBackupStatus() throws IOException {
98 BackupInfo ctx = createBackupContext();
99 table.updateBackupInfo(ctx);
100 BackupInfo readCtx = table.readBackupInfo(ctx.getBackupId());
101 assertTrue(compare(ctx, readCtx));
102
103 readCtx = table.readBackupInfo("fake");
104 assertNull(readCtx);
105
106 table.deleteBackupInfo(ctx.getBackupId());
107 readCtx = table.readBackupInfo(ctx.getBackupId());
108 assertNull(readCtx);
109 cleanBackupTable();
110 }
111
112 @Test
113 public void testWriteReadBackupStartCode() throws IOException {
114 Long code = 100L;
115 table.writeBackupStartCode(code, "root");
116 String readCode = table.readBackupStartCode("root");
117 assertEquals(code, new Long(Long.parseLong(readCode)));
118 cleanBackupTable();
119 }
120
121 private void cleanBackupTable() throws IOException {
122 Admin admin = UTIL.getHBaseAdmin();
123 admin.disableTable(TableName.BACKUP_TABLE_NAME);
124 admin.truncateTable(TableName.BACKUP_TABLE_NAME, true);
125 if (admin.isTableDisabled(TableName.BACKUP_TABLE_NAME)) {
126 admin.enableTable(TableName.BACKUP_TABLE_NAME);
127 }
128 }
129
130 @Test
131 public void testBackupHistory() throws IOException {
132 int n = 10;
133 List<BackupInfo> list = createBackupContextList(n);
134
135
136 for (BackupInfo bc : list) {
137
138 bc.setState(BackupState.COMPLETE);
139 table.updateBackupInfo(bc);
140 }
141
142
143 Collections.reverse(list);
144 ArrayList<BackupInfo> history = table.getBackupHistory();
145 assertTrue(history.size() == n);
146
147 for (int i = 0; i < n; i++) {
148 BackupInfo ctx = list.get(i);
149 BackupInfo data = history.get(i);
150 assertTrue(compare(ctx, data));
151 }
152
153 cleanBackupTable();
154
155 }
156
157 @Test
158 public void testBackupDelete() throws IOException {
159
160 try (BackupSystemTable table = new BackupSystemTable(conn)) {
161
162 int n = 10;
163 List<BackupInfo> list = createBackupContextList(n);
164
165
166 for (BackupInfo bc : list) {
167
168 bc.setState(BackupState.COMPLETE);
169 table.updateBackupInfo(bc);
170 }
171
172
173 for (BackupInfo bc : list) {
174 assertNotNull(table.readBackupInfo(bc.getBackupId()));
175 }
176
177
178 for (BackupInfo bc : list) {
179 table.deleteBackupInfo(bc.getBackupId());
180 }
181
182
183 for (BackupInfo bc : list) {
184 assertNull(table.readBackupInfo(bc.getBackupId()));
185 }
186
187 cleanBackupTable();
188 }
189
190 }
191
192
193
194 @Test
195 public void testRegionServerLastLogRollResults() throws IOException {
196 String[] servers = new String[] { "server1", "server2", "server3" };
197 Long[] timestamps = new Long[] { 100L, 102L, 107L };
198
199 for (int i = 0; i < servers.length; i++) {
200 table.writeRegionServerLastLogRollResult(servers[i], timestamps[i], "root");
201 }
202
203 HashMap<String, Long> result = table.readRegionServerLastLogRollResult("root");
204 assertTrue(servers.length == result.size());
205 Set<String> keys = result.keySet();
206 String[] keysAsArray = new String[keys.size()];
207 keys.toArray(keysAsArray);
208 Arrays.sort(keysAsArray);
209
210 for (int i = 0; i < keysAsArray.length; i++) {
211 assertEquals(keysAsArray[i], servers[i]);
212 Long ts1 = timestamps[i];
213 Long ts2 = result.get(keysAsArray[i]);
214 assertEquals(ts1, ts2);
215 }
216
217 cleanBackupTable();
218 }
219
220 @Test
221 public void testIncrementalBackupTableSet() throws IOException {
222 TreeSet<TableName> tables1 = new TreeSet<>();
223
224 tables1.add(TableName.valueOf("t1"));
225 tables1.add(TableName.valueOf("t2"));
226 tables1.add(TableName.valueOf("t3"));
227
228 TreeSet<TableName> tables2 = new TreeSet<>();
229
230 tables2.add(TableName.valueOf("t3"));
231 tables2.add(TableName.valueOf("t4"));
232 tables2.add(TableName.valueOf("t5"));
233
234 table.addIncrementalBackupTableSet(tables1, "root");
235 BackupSystemTable table = new BackupSystemTable(conn);
236 TreeSet<TableName> res1 = (TreeSet<TableName>)
237 table.getIncrementalBackupTableSet("root");
238 assertTrue(tables1.size() == res1.size());
239 Iterator<TableName> desc1 = tables1.descendingIterator();
240 Iterator<TableName> desc2 = res1.descendingIterator();
241 while (desc1.hasNext()) {
242 assertEquals(desc1.next(), desc2.next());
243 }
244
245 table.addIncrementalBackupTableSet(tables2, "root");
246 TreeSet<TableName> res2 = (TreeSet<TableName>)
247 table.getIncrementalBackupTableSet("root");
248 assertTrue((tables2.size() + tables1.size() - 1) == res2.size());
249
250 tables1.addAll(tables2);
251
252 desc1 = tables1.descendingIterator();
253 desc2 = res2.descendingIterator();
254
255 while (desc1.hasNext()) {
256 assertEquals(desc1.next(), desc2.next());
257 }
258 cleanBackupTable();
259
260 }
261
262 @Test
263 public void testRegionServerLogTimestampMap() throws IOException {
264 TreeSet<TableName> tables = new TreeSet<>();
265
266 tables.add(TableName.valueOf("t1"));
267 tables.add(TableName.valueOf("t2"));
268 tables.add(TableName.valueOf("t3"));
269
270 HashMap<String, Long> rsTimestampMap = new HashMap<String, Long>();
271
272 rsTimestampMap.put("rs1", 100L);
273 rsTimestampMap.put("rs2", 101L);
274 rsTimestampMap.put("rs3", 103L);
275
276 table.writeRegionServerLogTimestamp(tables, rsTimestampMap, "root");
277
278 HashMap<TableName, HashMap<String, Long>> result = table.readLogTimestampMap("root");
279
280 assertTrue(tables.size() == result.size());
281
282 for (TableName t : tables) {
283 HashMap<String, Long> rstm = result.get(t);
284 assertNotNull(rstm);
285 assertEquals(rstm.get("rs1"), new Long(100L));
286 assertEquals(rstm.get("rs2"), new Long(101L));
287 assertEquals(rstm.get("rs3"), new Long(103L));
288 }
289
290 Set<TableName> tables1 = new TreeSet<>();
291
292 tables1.add(TableName.valueOf("t3"));
293 tables1.add(TableName.valueOf("t4"));
294 tables1.add(TableName.valueOf("t5"));
295
296 HashMap<String, Long> rsTimestampMap1 = new HashMap<String, Long>();
297
298 rsTimestampMap1.put("rs1", 200L);
299 rsTimestampMap1.put("rs2", 201L);
300 rsTimestampMap1.put("rs3", 203L);
301
302 table.writeRegionServerLogTimestamp(tables1, rsTimestampMap1, "root");
303
304 result = table.readLogTimestampMap("root");
305
306 assertTrue(5 == result.size());
307
308 for (TableName t : tables) {
309 HashMap<String, Long> rstm = result.get(t);
310 assertNotNull(rstm);
311 if (t.equals(TableName.valueOf("t3")) == false) {
312 assertEquals(rstm.get("rs1"), new Long(100L));
313 assertEquals(rstm.get("rs2"), new Long(101L));
314 assertEquals(rstm.get("rs3"), new Long(103L));
315 } else {
316 assertEquals(rstm.get("rs1"), new Long(200L));
317 assertEquals(rstm.get("rs2"), new Long(201L));
318 assertEquals(rstm.get("rs3"), new Long(203L));
319 }
320 }
321
322 for (TableName t : tables1) {
323 HashMap<String, Long> rstm = result.get(t);
324 assertNotNull(rstm);
325 assertEquals(rstm.get("rs1"), new Long(200L));
326 assertEquals(rstm.get("rs2"), new Long(201L));
327 assertEquals(rstm.get("rs3"), new Long(203L));
328 }
329
330 cleanBackupTable();
331
332 }
333
334 @Test
335 public void testAddWALFiles() throws IOException {
336 List<String> files =
337 Arrays.asList("hdfs://server/WALs/srv1,101,15555/srv1,101,15555.default.1",
338 "hdfs://server/WALs/srv2,102,16666/srv2,102,16666.default.2",
339 "hdfs://server/WALs/srv3,103,17777/srv3,103,17777.default.3");
340 String newFile = "hdfs://server/WALs/srv1,101,15555/srv1,101,15555.default.5";
341
342 table.addWALFiles(files, "backup", "root");
343
344 assertTrue(table.isWALFileDeletable(files.get(0)));
345 assertTrue(table.isWALFileDeletable(files.get(1)));
346 assertTrue(table.isWALFileDeletable(files.get(2)));
347 assertFalse(table.isWALFileDeletable(newFile));
348
349 cleanBackupTable();
350 }
351
352
353
354
355
356
357 @Test
358 public void testBackupSetAddNotExists() throws IOException {
359 try (BackupSystemTable table = new BackupSystemTable(conn)) {
360
361 String[] tables = new String[] { "table1", "table2", "table3" };
362 String setName = "name";
363 table.addToBackupSet(setName, tables);
364 List<TableName> tnames = table.describeBackupSet(setName);
365 assertTrue(tnames != null);
366 assertTrue(tnames.size() == tables.length);
367 for (int i = 0; i < tnames.size(); i++) {
368 assertTrue(tnames.get(i).getNameAsString().equals(tables[i]));
369 }
370 cleanBackupTable();
371 }
372
373 }
374
375 @Test
376 public void testBackupSetAddExists() throws IOException {
377 try (BackupSystemTable table = new BackupSystemTable(conn)) {
378
379 String[] tables = new String[] { "table1", "table2", "table3" };
380 String setName = "name";
381 table.addToBackupSet(setName, tables);
382 String[] addTables = new String[] { "table4", "table5", "table6" };
383 table.addToBackupSet(setName, addTables);
384
385 List<TableName> tnames = table.describeBackupSet(setName);
386 assertTrue(tnames != null);
387 assertTrue(tnames.size() == tables.length + addTables.length);
388 for (int i = 0; i < tnames.size(); i++) {
389 assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1)));
390 }
391 cleanBackupTable();
392 }
393 }
394
395 @Test
396 public void testBackupSetAddExistsIntersects() throws IOException {
397 try (BackupSystemTable table = new BackupSystemTable(conn)) {
398
399 String[] tables = new String[] { "table1", "table2", "table3" };
400 String setName = "name";
401 table.addToBackupSet(setName, tables);
402 String[] addTables = new String[] { "table3", "table4", "table5", "table6" };
403 table.addToBackupSet(setName, addTables);
404
405 List<TableName> tnames = table.describeBackupSet(setName);
406 assertTrue(tnames != null);
407 assertTrue(tnames.size()== tables.length + addTables.length - 1);
408 for (int i = 0; i < tnames.size(); i++) {
409 assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1)));
410 }
411 cleanBackupTable();
412 }
413 }
414
415 @Test
416 public void testBackupSetRemoveSomeNotExists() throws IOException {
417 try (BackupSystemTable table = new BackupSystemTable(conn)) {
418
419 String[] tables = new String[] { "table1", "table2", "table3", "table4" };
420 String setName = "name";
421 table.addToBackupSet(setName, tables);
422 String[] removeTables = new String[] { "table4", "table5", "table6" };
423 table.removeFromBackupSet(setName, removeTables);
424
425 List<TableName> tnames = table.describeBackupSet(setName);
426 assertTrue(tnames != null);
427 assertTrue(tnames.size() == tables.length - 1);
428 for (int i = 0; i < tnames.size(); i++) {
429 assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1)));
430 }
431 cleanBackupTable();
432 }
433 }
434
435 @Test
436 public void testBackupSetRemove() throws IOException {
437 try (BackupSystemTable table = new BackupSystemTable(conn)) {
438
439 String[] tables = new String[] { "table1", "table2", "table3", "table4" };
440 String setName = "name";
441 table.addToBackupSet(setName, tables);
442 String[] removeTables = new String[] { "table4", "table3" };
443 table.removeFromBackupSet(setName, removeTables);
444
445 List<TableName> tnames = table.describeBackupSet(setName);
446 assertTrue(tnames != null);
447 assertTrue(tnames.size() == tables.length - 2);
448 for (int i = 0; i < tnames.size(); i++) {
449 assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1)));
450 }
451 cleanBackupTable();
452 }
453 }
454
455 @Test
456 public void testBackupSetDelete() throws IOException {
457 try (BackupSystemTable table = new BackupSystemTable(conn)) {
458
459 String[] tables = new String[] { "table1", "table2", "table3", "table4" };
460 String setName = "name";
461 table.addToBackupSet(setName, tables);
462 table.deleteBackupSet(setName);
463
464 List<TableName> tnames = table.describeBackupSet(setName);
465 assertTrue(tnames == null);
466 cleanBackupTable();
467 }
468 }
469
470 @Test
471 public void testBackupSetList() throws IOException {
472 try (BackupSystemTable table = new BackupSystemTable(conn)) {
473
474 String[] tables = new String[] { "table1", "table2", "table3", "table4" };
475 String setName1 = "name1";
476 String setName2 = "name2";
477 table.addToBackupSet(setName1, tables);
478 table.addToBackupSet(setName2, tables);
479
480 List<String> list = table.listBackupSets();
481
482 assertTrue(list.size() == 2);
483 assertTrue(list.get(0).equals(setName1));
484 assertTrue(list.get(1).equals(setName2));
485
486 cleanBackupTable();
487 }
488 }
489
490
491 private boolean compare(BackupInfo one, BackupInfo two) {
492 return one.getBackupId().equals(two.getBackupId()) && one.getType().equals(two.getType())
493 && one.getTargetRootDir().equals(two.getTargetRootDir())
494 && one.getStartTs() == two.getStartTs() && one.getEndTs() == two.getEndTs();
495 }
496
497 private BackupInfo createBackupContext() {
498
499 BackupInfo ctxt =
500 new BackupInfo("backup_" + System.nanoTime(), BackupType.FULL,
501 new TableName[] {
502 TableName.valueOf("t1"), TableName.valueOf("t2"), TableName.valueOf("t3") },
503 "/hbase/backup");
504 ctxt.setStartTs(System.currentTimeMillis());
505 ctxt.setEndTs(System.currentTimeMillis() + 1);
506 return ctxt;
507 }
508
509 private List<BackupInfo> createBackupContextList(int size) {
510 List<BackupInfo> list = new ArrayList<BackupInfo>();
511 for (int i = 0; i < size; i++) {
512 list.add(createBackupContext());
513 try {
514 Thread.sleep(10);
515 } catch (InterruptedException e) {
516 e.printStackTrace();
517 }
518 }
519 return list;
520 }
521
522 @AfterClass
523 public static void tearDown() throws IOException {
524 if (cluster != null) cluster.shutdown();
525 }
526 }