View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.backup;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.io.IOException;
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.Collections;
31  import java.util.HashMap;
32  import java.util.Iterator;
33  import java.util.List;
34  import java.util.Set;
35  import java.util.TreeSet;
36  
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.hbase.HBaseTestingUtility;
39  import org.apache.hadoop.hbase.MiniHBaseCluster;
40  import org.apache.hadoop.hbase.TableName;
41  import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
42  import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
43  import org.apache.hadoop.hbase.client.Admin;
44  import org.apache.hadoop.hbase.client.Connection;
45  import org.apache.hadoop.hbase.testclassification.MediumTests;
46  import org.junit.After;
47  import org.junit.AfterClass;
48  import org.junit.Before;
49  import org.junit.BeforeClass;
50  import org.junit.Test;
51  import org.junit.experimental.categories.Category;
52  
53  /**
54   * Test cases for hbase:backup API
55   *
56   */
57  @Category(MediumTests.class)
58  public class TestBackupSystemTable {
59  
60    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
61    protected static Configuration conf = UTIL.getConfiguration();
62    protected static MiniHBaseCluster cluster;
63    protected static Connection conn;
64    protected BackupSystemTable table;
65  
66    @BeforeClass
67    public static void setUp() throws Exception {
68      cluster = UTIL.startMiniCluster(); 
69      conn = UTIL.getConnection();
70      waitForSystemTable();
71    }
72    
73    static void waitForSystemTable() throws Exception
74    {
75      try(Admin admin = UTIL.getHBaseAdmin();) {
76        while (!admin.tableExists(BackupSystemTable.getTableName()) 
77            || !admin.isTableAvailable(BackupSystemTable.getTableName())) {
78          Thread.sleep(1000);
79        }      
80      }
81    }
82    
83    @Before
84    public void before() throws IOException {
85      table = new BackupSystemTable(conn);
86    }
87  
88    @After
89    public void after() {
90      if (table != null) {
91        table.close();
92      }
93  
94    }
95  
96    @Test
97    public void testUpdateReadDeleteBackupStatus() throws IOException {
98      BackupInfo ctx = createBackupContext();
99      table.updateBackupInfo(ctx);
100     BackupInfo readCtx = table.readBackupInfo(ctx.getBackupId());
101     assertTrue(compare(ctx, readCtx));
102     // try fake backup id
103     readCtx = table.readBackupInfo("fake");
104     assertNull(readCtx);
105     // delete backup context
106     table.deleteBackupInfo(ctx.getBackupId());
107     readCtx = table.readBackupInfo(ctx.getBackupId());
108     assertNull(readCtx);
109     cleanBackupTable();
110   }
111 
112   @Test
113   public void testWriteReadBackupStartCode() throws IOException {
114     Long code = 100L;
115     table.writeBackupStartCode(code, "root");
116     String readCode = table.readBackupStartCode("root");
117     assertEquals(code, new Long(Long.parseLong(readCode)));
118     cleanBackupTable();
119   }
120 
121   private void cleanBackupTable() throws IOException {
122     Admin admin = UTIL.getHBaseAdmin();
123     admin.disableTable(TableName.BACKUP_TABLE_NAME);
124     admin.truncateTable(TableName.BACKUP_TABLE_NAME, true);
125     if (admin.isTableDisabled(TableName.BACKUP_TABLE_NAME)) {
126       admin.enableTable(TableName.BACKUP_TABLE_NAME);
127     }
128   }
129 
130   @Test
131   public void testBackupHistory() throws IOException {
132     int n = 10;
133     List<BackupInfo> list = createBackupContextList(n);
134 
135     // Load data
136     for (BackupInfo bc : list) {
137       // Make sure we set right status
138       bc.setState(BackupState.COMPLETE);
139       table.updateBackupInfo(bc);
140     }
141 
142     // Reverse list for comparison
143     Collections.reverse(list);
144     ArrayList<BackupInfo> history = table.getBackupHistory();
145     assertTrue(history.size() == n);
146 
147     for (int i = 0; i < n; i++) {
148       BackupInfo ctx = list.get(i);
149       BackupInfo data = history.get(i);
150       assertTrue(compare(ctx, data));
151     }
152 
153     cleanBackupTable();
154 
155   }
156 
157   @Test
158   public void testBackupDelete() throws IOException {
159     
160     try (BackupSystemTable table = new BackupSystemTable(conn)) {
161 
162       int n = 10;
163       List<BackupInfo> list = createBackupContextList(n);
164 
165       // Load data
166       for (BackupInfo bc : list) {
167         // Make sure we set right status
168         bc.setState(BackupState.COMPLETE);
169         table.updateBackupInfo(bc);
170       }
171 
172       // Verify exists
173       for (BackupInfo bc : list) {
174         assertNotNull(table.readBackupInfo(bc.getBackupId()));
175       }
176 
177       // Delete all
178       for (BackupInfo bc : list) {
179         table.deleteBackupInfo(bc.getBackupId());
180       }
181 
182       // Verify do not exists
183       for (BackupInfo bc : list) {
184         assertNull(table.readBackupInfo(bc.getBackupId()));
185       }
186 
187       cleanBackupTable();
188     }
189 
190   }
191 
192   
193   
194   @Test
195   public void testRegionServerLastLogRollResults() throws IOException {
196     String[] servers = new String[] { "server1", "server2", "server3" };
197     Long[] timestamps = new Long[] { 100L, 102L, 107L };
198 
199     for (int i = 0; i < servers.length; i++) {
200       table.writeRegionServerLastLogRollResult(servers[i], timestamps[i], "root");
201     }
202 
203     HashMap<String, Long> result = table.readRegionServerLastLogRollResult("root");
204     assertTrue(servers.length == result.size());
205     Set<String> keys = result.keySet();
206     String[] keysAsArray = new String[keys.size()];
207     keys.toArray(keysAsArray);
208     Arrays.sort(keysAsArray);
209 
210     for (int i = 0; i < keysAsArray.length; i++) {
211       assertEquals(keysAsArray[i], servers[i]);
212       Long ts1 = timestamps[i];
213       Long ts2 = result.get(keysAsArray[i]);
214       assertEquals(ts1, ts2);
215     }
216 
217     cleanBackupTable();
218   }
219 
220   @Test
221   public void testIncrementalBackupTableSet() throws IOException {
222     TreeSet<TableName> tables1 = new TreeSet<>();
223 
224     tables1.add(TableName.valueOf("t1"));
225     tables1.add(TableName.valueOf("t2"));
226     tables1.add(TableName.valueOf("t3"));
227 
228     TreeSet<TableName> tables2 = new TreeSet<>();
229 
230     tables2.add(TableName.valueOf("t3"));
231     tables2.add(TableName.valueOf("t4"));
232     tables2.add(TableName.valueOf("t5"));
233 
234     table.addIncrementalBackupTableSet(tables1, "root");
235     BackupSystemTable table = new BackupSystemTable(conn);
236     TreeSet<TableName> res1 = (TreeSet<TableName>) 
237         table.getIncrementalBackupTableSet("root");
238     assertTrue(tables1.size() == res1.size());
239     Iterator<TableName> desc1 = tables1.descendingIterator();
240     Iterator<TableName> desc2 = res1.descendingIterator();
241     while (desc1.hasNext()) {
242       assertEquals(desc1.next(), desc2.next());
243     }
244 
245     table.addIncrementalBackupTableSet(tables2, "root");
246     TreeSet<TableName> res2 = (TreeSet<TableName>) 
247         table.getIncrementalBackupTableSet("root");
248     assertTrue((tables2.size() + tables1.size() - 1) == res2.size());
249 
250     tables1.addAll(tables2);
251 
252     desc1 = tables1.descendingIterator();
253     desc2 = res2.descendingIterator();
254 
255     while (desc1.hasNext()) {
256       assertEquals(desc1.next(), desc2.next());
257     }
258     cleanBackupTable();
259 
260   }
261 
262   @Test
263   public void testRegionServerLogTimestampMap() throws IOException {
264     TreeSet<TableName> tables = new TreeSet<>();
265 
266     tables.add(TableName.valueOf("t1"));
267     tables.add(TableName.valueOf("t2"));
268     tables.add(TableName.valueOf("t3"));
269 
270     HashMap<String, Long> rsTimestampMap = new HashMap<String, Long>();
271 
272     rsTimestampMap.put("rs1", 100L);
273     rsTimestampMap.put("rs2", 101L);
274     rsTimestampMap.put("rs3", 103L);
275 
276     table.writeRegionServerLogTimestamp(tables, rsTimestampMap, "root");
277 
278     HashMap<TableName, HashMap<String, Long>> result = table.readLogTimestampMap("root");
279 
280     assertTrue(tables.size() == result.size());
281 
282     for (TableName t : tables) {
283       HashMap<String, Long> rstm = result.get(t);
284       assertNotNull(rstm);
285       assertEquals(rstm.get("rs1"), new Long(100L));
286       assertEquals(rstm.get("rs2"), new Long(101L));
287       assertEquals(rstm.get("rs3"), new Long(103L));
288     }
289 
290     Set<TableName> tables1 = new TreeSet<>();
291 
292     tables1.add(TableName.valueOf("t3"));
293     tables1.add(TableName.valueOf("t4"));
294     tables1.add(TableName.valueOf("t5"));
295 
296     HashMap<String, Long> rsTimestampMap1 = new HashMap<String, Long>();
297 
298     rsTimestampMap1.put("rs1", 200L);
299     rsTimestampMap1.put("rs2", 201L);
300     rsTimestampMap1.put("rs3", 203L);
301 
302     table.writeRegionServerLogTimestamp(tables1, rsTimestampMap1, "root");
303 
304     result = table.readLogTimestampMap("root");
305 
306     assertTrue(5 == result.size());
307 
308     for (TableName t : tables) {
309       HashMap<String, Long> rstm = result.get(t);
310       assertNotNull(rstm);
311       if (t.equals(TableName.valueOf("t3")) == false) {
312         assertEquals(rstm.get("rs1"), new Long(100L));
313         assertEquals(rstm.get("rs2"), new Long(101L));
314         assertEquals(rstm.get("rs3"), new Long(103L));
315       } else {
316         assertEquals(rstm.get("rs1"), new Long(200L));
317         assertEquals(rstm.get("rs2"), new Long(201L));
318         assertEquals(rstm.get("rs3"), new Long(203L));
319       }
320     }
321 
322     for (TableName t : tables1) {
323       HashMap<String, Long> rstm = result.get(t);
324       assertNotNull(rstm);
325       assertEquals(rstm.get("rs1"), new Long(200L));
326       assertEquals(rstm.get("rs2"), new Long(201L));
327       assertEquals(rstm.get("rs3"), new Long(203L));
328     }
329 
330     cleanBackupTable();
331 
332   }
333 
334   @Test
335   public void testAddWALFiles() throws IOException {
336     List<String> files =
337         Arrays.asList("hdfs://server/WALs/srv1,101,15555/srv1,101,15555.default.1",
338           "hdfs://server/WALs/srv2,102,16666/srv2,102,16666.default.2",
339             "hdfs://server/WALs/srv3,103,17777/srv3,103,17777.default.3");
340     String newFile = "hdfs://server/WALs/srv1,101,15555/srv1,101,15555.default.5";
341 
342     table.addWALFiles(files, "backup", "root");
343 
344     assertTrue(table.isWALFileDeletable(files.get(0)));
345     assertTrue(table.isWALFileDeletable(files.get(1)));
346     assertTrue(table.isWALFileDeletable(files.get(2)));
347     assertFalse(table.isWALFileDeletable(newFile));
348 
349     cleanBackupTable();
350   }
351 
352   
353   /**
354    * Backup set tests
355    */
356 
357   @Test
358   public void testBackupSetAddNotExists() throws IOException {
359     try (BackupSystemTable table = new BackupSystemTable(conn)) {
360 
361       String[] tables = new String[] { "table1", "table2", "table3" };
362       String setName = "name";
363       table.addToBackupSet(setName, tables);
364       List<TableName> tnames = table.describeBackupSet(setName);
365       assertTrue(tnames != null);
366       assertTrue(tnames.size() == tables.length);
367       for (int i = 0; i < tnames.size(); i++) {
368         assertTrue(tnames.get(i).getNameAsString().equals(tables[i]));
369       }
370       cleanBackupTable();
371     }
372 
373   }
374 
375   @Test
376   public void testBackupSetAddExists() throws IOException {
377     try (BackupSystemTable table = new BackupSystemTable(conn)) {
378 
379       String[] tables = new String[] { "table1", "table2", "table3" };
380       String setName = "name";
381       table.addToBackupSet(setName, tables);
382       String[] addTables = new String[] { "table4", "table5", "table6" };
383       table.addToBackupSet(setName, addTables);
384 
385       List<TableName> tnames = table.describeBackupSet(setName);
386       assertTrue(tnames != null);
387       assertTrue(tnames.size() == tables.length + addTables.length);
388       for (int i = 0; i < tnames.size(); i++) {
389         assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1)));
390       }
391       cleanBackupTable();
392     }
393   }
394 
395   @Test
396   public void testBackupSetAddExistsIntersects() throws IOException {
397     try (BackupSystemTable table = new BackupSystemTable(conn)) {
398 
399       String[] tables = new String[] { "table1", "table2", "table3" };
400       String setName = "name";
401       table.addToBackupSet(setName, tables);
402       String[] addTables = new String[] { "table3", "table4", "table5", "table6" };
403       table.addToBackupSet(setName, addTables);
404 
405       List<TableName> tnames = table.describeBackupSet(setName);
406       assertTrue(tnames != null);
407       assertTrue(tnames.size()== tables.length + addTables.length - 1);
408       for (int i = 0; i < tnames.size(); i++) {
409         assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1)));
410       }
411       cleanBackupTable();
412     } 
413   }
414 
415   @Test
416   public void testBackupSetRemoveSomeNotExists() throws IOException {
417     try (BackupSystemTable table = new BackupSystemTable(conn)) {
418 
419       String[] tables = new String[] { "table1", "table2", "table3", "table4" };
420       String setName = "name";
421       table.addToBackupSet(setName, tables);
422       String[] removeTables = new String[] { "table4", "table5", "table6" };
423       table.removeFromBackupSet(setName, removeTables);
424 
425       List<TableName> tnames = table.describeBackupSet(setName);
426       assertTrue(tnames != null);
427       assertTrue(tnames.size() == tables.length - 1);
428       for (int i = 0; i < tnames.size(); i++) {
429         assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1)));
430       }
431       cleanBackupTable();
432     }
433   }
434 
435   @Test
436   public void testBackupSetRemove() throws IOException {
437     try (BackupSystemTable table = new BackupSystemTable(conn)) {
438 
439       String[] tables = new String[] { "table1", "table2", "table3", "table4" };
440       String setName = "name";
441       table.addToBackupSet(setName, tables);
442       String[] removeTables = new String[] { "table4", "table3" };
443       table.removeFromBackupSet(setName, removeTables);
444 
445       List<TableName> tnames = table.describeBackupSet(setName);
446       assertTrue(tnames != null);
447       assertTrue(tnames.size() == tables.length - 2);
448       for (int i = 0; i < tnames.size(); i++) {
449         assertTrue(tnames.get(i).getNameAsString().equals("table" + (i + 1)));
450       }
451       cleanBackupTable();
452     }
453   }
454 
455   @Test
456   public void testBackupSetDelete() throws IOException {
457     try (BackupSystemTable table = new BackupSystemTable(conn)) {
458 
459       String[] tables = new String[] { "table1", "table2", "table3", "table4" };
460       String setName = "name";
461       table.addToBackupSet(setName, tables);
462       table.deleteBackupSet(setName);
463 
464       List<TableName> tnames = table.describeBackupSet(setName);
465       assertTrue(tnames == null);
466       cleanBackupTable();
467     }
468   }
469 
470   @Test
471   public void testBackupSetList() throws IOException {
472     try (BackupSystemTable table = new BackupSystemTable(conn)) {
473 
474       String[] tables = new String[] { "table1", "table2", "table3", "table4" };
475       String setName1 = "name1";
476       String setName2 = "name2";
477       table.addToBackupSet(setName1, tables);
478       table.addToBackupSet(setName2, tables);
479 
480       List<String> list = table.listBackupSets();
481 
482       assertTrue(list.size() == 2);
483       assertTrue(list.get(0).equals(setName1));
484       assertTrue(list.get(1).equals(setName2));
485 
486       cleanBackupTable();
487     }
488   }
489    
490 
491   private boolean compare(BackupInfo one, BackupInfo two) {
492     return one.getBackupId().equals(two.getBackupId()) && one.getType().equals(two.getType())
493         && one.getTargetRootDir().equals(two.getTargetRootDir())
494         && one.getStartTs() == two.getStartTs() && one.getEndTs() == two.getEndTs();
495   }
496 
497   private BackupInfo createBackupContext() {
498 
499     BackupInfo ctxt =
500         new BackupInfo("backup_" + System.nanoTime(), BackupType.FULL,
501           new TableName[] {
502               TableName.valueOf("t1"), TableName.valueOf("t2"), TableName.valueOf("t3") },
503           "/hbase/backup");
504     ctxt.setStartTs(System.currentTimeMillis());
505     ctxt.setEndTs(System.currentTimeMillis() + 1);
506     return ctxt;
507   }
508 
509   private List<BackupInfo> createBackupContextList(int size) {
510     List<BackupInfo> list = new ArrayList<BackupInfo>();
511     for (int i = 0; i < size; i++) {
512       list.add(createBackupContext());
513       try {
514         Thread.sleep(10);
515       } catch (InterruptedException e) {
516         e.printStackTrace();
517       }
518     }
519     return list;
520   }
521 
522   @AfterClass
523   public static void tearDown() throws IOException {
524     if (cluster != null) cluster.shutdown();
525   }
526 }