View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.fail;
23  
24  import java.io.IOException;
25  import java.util.HashSet;
26  import java.util.Set;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.TableName;
33  import org.apache.hadoop.hbase.HBaseTestingUtility;
34  import org.apache.hadoop.hbase.HColumnDescriptor;
35  import org.apache.hadoop.hbase.HConstants;
36  import org.apache.hadoop.hbase.HTableDescriptor;
37  import org.apache.hadoop.hbase.testclassification.LargeTests;
38  import org.apache.hadoop.hbase.master.MasterFileSystem;
39  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
40  import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
41  import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
42  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.FSUtils;
45  import org.junit.After;
46  import org.junit.AfterClass;
47  import org.junit.Before;
48  import org.junit.BeforeClass;
49  import org.junit.Test;
50  import org.junit.experimental.categories.Category;
51  
52  /**
53   * Test restore snapshots from the client
54   */
55  @Category(LargeTests.class)
56  public class TestRestoreSnapshotFromClient {
57    final Log LOG = LogFactory.getLog(getClass());
58  
59    protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
60  
61    protected final byte[] FAMILY = Bytes.toBytes("cf");
62    protected final byte[] TEST_FAMILY2 = Bytes.toBytes("cf2");
63  
64    protected TableName tableName;
65    private byte[] emptySnapshot;
66    private byte[] snapshotName0;
67    private byte[] snapshotName1;
68    private byte[] snapshotName2;
69    private int snapshot0Rows;
70    private int snapshot1Rows;
71    private Admin admin;
72  
73    @BeforeClass
74    public static void setupCluster() throws Exception {
75      setupConf(TEST_UTIL.getConfiguration());
76      TEST_UTIL.startMiniCluster(3);
77    }
78  
79    protected static void setupConf(Configuration conf) {
80      TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
81      TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
82      TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
83      TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
84      TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
85      TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
86      TEST_UTIL.getConfiguration().setBoolean(
87          "hbase.master.enabletable.roundrobin", true);
88    }
89  
90    @AfterClass
91    public static void tearDownAfterClass() throws Exception {
92      TEST_UTIL.shutdownMiniCluster();
93    }
94  
95    /**
96     * Initialize the tests with a table filled with some data
97     * and two snapshots (snapshotName0, snapshotName1) of different states.
98     * The tableName, snapshotNames and the number of rows in the snapshot are initialized.
99     */
100   @Before
101   public void setup() throws Exception {
102     this.admin = TEST_UTIL.getHBaseAdmin();
103 
104     long tid = System.currentTimeMillis();
105     tableName =
106         TableName.valueOf("testtb-" + tid);
107     emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid);
108     snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
109     snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
110     snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
111 
112     // create Table and disable it
113     createTable();
114     admin.disableTable(tableName);
115 
116     // take an empty snapshot
117     admin.snapshot(emptySnapshot, tableName);
118 
119     // enable table and insert data
120     admin.enableTable(tableName);
121     SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
122     try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
123       snapshot0Rows = countRows(table);
124     }
125     admin.disableTable(tableName);
126 
127     // take a snapshot
128     admin.snapshot(snapshotName0, tableName);
129 
130     // enable table and insert more data
131     admin.enableTable(tableName);
132     SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
133     try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
134       snapshot1Rows = countRows(table);
135     }
136   }
137 
138   protected void createTable() throws Exception {
139     SnapshotTestingUtils.createTable(TEST_UTIL, tableName, getNumReplicas(), FAMILY);
140   }
141 
142   @After
143   public void tearDown() throws Exception {
144     TEST_UTIL.deleteTable(tableName);
145     SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
146     SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
147   }
148 
149   @Test
150   public void testRestoreSnapshot() throws IOException {
151     verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
152     admin.disableTable(tableName);
153     admin.snapshot(snapshotName1, tableName);
154     // Restore from snapshot-0
155     admin.restoreSnapshot(snapshotName0);
156     admin.enableTable(tableName);
157     verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
158     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
159 
160     // Restore from emptySnapshot
161     admin.disableTable(tableName);
162     admin.restoreSnapshot(emptySnapshot);
163     admin.enableTable(tableName);
164     verifyRowCount(TEST_UTIL, tableName, 0);
165     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
166 
167     // Restore from snapshot-1
168     admin.disableTable(tableName);
169     admin.restoreSnapshot(snapshotName1);
170     admin.enableTable(tableName);
171     verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
172     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
173 
174     // Restore from snapshot-1
175     TEST_UTIL.deleteTable(tableName);
176     admin.restoreSnapshot(snapshotName1);
177     verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
178     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
179   }
180 
181   protected int getNumReplicas() {
182     return 1;
183   }
184 
185   protected HColumnDescriptor getTestRestoreSchemaChangeHCD() {
186     return new HColumnDescriptor(TEST_FAMILY2);
187   }
188 
189   @Test
190   public void testRestoreSchemaChange() throws Exception {
191     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
192 
193     // Add one column family and put some data in it
194     admin.disableTable(tableName);
195     admin.addColumn(tableName, getTestRestoreSchemaChangeHCD());
196     admin.enableTable(tableName);
197     assertEquals(2, table.getTableDescriptor().getFamilies().size());
198     HTableDescriptor htd = admin.getTableDescriptor(tableName);
199     assertEquals(2, htd.getFamilies().size());
200     SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, TEST_FAMILY2);
201     long snapshot2Rows = snapshot1Rows + 500;
202     assertEquals(snapshot2Rows, countRows(table));
203     assertEquals(500, countRows(table, TEST_FAMILY2));
204     Set<String> fsFamilies = getFamiliesFromFS(tableName);
205     assertEquals(2, fsFamilies.size());
206 
207     // Take a snapshot
208     admin.disableTable(tableName);
209     admin.snapshot(snapshotName2, tableName);
210 
211     // Restore the snapshot (without the cf)
212     admin.restoreSnapshot(snapshotName0);
213     admin.enableTable(tableName);
214     assertEquals(1, table.getTableDescriptor().getFamilies().size());
215     try {
216       countRows(table, TEST_FAMILY2);
217       fail("family '" + Bytes.toString(TEST_FAMILY2) + "' should not exists");
218     } catch (NoSuchColumnFamilyException e) {
219       // expected
220     }
221     assertEquals(snapshot0Rows, countRows(table));
222     htd = admin.getTableDescriptor(tableName);
223     assertEquals(1, htd.getFamilies().size());
224     fsFamilies = getFamiliesFromFS(tableName);
225     assertEquals(1, fsFamilies.size());
226 
227     // Restore back the snapshot (with the cf)
228     admin.disableTable(tableName);
229     admin.restoreSnapshot(snapshotName2);
230     admin.enableTable(tableName);
231     htd = admin.getTableDescriptor(tableName);
232     assertEquals(2, htd.getFamilies().size());
233     assertEquals(2, table.getTableDescriptor().getFamilies().size());
234     assertEquals(500, countRows(table, TEST_FAMILY2));
235     assertEquals(snapshot2Rows, countRows(table));
236     fsFamilies = getFamiliesFromFS(tableName);
237     assertEquals(2, fsFamilies.size());
238     table.close();
239   }
240 
241   @Test
242   public void testCloneSnapshotOfCloned() throws IOException, InterruptedException {
243     TableName clonedTableName =
244         TableName.valueOf("clonedtb-" + System.currentTimeMillis());
245     admin.cloneSnapshot(snapshotName0, clonedTableName);
246     verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
247     SnapshotTestingUtils.verifyReplicasCameOnline(clonedTableName, admin, getNumReplicas());
248     admin.disableTable(clonedTableName);
249     admin.snapshot(snapshotName2, clonedTableName);
250     TEST_UTIL.deleteTable(clonedTableName);
251     waitCleanerRun();
252 
253     admin.cloneSnapshot(snapshotName2, clonedTableName);
254     verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
255     SnapshotTestingUtils.verifyReplicasCameOnline(clonedTableName, admin, getNumReplicas());
256     TEST_UTIL.deleteTable(clonedTableName);
257   }
258 
259   @Test
260   public void testCloneAndRestoreSnapshot() throws IOException, InterruptedException {
261     TEST_UTIL.deleteTable(tableName);
262     waitCleanerRun();
263 
264     admin.cloneSnapshot(snapshotName0, tableName);
265     verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
266     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
267     waitCleanerRun();
268 
269     admin.disableTable(tableName);
270     admin.restoreSnapshot(snapshotName0);
271     admin.enableTable(tableName);
272     verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
273     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
274   }
275 
276   @Test
277   public void testCorruptedSnapshot() throws IOException, InterruptedException {
278     SnapshotTestingUtils.corruptSnapshot(TEST_UTIL, Bytes.toString(snapshotName0));
279     TableName cloneName = TableName.valueOf("corruptedClone-" + System.currentTimeMillis());
280     try {
281       admin.cloneSnapshot(snapshotName0, cloneName);
282       fail("Expected CorruptedSnapshotException, got succeeded cloneSnapshot()");
283     } catch (CorruptedSnapshotException e) {
284       // Got the expected corruption exception.
285       // check for no references of the cloned table.
286       assertFalse(admin.tableExists(cloneName));
287     } catch (Exception e) {
288       fail("Expected CorruptedSnapshotException got: " + e);
289     }
290   }
291 
292   // ==========================================================================
293   //  Helpers
294   // ==========================================================================
295   private void waitCleanerRun() throws InterruptedException {
296     TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
297   }
298 
299   private Set<String> getFamiliesFromFS(final TableName tableName) throws IOException {
300     MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
301     Set<String> families = new HashSet<String>();
302     Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
303     for (Path regionDir: FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) {
304       for (Path familyDir: FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) {
305         families.add(familyDir.getName());
306       }
307     }
308     return families;
309   }
310 
311   protected void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
312       long expectedRows) throws IOException {
313     SnapshotTestingUtils.verifyRowCount(util, tableName, expectedRows);
314   }
315 
316   protected int countRows(final Table table, final byte[]... families) throws IOException {
317     return TEST_UTIL.countRows(table, families);
318   }
319 }