1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.client;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.fail;
23
24 import java.io.IOException;
25 import java.util.HashSet;
26 import java.util.Set;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.hbase.TableName;
33 import org.apache.hadoop.hbase.HBaseTestingUtility;
34 import org.apache.hadoop.hbase.HColumnDescriptor;
35 import org.apache.hadoop.hbase.HConstants;
36 import org.apache.hadoop.hbase.HTableDescriptor;
37 import org.apache.hadoop.hbase.testclassification.LargeTests;
38 import org.apache.hadoop.hbase.master.MasterFileSystem;
39 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
40 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
41 import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
42 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.FSUtils;
45 import org.junit.After;
46 import org.junit.AfterClass;
47 import org.junit.Before;
48 import org.junit.BeforeClass;
49 import org.junit.Test;
50 import org.junit.experimental.categories.Category;
51
52
53
54
55 @Category(LargeTests.class)
56 public class TestRestoreSnapshotFromClient {
57 final Log LOG = LogFactory.getLog(getClass());
58
59 protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
60
61 protected final byte[] FAMILY = Bytes.toBytes("cf");
62 protected final byte[] TEST_FAMILY2 = Bytes.toBytes("cf2");
63
64 protected TableName tableName;
65 private byte[] emptySnapshot;
66 private byte[] snapshotName0;
67 private byte[] snapshotName1;
68 private byte[] snapshotName2;
69 private int snapshot0Rows;
70 private int snapshot1Rows;
71 private Admin admin;
72
73 @BeforeClass
74 public static void setupCluster() throws Exception {
75 setupConf(TEST_UTIL.getConfiguration());
76 TEST_UTIL.startMiniCluster(3);
77 }
78
79 protected static void setupConf(Configuration conf) {
80 TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
81 TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
82 TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
83 TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
84 TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
85 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
86 TEST_UTIL.getConfiguration().setBoolean(
87 "hbase.master.enabletable.roundrobin", true);
88 }
89
90 @AfterClass
91 public static void tearDownAfterClass() throws Exception {
92 TEST_UTIL.shutdownMiniCluster();
93 }
94
95
96
97
98
99
100 @Before
101 public void setup() throws Exception {
102 this.admin = TEST_UTIL.getHBaseAdmin();
103
104 long tid = System.currentTimeMillis();
105 tableName =
106 TableName.valueOf("testtb-" + tid);
107 emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid);
108 snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
109 snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
110 snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
111
112
113 createTable();
114 admin.disableTable(tableName);
115
116
117 admin.snapshot(emptySnapshot, tableName);
118
119
120 admin.enableTable(tableName);
121 SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
122 try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
123 snapshot0Rows = countRows(table);
124 }
125 admin.disableTable(tableName);
126
127
128 admin.snapshot(snapshotName0, tableName);
129
130
131 admin.enableTable(tableName);
132 SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
133 try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
134 snapshot1Rows = countRows(table);
135 }
136 }
137
138 protected void createTable() throws Exception {
139 SnapshotTestingUtils.createTable(TEST_UTIL, tableName, getNumReplicas(), FAMILY);
140 }
141
142 @After
143 public void tearDown() throws Exception {
144 TEST_UTIL.deleteTable(tableName);
145 SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
146 SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
147 }
148
149 @Test
150 public void testRestoreSnapshot() throws IOException {
151 verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
152 admin.disableTable(tableName);
153 admin.snapshot(snapshotName1, tableName);
154
155 admin.restoreSnapshot(snapshotName0);
156 admin.enableTable(tableName);
157 verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
158 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
159
160
161 admin.disableTable(tableName);
162 admin.restoreSnapshot(emptySnapshot);
163 admin.enableTable(tableName);
164 verifyRowCount(TEST_UTIL, tableName, 0);
165 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
166
167
168 admin.disableTable(tableName);
169 admin.restoreSnapshot(snapshotName1);
170 admin.enableTable(tableName);
171 verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
172 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
173
174
175 TEST_UTIL.deleteTable(tableName);
176 admin.restoreSnapshot(snapshotName1);
177 verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
178 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
179 }
180
181 protected int getNumReplicas() {
182 return 1;
183 }
184
185 protected HColumnDescriptor getTestRestoreSchemaChangeHCD() {
186 return new HColumnDescriptor(TEST_FAMILY2);
187 }
188
189 @Test
190 public void testRestoreSchemaChange() throws Exception {
191 HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
192
193
194 admin.disableTable(tableName);
195 admin.addColumn(tableName, getTestRestoreSchemaChangeHCD());
196 admin.enableTable(tableName);
197 assertEquals(2, table.getTableDescriptor().getFamilies().size());
198 HTableDescriptor htd = admin.getTableDescriptor(tableName);
199 assertEquals(2, htd.getFamilies().size());
200 SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, TEST_FAMILY2);
201 long snapshot2Rows = snapshot1Rows + 500;
202 assertEquals(snapshot2Rows, countRows(table));
203 assertEquals(500, countRows(table, TEST_FAMILY2));
204 Set<String> fsFamilies = getFamiliesFromFS(tableName);
205 assertEquals(2, fsFamilies.size());
206
207
208 admin.disableTable(tableName);
209 admin.snapshot(snapshotName2, tableName);
210
211
212 admin.restoreSnapshot(snapshotName0);
213 admin.enableTable(tableName);
214 assertEquals(1, table.getTableDescriptor().getFamilies().size());
215 try {
216 countRows(table, TEST_FAMILY2);
217 fail("family '" + Bytes.toString(TEST_FAMILY2) + "' should not exists");
218 } catch (NoSuchColumnFamilyException e) {
219
220 }
221 assertEquals(snapshot0Rows, countRows(table));
222 htd = admin.getTableDescriptor(tableName);
223 assertEquals(1, htd.getFamilies().size());
224 fsFamilies = getFamiliesFromFS(tableName);
225 assertEquals(1, fsFamilies.size());
226
227
228 admin.disableTable(tableName);
229 admin.restoreSnapshot(snapshotName2);
230 admin.enableTable(tableName);
231 htd = admin.getTableDescriptor(tableName);
232 assertEquals(2, htd.getFamilies().size());
233 assertEquals(2, table.getTableDescriptor().getFamilies().size());
234 assertEquals(500, countRows(table, TEST_FAMILY2));
235 assertEquals(snapshot2Rows, countRows(table));
236 fsFamilies = getFamiliesFromFS(tableName);
237 assertEquals(2, fsFamilies.size());
238 table.close();
239 }
240
241 @Test
242 public void testCloneSnapshotOfCloned() throws IOException, InterruptedException {
243 TableName clonedTableName =
244 TableName.valueOf("clonedtb-" + System.currentTimeMillis());
245 admin.cloneSnapshot(snapshotName0, clonedTableName);
246 verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
247 SnapshotTestingUtils.verifyReplicasCameOnline(clonedTableName, admin, getNumReplicas());
248 admin.disableTable(clonedTableName);
249 admin.snapshot(snapshotName2, clonedTableName);
250 TEST_UTIL.deleteTable(clonedTableName);
251 waitCleanerRun();
252
253 admin.cloneSnapshot(snapshotName2, clonedTableName);
254 verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
255 SnapshotTestingUtils.verifyReplicasCameOnline(clonedTableName, admin, getNumReplicas());
256 TEST_UTIL.deleteTable(clonedTableName);
257 }
258
259 @Test
260 public void testCloneAndRestoreSnapshot() throws IOException, InterruptedException {
261 TEST_UTIL.deleteTable(tableName);
262 waitCleanerRun();
263
264 admin.cloneSnapshot(snapshotName0, tableName);
265 verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
266 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
267 waitCleanerRun();
268
269 admin.disableTable(tableName);
270 admin.restoreSnapshot(snapshotName0);
271 admin.enableTable(tableName);
272 verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
273 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
274 }
275
276 @Test
277 public void testCorruptedSnapshot() throws IOException, InterruptedException {
278 SnapshotTestingUtils.corruptSnapshot(TEST_UTIL, Bytes.toString(snapshotName0));
279 TableName cloneName = TableName.valueOf("corruptedClone-" + System.currentTimeMillis());
280 try {
281 admin.cloneSnapshot(snapshotName0, cloneName);
282 fail("Expected CorruptedSnapshotException, got succeeded cloneSnapshot()");
283 } catch (CorruptedSnapshotException e) {
284
285
286 assertFalse(admin.tableExists(cloneName));
287 } catch (Exception e) {
288 fail("Expected CorruptedSnapshotException got: " + e);
289 }
290 }
291
292
293
294
295 private void waitCleanerRun() throws InterruptedException {
296 TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
297 }
298
299 private Set<String> getFamiliesFromFS(final TableName tableName) throws IOException {
300 MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
301 Set<String> families = new HashSet<String>();
302 Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
303 for (Path regionDir: FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) {
304 for (Path familyDir: FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) {
305 families.add(familyDir.getName());
306 }
307 }
308 return families;
309 }
310
311 protected void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
312 long expectedRows) throws IOException {
313 SnapshotTestingUtils.verifyRowCount(util, tableName, expectedRows);
314 }
315
316 protected int countRows(final Table table, final byte[]... families) throws IOException {
317 return TEST_UTIL.countRows(table, families);
318 }
319 }