1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.IOException;
26  import java.net.URI;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.List;
30  import java.util.HashSet;
31  import java.util.Set;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FileSystem;
38  import org.apache.hadoop.fs.FileStatus;
39  import org.apache.hadoop.fs.FileUtil;
40  import org.apache.hadoop.fs.FSDataOutputStream;
41  import org.apache.hadoop.fs.Path;
42  import org.apache.hadoop.hbase.HBaseTestingUtility;
43  import org.apache.hadoop.hbase.HColumnDescriptor;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.HRegionInfo;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.KeyValue;
48  import org.apache.hadoop.hbase.MediumTests;
49  import org.apache.hadoop.hbase.MiniHBaseCluster;
50  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
51  import org.apache.hadoop.hbase.client.HBaseAdmin;
52  import org.apache.hadoop.hbase.client.HTable;
53  import org.apache.hadoop.hbase.util.Bytes;
54  import org.apache.hadoop.hbase.util.FSUtils;
55  import org.apache.hadoop.hbase.util.Pair;
56  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
57  import org.apache.hadoop.hbase.regionserver.HRegion;
58  import org.apache.hadoop.hbase.snapshot.ExportSnapshot;
59  import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
60  import org.apache.hadoop.mapreduce.Job;
61  import org.junit.After;
62  import org.junit.AfterClass;
63  import org.junit.Before;
64  import org.junit.BeforeClass;
65  import org.junit.Test;
66  import org.junit.experimental.categories.Category;
67  
68  /**
69   * Test Export Snapshot Tool
70   */
71  @Category(MediumTests.class)
72  public class TestExportSnapshot {
73    private final Log LOG = LogFactory.getLog(getClass());
74  
75    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
76  
77    private final static byte[] FAMILY = Bytes.toBytes("cf");
78  
79    private byte[] snapshotName;
80    private byte[] tableName;
81    private HBaseAdmin admin;
82  
83    @BeforeClass
84    public static void setUpBeforeClass() throws Exception {
85      TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
86      TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
87      TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
88      TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
89      TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
90      TEST_UTIL.startMiniCluster(3);
91    }
92  
93    @AfterClass
94    public static void tearDownAfterClass() throws Exception {
95      TEST_UTIL.shutdownMiniCluster();
96    }
97  
98    /**
99     * Create a table and take a snapshot of the table used by the export test.
100    */
101   @Before
102   public void setUp() throws Exception {
103     this.admin = TEST_UTIL.getHBaseAdmin();
104 
105     long tid = System.currentTimeMillis();
106     tableName = Bytes.toBytes("testtb-" + tid);
107     snapshotName = Bytes.toBytes("snaptb0-" + tid);
108 
109     // create Table
110     HTableDescriptor htd = new HTableDescriptor(tableName);
111     htd.addFamily(new HColumnDescriptor(FAMILY));
112     admin.createTable(htd, null);
113     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
114     TEST_UTIL.loadTable(table, FAMILY);
115 
116     // take a snapshot
117     admin.snapshot(snapshotName, tableName);
118   }
119 
120   @After
121   public void tearDown() throws Exception {
122     this.admin.close();
123   }
124 
125   /**
126    * Verfy the result of getBalanceSplits() method.
127    * The result are groups of files, used as input list for the "export" mappers.
128    * All the groups should have similar amount of data.
129    *
130    * The input list is a pair of file path and length.
131    * The getBalanceSplits() function sort it by length,
132    * and assign to each group a file, going back and forth through the groups.
133    */
134   @Test
135   public void testBalanceSplit() throws Exception {
136     // Create a list of files
137     List<Pair<Path, Long>> files = new ArrayList<Pair<Path, Long>>();
138     for (long i = 0; i <= 20; i++) {
139       files.add(new Pair<Path, Long>(new Path("file-" + i), i));
140     }
141 
142     // Create 5 groups (total size 210)
143     //    group 0: 20, 11, 10,  1 (total size: 42)
144     //    group 1: 19, 12,  9,  2 (total size: 42)
145     //    group 2: 18, 13,  8,  3 (total size: 42)
146     //    group 3: 17, 12,  7,  4 (total size: 42)
147     //    group 4: 16, 11,  6,  5 (total size: 42)
148     List<List<Path>> splits = ExportSnapshot.getBalancedSplits(files, 5);
149     assertEquals(5, splits.size());
150     assertEquals(Arrays.asList(new Path("file-20"), new Path("file-11"),
151       new Path("file-10"), new Path("file-1"), new Path("file-0")), splits.get(0));
152     assertEquals(Arrays.asList(new Path("file-19"), new Path("file-12"),
153       new Path("file-9"), new Path("file-2")), splits.get(1));
154     assertEquals(Arrays.asList(new Path("file-18"), new Path("file-13"),
155       new Path("file-8"), new Path("file-3")), splits.get(2));
156     assertEquals(Arrays.asList(new Path("file-17"), new Path("file-14"),
157       new Path("file-7"), new Path("file-4")), splits.get(3));
158     assertEquals(Arrays.asList(new Path("file-16"), new Path("file-15"),
159       new Path("file-6"), new Path("file-5")), splits.get(4));
160   }
161 
162   /**
163    * Verify if exported snapshot and copied files matches the original one.
164    */
165   @Test
166   public void testExportFileSystemState() throws Exception {
167     testExportFileSystemState(tableName, snapshotName, 2);
168   }
169   
170   public void testExportFileSystemState(final byte[] tableName, final byte[] snapshotName,
171       int filesExpected) throws Exception {
172     Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
173     URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri();
174     FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration());
175     copyDir = copyDir.makeQualified(fs);
176 
177     // Export Snapshot
178     int res = ExportSnapshot.innerMain(TEST_UTIL.getConfiguration(), new String[] {
179       "-snapshot", Bytes.toString(snapshotName),
180       "-copy-to", copyDir.toString()
181     });
182     assertEquals(0, res);
183 
184     // Verify File-System state
185     FileStatus[] rootFiles = fs.listStatus(copyDir);
186     assertEquals(filesExpected, rootFiles.length);
187     for (FileStatus fileStatus: rootFiles) {
188       String name = fileStatus.getPath().getName();
189       assertTrue(fileStatus.isDir());
190       assertTrue(name.equals(HConstants.SNAPSHOT_DIR_NAME) || name.equals(".archive"));
191     }
192 
193     // compare the snapshot metadata and verify the hfiles
194     final FileSystem hdfs = FileSystem.get(hdfsUri, TEST_UTIL.getConfiguration());
195     final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName));
196     verifySnapshot(hdfs, new Path(TEST_UTIL.getDefaultRootDirPath(), snapshotDir),
197         fs, new Path(copyDir, snapshotDir));
198     verifyArchive(fs, copyDir, tableName, Bytes.toString(snapshotName));
199     FSUtils.logFileSystemState(hdfs, snapshotDir, LOG);
200 
201     // Remove the exported dir
202     fs.delete(copyDir, true);
203   }
204 
205   /**
206     * Mock a snapshot with files in the archive dir,
207     * two regions, and one reference file.
208     */
209   @Test
210   public void testSnapshotWithRefsExportFileSystemState() throws Exception {
211     Configuration conf = TEST_UTIL.getConfiguration();
212 
213     final byte[] tableWithRefsName = Bytes.toBytes("tableWithRefs");
214     final String snapshotName = "tableWithRefs";
215     final String TEST_FAMILY = Bytes.toString(FAMILY);
216     final String TEST_HFILE = "abc";
217 
218     final SnapshotDescription sd = SnapshotDescription.newBuilder()
219         .setName(snapshotName).setTable(Bytes.toString(tableWithRefsName)).build();
220 
221     FileSystem fs = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
222     Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
223     Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
224 
225     HTableDescriptor htd = new HTableDescriptor(tableWithRefsName);
226     htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
227 
228     // First region, simple with one plain hfile.
229     HRegion r0 = HRegion.createHRegion(new HRegionInfo(htd.getName()), archiveDir,
230       conf, htd, null, true, true);
231     Path storeFile = new Path(new Path(r0.getRegionDir(), TEST_FAMILY), TEST_HFILE);
232     FSDataOutputStream out = fs.create(storeFile);
233     out.write(Bytes.toBytes("Test Data"));
234     out.close();
235     r0.close();
236 
237     // Second region, used to test the split case.
238     // This region contains a reference to the hfile in the first region.
239     HRegion r1 = HRegion.createHRegion(new HRegionInfo(htd.getName()), archiveDir,
240       conf, htd, null, true, true);
241     out = fs.create(new Path(new Path(r1.getRegionDir(), TEST_FAMILY),
242       storeFile.getName() + '.' + r0.getRegionInfo().getEncodedName()));
243     out.write(Bytes.toBytes("Test Data"));
244     out.close();
245     r1.close();
246 
247     Path tableDir = HTableDescriptor.getTableDir(archiveDir, tableWithRefsName);
248     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
249     FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
250     SnapshotDescriptionUtils.writeSnapshotInfo(sd, snapshotDir, fs);
251 
252     testExportFileSystemState(tableWithRefsName, Bytes.toBytes(snapshotName), 2);
253   }
254 
255   /*
256    * verify if the snapshot folder on file-system 1 match the one on file-system 2
257    */
258   private void verifySnapshot(final FileSystem fs1, final Path root1,
259       final FileSystem fs2, final Path root2) throws IOException {
260     Set<String> s = new HashSet<String>();
261     assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2));
262   }
263 
264   /*
265    * Verify if the files exists
266    */
267   private void verifyArchive(final FileSystem fs, final Path rootDir,
268       final byte[] tableName, final String snapshotName) throws IOException {
269     final Path exportedSnapshot = new Path(rootDir,
270       new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
271     final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
272     LOG.debug(listFiles(fs, exportedArchive, exportedArchive));
273     SnapshotReferenceUtil.visitReferencedFiles(fs, exportedSnapshot,
274         new SnapshotReferenceUtil.FileVisitor() {
275         public void storeFile (final String region, final String family, final String hfile)
276             throws IOException {
277           verifyNonEmptyFile(new Path(exportedArchive,
278             new Path(Bytes.toString(tableName), new Path(region, new Path(family, hfile)))));
279         }
280 
281         public void recoveredEdits (final String region, final String logfile)
282             throws IOException {
283           verifyNonEmptyFile(new Path(exportedSnapshot,
284             new Path(Bytes.toString(tableName), new Path(region, logfile))));
285         }
286 
287         public void logFile (final String server, final String logfile)
288             throws IOException {
289           verifyNonEmptyFile(new Path(exportedSnapshot, new Path(server, logfile)));
290         }
291 
292         private void verifyNonEmptyFile(final Path path) throws IOException {
293           assertTrue(path + " should exist", fs.exists(path));
294           assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0);
295         }
296     });
297   }
298 
299   private Set<String> listFiles(final FileSystem fs, final Path root, final Path dir)
300       throws IOException {
301     Set<String> files = new HashSet<String>();
302     int rootPrefix = root.toString().length();
303     FileStatus[] list = FSUtils.listStatus(fs, dir);
304     if (list != null) {
305       for (FileStatus fstat: list) {
306         LOG.debug(fstat.getPath());
307         if (fstat.isDir()) {
308           files.addAll(listFiles(fs, root, fstat.getPath()));
309         } else {
310           files.add(fstat.getPath().toString().substring(rootPrefix));
311         }
312       }
313     }
314     return files;
315   }
316 }
317