View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.mob.compactions;
20  
21  import static org.junit.Assert.assertEquals;
22  
23  import java.io.IOException;
24  import java.security.Key;
25  import java.security.SecureRandom;
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.Collections;
29  import java.util.List;
30  import java.util.Random;
31  import java.util.concurrent.ExecutorService;
32  import java.util.concurrent.RejectedExecutionException;
33  import java.util.concurrent.RejectedExecutionHandler;
34  import java.util.concurrent.SynchronousQueue;
35  import java.util.concurrent.ThreadPoolExecutor;
36  import java.util.concurrent.TimeUnit;
37  
38  import javax.crypto.spec.SecretKeySpec;
39  
40  import org.apache.hadoop.conf.Configuration;
41  import org.apache.hadoop.fs.FileStatus;
42  import org.apache.hadoop.fs.FileSystem;
43  import org.apache.hadoop.fs.Path;
44  import org.apache.hadoop.hbase.Cell;
45  import org.apache.hadoop.hbase.CellUtil;
46  import org.apache.hadoop.hbase.HBaseTestingUtility;
47  import org.apache.hadoop.hbase.HColumnDescriptor;
48  import org.apache.hadoop.hbase.HConstants;
49  import org.apache.hadoop.hbase.HTableDescriptor;
50  import org.apache.hadoop.hbase.NamespaceDescriptor;
51  import org.apache.hadoop.hbase.TableName;
52  import org.apache.hadoop.hbase.client.Admin;
53  import org.apache.hadoop.hbase.client.Admin.CompactType;
54  import org.apache.hadoop.hbase.client.BufferedMutator;
55  import org.apache.hadoop.hbase.client.Connection;
56  import org.apache.hadoop.hbase.client.ConnectionFactory;
57  import org.apache.hadoop.hbase.client.Delete;
58  import org.apache.hadoop.hbase.client.Durability;
59  import org.apache.hadoop.hbase.client.Get;
60  import org.apache.hadoop.hbase.client.Put;
61  import org.apache.hadoop.hbase.client.Result;
62  import org.apache.hadoop.hbase.client.ResultScanner;
63  import org.apache.hadoop.hbase.client.Scan;
64  import org.apache.hadoop.hbase.client.Table;
65  import org.apache.hadoop.hbase.io.HFileLink;
66  import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
67  import org.apache.hadoop.hbase.io.crypto.aes.AES;
68  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
69  import org.apache.hadoop.hbase.io.hfile.HFile;
70  import org.apache.hadoop.hbase.mob.MobConstants;
71  import org.apache.hadoop.hbase.mob.MobUtils;
72  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
73  import org.apache.hadoop.hbase.regionserver.BloomType;
74  import org.apache.hadoop.hbase.regionserver.HRegion;
75  import org.apache.hadoop.hbase.regionserver.StoreFile;
76  import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
77  import org.apache.hadoop.hbase.security.EncryptionUtil;
78  import org.apache.hadoop.hbase.security.User;
79  import org.apache.hadoop.hbase.testclassification.LargeTests;
80  import org.apache.hadoop.hbase.util.Bytes;
81  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
82  import org.apache.hadoop.hbase.util.Threads;
83  import org.junit.AfterClass;
84  import org.junit.Assert;
85  import org.junit.BeforeClass;
86  import org.junit.Test;
87  import org.junit.experimental.categories.Category;
88  
89  @Category(LargeTests.class)
90  public class TestMobCompactor {
91    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
92    private static Configuration conf = null;
93    private TableName tableName;
94    private static Connection conn;
95    private BufferedMutator bufMut;
96    private Table table;
97    private static Admin admin;
98    private HTableDescriptor desc;
99    private HColumnDescriptor hcd1;
100   private HColumnDescriptor hcd2;
101   private static FileSystem fs;
102   private static final String family1 = "family1";
103   private static final String family2 = "family2";
104   private static final String qf1 = "qualifier1";
105   private static final String qf2 = "qualifier2";
106   private static byte[] KEYS = Bytes.toBytes("012");
107   private static int regionNum = KEYS.length;
108   private static int delRowNum = 1;
109   private static int delCellNum = 6;
110   private static int cellNumPerRow = 3;
111   private static int rowNumPerFile = 2;
112   private static ExecutorService pool;
113 
114   @BeforeClass
115   public static void setUpBeforeClass() throws Exception {
116     TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
117     TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
118     TEST_UTIL.getConfiguration()
119       .setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, 5000);
120     TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY,
121       KeyProviderForTesting.class.getName());
122     TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
123     TEST_UTIL.startMiniCluster(1);
124     pool = createThreadPool(TEST_UTIL.getConfiguration());
125     conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration(), pool);
126     fs = TEST_UTIL.getTestFileSystem();
127     conf = TEST_UTIL.getConfiguration();
128     admin = TEST_UTIL.getHBaseAdmin();
129   }
130 
131   @AfterClass
132   public static void tearDownAfterClass() throws Exception {
133     pool.shutdown();
134     conn.close();
135     TEST_UTIL.shutdownMiniCluster();
136   }
137 
138   public void setUp(String tableNameAsString) throws IOException {
139     tableName = TableName.valueOf(tableNameAsString);
140     hcd1 = new HColumnDescriptor(family1);
141     hcd1.setMobEnabled(true);
142     hcd1.setMobThreshold(5);
143     hcd2 = new HColumnDescriptor(family2);
144     hcd2.setMobEnabled(true);
145     hcd2.setMobThreshold(5);
146     desc = new HTableDescriptor(tableName);
147     desc.addFamily(hcd1);
148     desc.addFamily(hcd2);
149     admin.createTable(desc, getSplitKeys());
150     table = conn.getTable(tableName);
151     bufMut = conn.getBufferedMutator(tableName);
152   }
153 
154   @Test(timeout = 300000)
155   public void testMinorCompaction() throws Exception {
156     resetConf();
157     int mergeSize = 5000;
158     // change the mob compaction merge size
159     conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
160 
161     // create a table with namespace
162     NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("ns").build();
163     String tableNameAsString = "ns:testMinorCompaction";
164     admin.createNamespace(namespaceDescriptor);
165     setUp(tableNameAsString);
166     int count = 4;
167     // generate mob files
168     loadData(admin, bufMut, tableName, count, rowNumPerFile);
169     int rowNumPerRegion = count * rowNumPerFile;
170 
171     assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion,
172       countMobRows(table));
173     assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion,
174       countMobCells(table));
175     assertEquals("Before deleting: mob file count", regionNum * count,
176       countFiles(tableName, true, family1));
177 
178     int largeFilesCount = countLargeFiles(mergeSize, tableName, family1);
179     createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1));
180 
181     assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
182       countMobRows(table));
183     assertEquals("Before compaction: mob cells count", regionNum
184       * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
185     assertEquals("Before compaction: family1 mob file count", regionNum * count,
186       countFiles(tableName, true, family1));
187     assertEquals("Before compaction: family2 mob file count", regionNum * count,
188       countFiles(tableName, true, family2));
189     assertEquals("Before compaction: family1 del file count", regionNum,
190       countFiles(tableName, false, family1));
191     assertEquals("Before compaction: family2 del file count", regionNum,
192       countFiles(tableName, false, family2));
193 
194     // do the mob file compaction
195     MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
196     compactor.compact();
197 
198     assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
199       countMobRows(table));
200     assertEquals("After compaction: mob cells count", regionNum
201       * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
202     // After the compaction, the files smaller than the mob compaction merge size
203     // is merge to one file
204     assertEquals("After compaction: family1 mob file count", largeFilesCount + regionNum,
205       countFiles(tableName, true, family1));
206     assertEquals("After compaction: family2 mob file count", regionNum * count,
207       countFiles(tableName, true, family2));
208     assertEquals("After compaction: family1 del file count", regionNum,
209       countFiles(tableName, false, family1));
210     assertEquals("After compaction: family2 del file count", regionNum,
211       countFiles(tableName, false, family2));
212   }
213 
214   @Test(timeout = 300000)
215   public void testCompactionWithHFileLink() throws IOException, InterruptedException {
216     resetConf();
217     String tableNameAsString = "testCompactionWithHFileLink";
218     setUp(tableNameAsString);
219     int count = 4;
220     // generate mob files
221     loadData(admin, bufMut, tableName, count, rowNumPerFile);
222     int rowNumPerRegion = count * rowNumPerFile;
223 
224     long tid = System.currentTimeMillis();
225     byte[] snapshotName1 = Bytes.toBytes("snaptb-" + tid);
226     // take a snapshot
227     admin.snapshot(snapshotName1, tableName);
228 
229     createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1));
230 
231     assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
232       countMobRows(table));
233     assertEquals("Before compaction: mob cells count", regionNum
234       * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
235     assertEquals("Before compaction: family1 mob file count", regionNum * count,
236       countFiles(tableName, true, family1));
237     assertEquals("Before compaction: family2 mob file count", regionNum * count,
238       countFiles(tableName, true, family2));
239     assertEquals("Before compaction: family1 del file count", regionNum,
240       countFiles(tableName, false, family1));
241     assertEquals("Before compaction: family2 del file count", regionNum,
242       countFiles(tableName, false, family2));
243 
244     // do the mob compaction
245     MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
246     compactor.compact();
247 
248     assertEquals("After first compaction: mob rows count", regionNum
249       * (rowNumPerRegion - delRowNum), countMobRows(table));
250     assertEquals("After first compaction: mob cells count", regionNum
251       * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
252     assertEquals("After first compaction: family1 mob file count", regionNum,
253       countFiles(tableName, true, family1));
254     assertEquals("After first compaction: family2 mob file count", regionNum * count,
255       countFiles(tableName, true, family2));
256     assertEquals("After first compaction: family1 del file count", 0,
257       countFiles(tableName, false, family1));
258     assertEquals("After first compaction: family2 del file count", regionNum,
259       countFiles(tableName, false, family2));
260     assertEquals("After first compaction: family1 hfilelink count", 0, countHFileLinks(family1));
261     assertEquals("After first compaction: family2 hfilelink count", 0, countHFileLinks(family2));
262 
263     admin.disableTable(tableName);
264     // Restore from snapshot, the hfilelink will exist in mob dir
265     admin.restoreSnapshot(snapshotName1);
266     admin.enableTable(tableName);
267 
268     assertEquals("After restoring snapshot: mob rows count", regionNum * rowNumPerRegion,
269       countMobRows(table));
270     assertEquals("After restoring snapshot: mob cells count", regionNum * cellNumPerRow
271       * rowNumPerRegion, countMobCells(table));
272     assertEquals("After restoring snapshot: family1 mob file count", regionNum * count,
273       countFiles(tableName, true, family1));
274     assertEquals("After restoring snapshot: family2 mob file count", regionNum * count,
275       countFiles(tableName, true, family2));
276     assertEquals("After restoring snapshot: family1 del file count", 0,
277       countFiles(tableName, false, family1));
278     assertEquals("After restoring snapshot: family2 del file count", 0,
279       countFiles(tableName, false, family2));
280     assertEquals("After restoring snapshot: family1 hfilelink count", regionNum * count,
281       countHFileLinks(family1));
282     assertEquals("After restoring snapshot: family2 hfilelink count", 0, countHFileLinks(family2));
283 
284     compactor.compact();
285 
286     assertEquals("After second compaction: mob rows count", regionNum * rowNumPerRegion,
287       countMobRows(table));
288     assertEquals("After second compaction: mob cells count", regionNum * cellNumPerRow
289       * rowNumPerRegion, countMobCells(table));
290     assertEquals("After second compaction: family1 mob file count", regionNum,
291       countFiles(tableName, true, family1));
292     assertEquals("After second compaction: family2 mob file count", regionNum * count,
293       countFiles(tableName, true, family2));
294     assertEquals("After second compaction: family1 del file count", 0,
295       countFiles(tableName, false, family1));
296     assertEquals("After second compaction: family2 del file count", 0,
297       countFiles(tableName, false, family2));
298     assertEquals("After second compaction: family1 hfilelink count", 0, countHFileLinks(family1));
299     assertEquals("After second compaction: family2 hfilelink count", 0, countHFileLinks(family2));
300     assertRefFileNameEqual(family1);
301   }
302 
303   @Test(timeout = 300000)
304   public void testMajorCompactionFromAdmin() throws Exception {
305     resetConf();
306     int mergeSize = 5000;
307     // change the mob compaction merge size
308     conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
309     String tableNameAsString = "testMajorCompactionFromAdmin";
310     SecureRandom rng = new SecureRandom();
311     byte[] keyBytes = new byte[AES.KEY_LENGTH];
312     rng.nextBytes(keyBytes);
313     String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
314     Key cfKey = new SecretKeySpec(keyBytes, algorithm);
315     byte[] encryptionKey = EncryptionUtil.wrapKey(conf,
316       conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey);
317     TableName tableName = TableName.valueOf(tableNameAsString);
318     HTableDescriptor desc = new HTableDescriptor(tableName);
319     HColumnDescriptor hcd1 = new HColumnDescriptor(family1);
320     hcd1.setMobEnabled(true);
321     hcd1.setMobThreshold(0);
322     hcd1.setEncryptionType(algorithm);
323     hcd1.setEncryptionKey(encryptionKey);
324     HColumnDescriptor hcd2 = new HColumnDescriptor(family2);
325     hcd2.setMobEnabled(true);
326     hcd2.setMobThreshold(0);
327     desc.addFamily(hcd1);
328     desc.addFamily(hcd2);
329     admin.createTable(desc, getSplitKeys());
330     Table table = conn.getTable(tableName);
331     BufferedMutator bufMut = conn.getBufferedMutator(tableName);
332     int count = 4;
333     // generate mob files
334     loadData(admin, bufMut, tableName, count, rowNumPerFile);
335     int rowNumPerRegion = count * rowNumPerFile;
336 
337     assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion,
338       countMobRows(table));
339     assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion,
340       countMobCells(table));
341     assertEquals("Before deleting: mob file count", regionNum * count,
342       countFiles(tableName, true, family1));
343 
344     createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1));
345 
346     assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
347       countMobRows(table));
348     assertEquals("Before compaction: mob cells count", regionNum
349       * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
350     assertEquals("Before compaction: family1 mob file count", regionNum * count,
351       countFiles(tableName, true, family1));
352     assertEquals("Before compaction: family2 mob file count", regionNum * count,
353       countFiles(tableName, true, family2));
354     assertEquals("Before compaction: family1 del file count", regionNum,
355       countFiles(tableName, false, family1));
356     assertEquals("Before compaction: family2 del file count", regionNum,
357       countFiles(tableName, false, family2));
358 
359     // do the major mob compaction, it will force all files to compaction
360     admin.majorCompact(tableName, hcd1.getName(), Admin.CompactType.MOB);
361 
362     waitUntilMobCompactionFinished(tableName);
363     assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
364       countMobRows(table));
365     assertEquals("After compaction: mob cells count", regionNum
366       * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
367     assertEquals("After compaction: family1 mob file count", regionNum,
368       countFiles(tableName, true, family1));
369     assertEquals("After compaction: family2 mob file count", regionNum * count,
370       countFiles(tableName, true, family2));
371     assertEquals("After compaction: family1 del file count", 0,
372       countFiles(tableName, false, family1));
373     assertEquals("After compaction: family2 del file count", regionNum,
374       countFiles(tableName, false, family2));
375     Assert.assertTrue(verifyEncryption(tableName, family1));
376     table.close();
377   }
378 
379   @Test(timeout = 300000)
380   public void testScannerOnBulkLoadRefHFiles() throws Exception {
381     resetConf();
382     setUp("testScannerOnBulkLoadRefHFiles");
383     long ts = EnvironmentEdgeManager.currentTime();
384     byte[] key0 = Bytes.toBytes("k0");
385     byte[] key1 = Bytes.toBytes("k1");
386     String value0 = "mobValue0";
387     String value1 = "mobValue1";
388     String newValue0 = "new";
389     Put put0 = new Put(key0);
390     put0.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), ts, Bytes.toBytes(value0));
391     loadData(admin, bufMut, tableName, new Put[] { put0 });
392     put0 = new Put(key0);
393     put0.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), ts, Bytes.toBytes(newValue0));
394     Put put1 = new Put(key1);
395     put1.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), ts, Bytes.toBytes(value1));
396     loadData(admin, bufMut, tableName, new Put[] { put0, put1 });
397     // read the latest cell of key0.
398     Get get = new Get(key0);
399     Result result = table.get(get);
400     Cell cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
401     assertEquals("Before compaction: mob value of k0", newValue0,
402       Bytes.toString(CellUtil.cloneValue(cell)));
403     admin.majorCompact(tableName, hcd1.getName(), Admin.CompactType.MOB);
404     waitUntilMobCompactionFinished(tableName);
405     // read the latest cell of key0, the cell seqId in bulk loaded file is not reset in the
406     // scanner. The cell that has "new" value is still visible.
407     result = table.get(get);
408     cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
409     assertEquals("After compaction: mob value of k0", newValue0,
410       Bytes.toString(CellUtil.cloneValue(cell)));
411     // read the ref cell, not read further to the mob cell.
412     get = new Get(key1);
413     get.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(true));
414     result = table.get(get);
415     cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
416     // the ref name is the new file
417     Path mobFamilyPath =
418       MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, hcd1.getNameAsString());
419     List<Path> paths = new ArrayList<Path>();
420     if (fs.exists(mobFamilyPath)) {
421       FileStatus[] files = fs.listStatus(mobFamilyPath);
422       for (FileStatus file : files) {
423         if (!StoreFileInfo.isDelFile(file.getPath())) {
424           paths.add(file.getPath());
425         }
426       }
427     }
428     assertEquals("After compaction: number of mob files:", 1, paths.size());
429     assertEquals("After compaction: mob file name:", MobUtils.getMobFileName(cell), paths.get(0)
430       .getName());
431   }
432 
433   @Test(timeout = 300000)
434   public void testScannerAfterCompactions() throws Exception {
435     resetConf();
436     setUp("testScannerAfterCompactions");
437     long ts = EnvironmentEdgeManager.currentTime();
438     byte[] key0 = Bytes.toBytes("k0");
439     byte[] key1 = Bytes.toBytes("k1");
440     String value = "mobValue"; // larger than threshold
441     String newValue = "new";
442     Put put0 = new Put(key0);
443     put0.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), ts, Bytes.toBytes(value));
444     loadData(admin, bufMut, tableName, new Put[] { put0 });
445     Put put1 = new Put(key1);
446     put1.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), ts, Bytes.toBytes(value));
447     loadData(admin, bufMut, tableName, new Put[] { put1 });
448     put1 = new Put(key1);
449     put1.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), ts, Bytes.toBytes(newValue));
450     loadData(admin, bufMut, tableName, new Put[] { put1 }); // now two mob files
451     admin.majorCompact(tableName);
452     waitUntilCompactionFinished(tableName);
453     admin.majorCompact(tableName, hcd1.getName(), Admin.CompactType.MOB);
454     waitUntilMobCompactionFinished(tableName);
455     // read the latest cell of key1.
456     Get get = new Get(key1);
457     Result result = table.get(get);
458     Cell cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
459     assertEquals("After compaction: mob value", "new", Bytes.toString(CellUtil.cloneValue(cell)));
460   }
461 
462   private void waitUntilCompactionFinished(TableName tableName) throws IOException,
463     InterruptedException {
464     long finished = EnvironmentEdgeManager.currentTime() + 60000;
465     CompactionState state = admin.getCompactionState(tableName);
466     while (EnvironmentEdgeManager.currentTime() < finished) {
467       if (state == CompactionState.NONE) {
468         break;
469       }
470       state = admin.getCompactionState(tableName);
471       Thread.sleep(10);
472     }
473     assertEquals(CompactionState.NONE, state);
474   }
475 
476   private void waitUntilMobCompactionFinished(TableName tableName) throws IOException,
477     InterruptedException {
478     long finished = EnvironmentEdgeManager.currentTime() + 60000;
479     CompactionState state = admin.getCompactionState(tableName, Admin.CompactType.MOB);
480     while (EnvironmentEdgeManager.currentTime() < finished) {
481       if (state == CompactionState.NONE) {
482         break;
483       }
484       state = admin.getCompactionState(tableName, Admin.CompactType.MOB);
485       Thread.sleep(10);
486     }
487     assertEquals(CompactionState.NONE, state);
488   }
489 
490   /**
491    * Gets the number of rows in the given table.
492    * @param table to get the  scanner
493    * @return the number of rows
494    */
495   private int countMobRows(final Table table) throws IOException {
496     Scan scan = new Scan();
497     // Do not retrieve the mob data when scanning
498     scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
499     return TEST_UTIL.countRows(table, scan);
500   }
501 
502   /**
503    * Gets the number of cells in the given table.
504    * @param table to get the  scanner
505    * @return the number of cells
506    */
507   private int countMobCells(final Table table) throws IOException {
508     Scan scan = new Scan();
509     // Do not retrieve the mob data when scanning
510     scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
511     ResultScanner results = table.getScanner(scan);
512     int count = 0;
513     for (Result res : results) {
514       for (Cell cell : res.listCells()) {
515         count++;
516       }
517     }
518     results.close();
519     return count;
520   }
521 
522   /**
523    * Gets the number of files in the mob path.
524    * @param isMobFile gets number of the mob files or del files
525    * @param familyName the family name
526    * @return the number of the files
527    */
528   private int countFiles(TableName tableName, boolean isMobFile, String familyName)
529     throws IOException {
530     Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName);
531     int count = 0;
532     if (fs.exists(mobDirPath)) {
533       FileStatus[] files = fs.listStatus(mobDirPath);
534       for (FileStatus file : files) {
535         if (isMobFile == true) {
536           if (!StoreFileInfo.isDelFile(file.getPath())) {
537             count++;
538           }
539         } else {
540           if (StoreFileInfo.isDelFile(file.getPath())) {
541             count++;
542           }
543         }
544       }
545     }
546     return count;
547   }
548 
549   private boolean verifyEncryption(TableName tableName, String familyName) throws IOException {
550     Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName);
551     boolean hasFiles = false;
552     if (fs.exists(mobDirPath)) {
553       FileStatus[] files = fs.listStatus(mobDirPath);
554       hasFiles = files != null && files.length > 0;
555       Assert.assertTrue(hasFiles);
556       Path path = files[0].getPath();
557       CacheConfig cacheConf = new CacheConfig(conf);
558       StoreFile sf = new StoreFile(TEST_UTIL.getTestFileSystem(), path, conf, cacheConf,
559         BloomType.NONE);
560       HFile.Reader reader = sf.createReader().getHFileReader();
561       byte[] encryptionKey = reader.getTrailer().getEncryptionKey();
562       Assert.assertTrue(null != encryptionKey);
563       Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName()
564         .equals(HConstants.CIPHER_AES));
565     }
566     return hasFiles;
567   }
568 
569   /**
570    * Gets the number of HFileLink in the mob path.
571    * @param familyName the family name
572    * @return the number of the HFileLink
573    */
574   private int countHFileLinks(String familyName) throws IOException {
575     Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName);
576     int count = 0;
577     if (fs.exists(mobDirPath)) {
578       FileStatus[] files = fs.listStatus(mobDirPath);
579       for (FileStatus file : files) {
580         if (HFileLink.isHFileLink(file.getPath())) {
581           count++;
582         }
583       }
584     }
585     return count;
586   }
587 
588   /**
589    * Gets the number of files.
590    * @param size the size of the file
591    * @param tableName the current table name
592    * @param familyName the family name
593    * @return the number of files large than the size
594    */
595   private int countLargeFiles(int size, TableName tableName, String familyName) throws IOException {
596     Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName);
597     int count = 0;
598     if (fs.exists(mobDirPath)) {
599       FileStatus[] files = fs.listStatus(mobDirPath);
600       for (FileStatus file : files) {
601         // ignore the del files in the mob path
602         if ((!StoreFileInfo.isDelFile(file.getPath())) && (file.getLen() > size)) {
603           count++;
604         }
605       }
606     }
607     return count;
608   }
609 
610   /**
611    * loads some data to the table.
612    */
613   private void loadData(Admin admin, BufferedMutator table, TableName tableName, int fileNum,
614     int rowNumPerFile) throws IOException, InterruptedException {
615     if (fileNum <= 0) {
616       throw new IllegalArgumentException();
617     }
618     for (int i = 0; i < fileNum * rowNumPerFile; i++) {
619       for (byte k0 : KEYS) {
620         byte[] k = new byte[] { k0 };
621         byte[] key = Bytes.add(k, Bytes.toBytes(i));
622         byte[] mobVal = makeDummyData(10 * (i + 1));
623         Put put = new Put(key);
624         put.setDurability(Durability.SKIP_WAL);
625         put.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), mobVal);
626         put.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf2), mobVal);
627         put.addColumn(Bytes.toBytes(family2), Bytes.toBytes(qf1), mobVal);
628         table.mutate(put);
629       }
630       if ((i + 1) % rowNumPerFile == 0) {
631         table.flush();
632         admin.flush(tableName);
633       }
634     }
635   }
636 
637   private void loadData(Admin admin, BufferedMutator table, TableName tableName, Put[] puts)
638     throws IOException {
639     table.mutate(Arrays.asList(puts));
640     table.flush();
641     admin.flush(tableName);
642   }
643 
644   /**
645    * delete the row, family and cell to create the del file
646    */
647   private void createDelFile(Table table, TableName tableName, byte[] family, byte[] qf)
648     throws IOException, InterruptedException {
649     for (byte k0 : KEYS) {
650       byte[] k = new byte[] { k0 };
651       // delete a family
652       byte[] key1 = Bytes.add(k, Bytes.toBytes(0));
653       Delete delete1 = new Delete(key1);
654       delete1.addFamily(family);
655       table.delete(delete1);
656       // delete one row
657       byte[] key2 = Bytes.add(k, Bytes.toBytes(2));
658       Delete delete2 = new Delete(key2);
659       table.delete(delete2);
660       // delete one cell
661       byte[] key3 = Bytes.add(k, Bytes.toBytes(4));
662       Delete delete3 = new Delete(key3);
663       delete3.addColumn(family, qf);
664       table.delete(delete3);
665     }
666     admin.flush(tableName);
667     List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
668     for (HRegion region : regions) {
669       region.waitForFlushesAndCompactions();
670       region.compact(true);
671     }
672   }
673   /**
674    * Creates the dummy data with a specific size.
675    * @param size the size of value
676    * @return the dummy data
677    */
678   private byte[] makeDummyData(int size) {
679     byte[] dummyData = new byte[size];
680     new Random().nextBytes(dummyData);
681     return dummyData;
682   }
683 
684   /**
685    * Gets the split keys
686    */
687   private byte[][] getSplitKeys() {
688     byte[][] splitKeys = new byte[KEYS.length - 1][];
689     for (int i = 0; i < splitKeys.length; ++i) {
690       splitKeys[i] = new byte[] { KEYS[i + 1] };
691     }
692     return splitKeys;
693   }
694 
695   private static ExecutorService createThreadPool(Configuration conf) {
696     int maxThreads = 10;
697     long keepAliveTime = 60;
698     final SynchronousQueue<Runnable> queue = new SynchronousQueue<Runnable>();
699     ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads,
700         keepAliveTime, TimeUnit.SECONDS, queue,
701         Threads.newDaemonThreadFactory("MobFileCompactionChore"),
702         new RejectedExecutionHandler() {
703           @Override
704           public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
705             try {
706               // waiting for a thread to pick up instead of throwing exceptions.
707               queue.put(r);
708             } catch (InterruptedException e) {
709               throw new RejectedExecutionException(e);
710             }
711           }
712         });
713     ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true);
714     return pool;
715   }
716 
717   private void assertRefFileNameEqual(String familyName) throws IOException {
718     Scan scan = new Scan();
719     scan.addFamily(Bytes.toBytes(familyName));
720     // Do not retrieve the mob data when scanning
721     scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
722     ResultScanner results = table.getScanner(scan);
723     Path mobFamilyPath = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(),
724         tableName, familyName);
725     List<Path> actualFilePaths = new ArrayList<>();
726     List<Path> expectFilePaths = new ArrayList<>();
727     for (Result res : results) {
728       for (Cell cell : res.listCells()) {
729         byte[] referenceValue = CellUtil.cloneValue(cell);
730         String fileName = Bytes.toString(referenceValue, Bytes.SIZEOF_INT,
731             referenceValue.length - Bytes.SIZEOF_INT);
732         Path targetPath = new Path(mobFamilyPath, fileName);
733         if(!actualFilePaths.contains(targetPath)) {
734           actualFilePaths.add(targetPath);
735         }
736       }
737     }
738     results.close();
739     if (fs.exists(mobFamilyPath)) {
740       FileStatus[] files = fs.listStatus(mobFamilyPath);
741       for (FileStatus file : files) {
742         if (!StoreFileInfo.isDelFile(file.getPath())) {
743           expectFilePaths.add(file.getPath());
744         }
745       }
746     }
747     Collections.sort(actualFilePaths);
748     Collections.sort(expectFilePaths);
749     assertEquals(expectFilePaths, actualFilePaths);
750   }
751 
752   /**
753    * Resets the configuration.
754    */
755   private void resetConf() {
756     conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD,
757       MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD);
758     conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE,
759       MobConstants.DEFAULT_MOB_COMPACTION_BATCH_SIZE);
760   }
761 }