1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mob.compactions;
20
21 import static org.junit.Assert.assertEquals;
22
23 import java.io.IOException;
24 import java.security.Key;
25 import java.security.SecureRandom;
26 import java.util.ArrayList;
27 import java.util.Arrays;
28 import java.util.Collections;
29 import java.util.List;
30 import java.util.Random;
31 import java.util.concurrent.ExecutorService;
32 import java.util.concurrent.RejectedExecutionException;
33 import java.util.concurrent.RejectedExecutionHandler;
34 import java.util.concurrent.SynchronousQueue;
35 import java.util.concurrent.ThreadPoolExecutor;
36 import java.util.concurrent.TimeUnit;
37
38 import javax.crypto.spec.SecretKeySpec;
39
40 import org.apache.hadoop.conf.Configuration;
41 import org.apache.hadoop.fs.FileStatus;
42 import org.apache.hadoop.fs.FileSystem;
43 import org.apache.hadoop.fs.Path;
44 import org.apache.hadoop.hbase.Cell;
45 import org.apache.hadoop.hbase.CellUtil;
46 import org.apache.hadoop.hbase.HBaseTestingUtility;
47 import org.apache.hadoop.hbase.HColumnDescriptor;
48 import org.apache.hadoop.hbase.HConstants;
49 import org.apache.hadoop.hbase.HTableDescriptor;
50 import org.apache.hadoop.hbase.NamespaceDescriptor;
51 import org.apache.hadoop.hbase.TableName;
52 import org.apache.hadoop.hbase.client.Admin;
53 import org.apache.hadoop.hbase.client.Admin.CompactType;
54 import org.apache.hadoop.hbase.client.BufferedMutator;
55 import org.apache.hadoop.hbase.client.Connection;
56 import org.apache.hadoop.hbase.client.ConnectionFactory;
57 import org.apache.hadoop.hbase.client.Delete;
58 import org.apache.hadoop.hbase.client.Durability;
59 import org.apache.hadoop.hbase.client.Get;
60 import org.apache.hadoop.hbase.client.Put;
61 import org.apache.hadoop.hbase.client.Result;
62 import org.apache.hadoop.hbase.client.ResultScanner;
63 import org.apache.hadoop.hbase.client.Scan;
64 import org.apache.hadoop.hbase.client.Table;
65 import org.apache.hadoop.hbase.io.HFileLink;
66 import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
67 import org.apache.hadoop.hbase.io.crypto.aes.AES;
68 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
69 import org.apache.hadoop.hbase.io.hfile.HFile;
70 import org.apache.hadoop.hbase.mob.MobConstants;
71 import org.apache.hadoop.hbase.mob.MobUtils;
72 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
73 import org.apache.hadoop.hbase.regionserver.BloomType;
74 import org.apache.hadoop.hbase.regionserver.HRegion;
75 import org.apache.hadoop.hbase.regionserver.StoreFile;
76 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
77 import org.apache.hadoop.hbase.security.EncryptionUtil;
78 import org.apache.hadoop.hbase.security.User;
79 import org.apache.hadoop.hbase.testclassification.LargeTests;
80 import org.apache.hadoop.hbase.util.Bytes;
81 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
82 import org.apache.hadoop.hbase.util.Threads;
83 import org.junit.AfterClass;
84 import org.junit.Assert;
85 import org.junit.BeforeClass;
86 import org.junit.Test;
87 import org.junit.experimental.categories.Category;
88
89 @Category(LargeTests.class)
90 public class TestMobCompactor {
91 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
92 private static Configuration conf = null;
93 private TableName tableName;
94 private static Connection conn;
95 private BufferedMutator bufMut;
96 private Table table;
97 private static Admin admin;
98 private HTableDescriptor desc;
99 private HColumnDescriptor hcd1;
100 private HColumnDescriptor hcd2;
101 private static FileSystem fs;
102 private static final String family1 = "family1";
103 private static final String family2 = "family2";
104 private static final String qf1 = "qualifier1";
105 private static final String qf2 = "qualifier2";
106 private static byte[] KEYS = Bytes.toBytes("012");
107 private static int regionNum = KEYS.length;
108 private static int delRowNum = 1;
109 private static int delCellNum = 6;
110 private static int cellNumPerRow = 3;
111 private static int rowNumPerFile = 2;
112 private static ExecutorService pool;
113
114 @BeforeClass
115 public static void setUpBeforeClass() throws Exception {
116 TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
117 TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
118 TEST_UTIL.getConfiguration()
119 .setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, 5000);
120 TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY,
121 KeyProviderForTesting.class.getName());
122 TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
123 TEST_UTIL.startMiniCluster(1);
124 pool = createThreadPool(TEST_UTIL.getConfiguration());
125 conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration(), pool);
126 fs = TEST_UTIL.getTestFileSystem();
127 conf = TEST_UTIL.getConfiguration();
128 admin = TEST_UTIL.getHBaseAdmin();
129 }
130
131 @AfterClass
132 public static void tearDownAfterClass() throws Exception {
133 pool.shutdown();
134 conn.close();
135 TEST_UTIL.shutdownMiniCluster();
136 }
137
138 public void setUp(String tableNameAsString) throws IOException {
139 tableName = TableName.valueOf(tableNameAsString);
140 hcd1 = new HColumnDescriptor(family1);
141 hcd1.setMobEnabled(true);
142 hcd1.setMobThreshold(5);
143 hcd2 = new HColumnDescriptor(family2);
144 hcd2.setMobEnabled(true);
145 hcd2.setMobThreshold(5);
146 desc = new HTableDescriptor(tableName);
147 desc.addFamily(hcd1);
148 desc.addFamily(hcd2);
149 admin.createTable(desc, getSplitKeys());
150 table = conn.getTable(tableName);
151 bufMut = conn.getBufferedMutator(tableName);
152 }
153
154 @Test(timeout = 300000)
155 public void testMinorCompaction() throws Exception {
156 resetConf();
157 int mergeSize = 5000;
158
159 conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
160
161
162 NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("ns").build();
163 String tableNameAsString = "ns:testMinorCompaction";
164 admin.createNamespace(namespaceDescriptor);
165 setUp(tableNameAsString);
166 int count = 4;
167
168 loadData(admin, bufMut, tableName, count, rowNumPerFile);
169 int rowNumPerRegion = count * rowNumPerFile;
170
171 assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion,
172 countMobRows(table));
173 assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion,
174 countMobCells(table));
175 assertEquals("Before deleting: mob file count", regionNum * count,
176 countFiles(tableName, true, family1));
177
178 int largeFilesCount = countLargeFiles(mergeSize, tableName, family1);
179 createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1));
180
181 assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
182 countMobRows(table));
183 assertEquals("Before compaction: mob cells count", regionNum
184 * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
185 assertEquals("Before compaction: family1 mob file count", regionNum * count,
186 countFiles(tableName, true, family1));
187 assertEquals("Before compaction: family2 mob file count", regionNum * count,
188 countFiles(tableName, true, family2));
189 assertEquals("Before compaction: family1 del file count", regionNum,
190 countFiles(tableName, false, family1));
191 assertEquals("Before compaction: family2 del file count", regionNum,
192 countFiles(tableName, false, family2));
193
194
195 MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
196 compactor.compact();
197
198 assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
199 countMobRows(table));
200 assertEquals("After compaction: mob cells count", regionNum
201 * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
202
203
204 assertEquals("After compaction: family1 mob file count", largeFilesCount + regionNum,
205 countFiles(tableName, true, family1));
206 assertEquals("After compaction: family2 mob file count", regionNum * count,
207 countFiles(tableName, true, family2));
208 assertEquals("After compaction: family1 del file count", regionNum,
209 countFiles(tableName, false, family1));
210 assertEquals("After compaction: family2 del file count", regionNum,
211 countFiles(tableName, false, family2));
212 }
213
214 @Test(timeout = 300000)
215 public void testCompactionWithHFileLink() throws IOException, InterruptedException {
216 resetConf();
217 String tableNameAsString = "testCompactionWithHFileLink";
218 setUp(tableNameAsString);
219 int count = 4;
220
221 loadData(admin, bufMut, tableName, count, rowNumPerFile);
222 int rowNumPerRegion = count * rowNumPerFile;
223
224 long tid = System.currentTimeMillis();
225 byte[] snapshotName1 = Bytes.toBytes("snaptb-" + tid);
226
227 admin.snapshot(snapshotName1, tableName);
228
229 createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1));
230
231 assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
232 countMobRows(table));
233 assertEquals("Before compaction: mob cells count", regionNum
234 * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
235 assertEquals("Before compaction: family1 mob file count", regionNum * count,
236 countFiles(tableName, true, family1));
237 assertEquals("Before compaction: family2 mob file count", regionNum * count,
238 countFiles(tableName, true, family2));
239 assertEquals("Before compaction: family1 del file count", regionNum,
240 countFiles(tableName, false, family1));
241 assertEquals("Before compaction: family2 del file count", regionNum,
242 countFiles(tableName, false, family2));
243
244
245 MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
246 compactor.compact();
247
248 assertEquals("After first compaction: mob rows count", regionNum
249 * (rowNumPerRegion - delRowNum), countMobRows(table));
250 assertEquals("After first compaction: mob cells count", regionNum
251 * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
252 assertEquals("After first compaction: family1 mob file count", regionNum,
253 countFiles(tableName, true, family1));
254 assertEquals("After first compaction: family2 mob file count", regionNum * count,
255 countFiles(tableName, true, family2));
256 assertEquals("After first compaction: family1 del file count", 0,
257 countFiles(tableName, false, family1));
258 assertEquals("After first compaction: family2 del file count", regionNum,
259 countFiles(tableName, false, family2));
260 assertEquals("After first compaction: family1 hfilelink count", 0, countHFileLinks(family1));
261 assertEquals("After first compaction: family2 hfilelink count", 0, countHFileLinks(family2));
262
263 admin.disableTable(tableName);
264
265 admin.restoreSnapshot(snapshotName1);
266 admin.enableTable(tableName);
267
268 assertEquals("After restoring snapshot: mob rows count", regionNum * rowNumPerRegion,
269 countMobRows(table));
270 assertEquals("After restoring snapshot: mob cells count", regionNum * cellNumPerRow
271 * rowNumPerRegion, countMobCells(table));
272 assertEquals("After restoring snapshot: family1 mob file count", regionNum * count,
273 countFiles(tableName, true, family1));
274 assertEquals("After restoring snapshot: family2 mob file count", regionNum * count,
275 countFiles(tableName, true, family2));
276 assertEquals("After restoring snapshot: family1 del file count", 0,
277 countFiles(tableName, false, family1));
278 assertEquals("After restoring snapshot: family2 del file count", 0,
279 countFiles(tableName, false, family2));
280 assertEquals("After restoring snapshot: family1 hfilelink count", regionNum * count,
281 countHFileLinks(family1));
282 assertEquals("After restoring snapshot: family2 hfilelink count", 0, countHFileLinks(family2));
283
284 compactor.compact();
285
286 assertEquals("After second compaction: mob rows count", regionNum * rowNumPerRegion,
287 countMobRows(table));
288 assertEquals("After second compaction: mob cells count", regionNum * cellNumPerRow
289 * rowNumPerRegion, countMobCells(table));
290 assertEquals("After second compaction: family1 mob file count", regionNum,
291 countFiles(tableName, true, family1));
292 assertEquals("After second compaction: family2 mob file count", regionNum * count,
293 countFiles(tableName, true, family2));
294 assertEquals("After second compaction: family1 del file count", 0,
295 countFiles(tableName, false, family1));
296 assertEquals("After second compaction: family2 del file count", 0,
297 countFiles(tableName, false, family2));
298 assertEquals("After second compaction: family1 hfilelink count", 0, countHFileLinks(family1));
299 assertEquals("After second compaction: family2 hfilelink count", 0, countHFileLinks(family2));
300 assertRefFileNameEqual(family1);
301 }
302
303 @Test(timeout = 300000)
304 public void testMajorCompactionFromAdmin() throws Exception {
305 resetConf();
306 int mergeSize = 5000;
307
308 conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
309 String tableNameAsString = "testMajorCompactionFromAdmin";
310 SecureRandom rng = new SecureRandom();
311 byte[] keyBytes = new byte[AES.KEY_LENGTH];
312 rng.nextBytes(keyBytes);
313 String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
314 Key cfKey = new SecretKeySpec(keyBytes, algorithm);
315 byte[] encryptionKey = EncryptionUtil.wrapKey(conf,
316 conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey);
317 TableName tableName = TableName.valueOf(tableNameAsString);
318 HTableDescriptor desc = new HTableDescriptor(tableName);
319 HColumnDescriptor hcd1 = new HColumnDescriptor(family1);
320 hcd1.setMobEnabled(true);
321 hcd1.setMobThreshold(0);
322 hcd1.setEncryptionType(algorithm);
323 hcd1.setEncryptionKey(encryptionKey);
324 HColumnDescriptor hcd2 = new HColumnDescriptor(family2);
325 hcd2.setMobEnabled(true);
326 hcd2.setMobThreshold(0);
327 desc.addFamily(hcd1);
328 desc.addFamily(hcd2);
329 admin.createTable(desc, getSplitKeys());
330 Table table = conn.getTable(tableName);
331 BufferedMutator bufMut = conn.getBufferedMutator(tableName);
332 int count = 4;
333
334 loadData(admin, bufMut, tableName, count, rowNumPerFile);
335 int rowNumPerRegion = count * rowNumPerFile;
336
337 assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion,
338 countMobRows(table));
339 assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion,
340 countMobCells(table));
341 assertEquals("Before deleting: mob file count", regionNum * count,
342 countFiles(tableName, true, family1));
343
344 createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1));
345
346 assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
347 countMobRows(table));
348 assertEquals("Before compaction: mob cells count", regionNum
349 * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
350 assertEquals("Before compaction: family1 mob file count", regionNum * count,
351 countFiles(tableName, true, family1));
352 assertEquals("Before compaction: family2 mob file count", regionNum * count,
353 countFiles(tableName, true, family2));
354 assertEquals("Before compaction: family1 del file count", regionNum,
355 countFiles(tableName, false, family1));
356 assertEquals("Before compaction: family2 del file count", regionNum,
357 countFiles(tableName, false, family2));
358
359
360 admin.majorCompact(tableName, hcd1.getName(), Admin.CompactType.MOB);
361
362 waitUntilMobCompactionFinished(tableName);
363 assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
364 countMobRows(table));
365 assertEquals("After compaction: mob cells count", regionNum
366 * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
367 assertEquals("After compaction: family1 mob file count", regionNum,
368 countFiles(tableName, true, family1));
369 assertEquals("After compaction: family2 mob file count", regionNum * count,
370 countFiles(tableName, true, family2));
371 assertEquals("After compaction: family1 del file count", 0,
372 countFiles(tableName, false, family1));
373 assertEquals("After compaction: family2 del file count", regionNum,
374 countFiles(tableName, false, family2));
375 Assert.assertTrue(verifyEncryption(tableName, family1));
376 table.close();
377 }
378
379 @Test(timeout = 300000)
380 public void testScannerOnBulkLoadRefHFiles() throws Exception {
381 resetConf();
382 setUp("testScannerOnBulkLoadRefHFiles");
383 long ts = EnvironmentEdgeManager.currentTime();
384 byte[] key0 = Bytes.toBytes("k0");
385 byte[] key1 = Bytes.toBytes("k1");
386 String value0 = "mobValue0";
387 String value1 = "mobValue1";
388 String newValue0 = "new";
389 Put put0 = new Put(key0);
390 put0.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), ts, Bytes.toBytes(value0));
391 loadData(admin, bufMut, tableName, new Put[] { put0 });
392 put0 = new Put(key0);
393 put0.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), ts, Bytes.toBytes(newValue0));
394 Put put1 = new Put(key1);
395 put1.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), ts, Bytes.toBytes(value1));
396 loadData(admin, bufMut, tableName, new Put[] { put0, put1 });
397
398 Get get = new Get(key0);
399 Result result = table.get(get);
400 Cell cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
401 assertEquals("Before compaction: mob value of k0", newValue0,
402 Bytes.toString(CellUtil.cloneValue(cell)));
403 admin.majorCompact(tableName, hcd1.getName(), Admin.CompactType.MOB);
404 waitUntilMobCompactionFinished(tableName);
405
406
407 result = table.get(get);
408 cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
409 assertEquals("After compaction: mob value of k0", newValue0,
410 Bytes.toString(CellUtil.cloneValue(cell)));
411
412 get = new Get(key1);
413 get.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(true));
414 result = table.get(get);
415 cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
416
417 Path mobFamilyPath =
418 MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, hcd1.getNameAsString());
419 List<Path> paths = new ArrayList<Path>();
420 if (fs.exists(mobFamilyPath)) {
421 FileStatus[] files = fs.listStatus(mobFamilyPath);
422 for (FileStatus file : files) {
423 if (!StoreFileInfo.isDelFile(file.getPath())) {
424 paths.add(file.getPath());
425 }
426 }
427 }
428 assertEquals("After compaction: number of mob files:", 1, paths.size());
429 assertEquals("After compaction: mob file name:", MobUtils.getMobFileName(cell), paths.get(0)
430 .getName());
431 }
432
433 @Test(timeout = 300000)
434 public void testScannerAfterCompactions() throws Exception {
435 resetConf();
436 setUp("testScannerAfterCompactions");
437 long ts = EnvironmentEdgeManager.currentTime();
438 byte[] key0 = Bytes.toBytes("k0");
439 byte[] key1 = Bytes.toBytes("k1");
440 String value = "mobValue";
441 String newValue = "new";
442 Put put0 = new Put(key0);
443 put0.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), ts, Bytes.toBytes(value));
444 loadData(admin, bufMut, tableName, new Put[] { put0 });
445 Put put1 = new Put(key1);
446 put1.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), ts, Bytes.toBytes(value));
447 loadData(admin, bufMut, tableName, new Put[] { put1 });
448 put1 = new Put(key1);
449 put1.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), ts, Bytes.toBytes(newValue));
450 loadData(admin, bufMut, tableName, new Put[] { put1 });
451 admin.majorCompact(tableName);
452 waitUntilCompactionFinished(tableName);
453 admin.majorCompact(tableName, hcd1.getName(), Admin.CompactType.MOB);
454 waitUntilMobCompactionFinished(tableName);
455
456 Get get = new Get(key1);
457 Result result = table.get(get);
458 Cell cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
459 assertEquals("After compaction: mob value", "new", Bytes.toString(CellUtil.cloneValue(cell)));
460 }
461
462 private void waitUntilCompactionFinished(TableName tableName) throws IOException,
463 InterruptedException {
464 long finished = EnvironmentEdgeManager.currentTime() + 60000;
465 CompactionState state = admin.getCompactionState(tableName);
466 while (EnvironmentEdgeManager.currentTime() < finished) {
467 if (state == CompactionState.NONE) {
468 break;
469 }
470 state = admin.getCompactionState(tableName);
471 Thread.sleep(10);
472 }
473 assertEquals(CompactionState.NONE, state);
474 }
475
476 private void waitUntilMobCompactionFinished(TableName tableName) throws IOException,
477 InterruptedException {
478 long finished = EnvironmentEdgeManager.currentTime() + 60000;
479 CompactionState state = admin.getCompactionState(tableName, Admin.CompactType.MOB);
480 while (EnvironmentEdgeManager.currentTime() < finished) {
481 if (state == CompactionState.NONE) {
482 break;
483 }
484 state = admin.getCompactionState(tableName, Admin.CompactType.MOB);
485 Thread.sleep(10);
486 }
487 assertEquals(CompactionState.NONE, state);
488 }
489
490
491
492
493
494
495 private int countMobRows(final Table table) throws IOException {
496 Scan scan = new Scan();
497
498 scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
499 return TEST_UTIL.countRows(table, scan);
500 }
501
502
503
504
505
506
507 private int countMobCells(final Table table) throws IOException {
508 Scan scan = new Scan();
509
510 scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
511 ResultScanner results = table.getScanner(scan);
512 int count = 0;
513 for (Result res : results) {
514 for (Cell cell : res.listCells()) {
515 count++;
516 }
517 }
518 results.close();
519 return count;
520 }
521
522
523
524
525
526
527
528 private int countFiles(TableName tableName, boolean isMobFile, String familyName)
529 throws IOException {
530 Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName);
531 int count = 0;
532 if (fs.exists(mobDirPath)) {
533 FileStatus[] files = fs.listStatus(mobDirPath);
534 for (FileStatus file : files) {
535 if (isMobFile == true) {
536 if (!StoreFileInfo.isDelFile(file.getPath())) {
537 count++;
538 }
539 } else {
540 if (StoreFileInfo.isDelFile(file.getPath())) {
541 count++;
542 }
543 }
544 }
545 }
546 return count;
547 }
548
549 private boolean verifyEncryption(TableName tableName, String familyName) throws IOException {
550 Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName);
551 boolean hasFiles = false;
552 if (fs.exists(mobDirPath)) {
553 FileStatus[] files = fs.listStatus(mobDirPath);
554 hasFiles = files != null && files.length > 0;
555 Assert.assertTrue(hasFiles);
556 Path path = files[0].getPath();
557 CacheConfig cacheConf = new CacheConfig(conf);
558 StoreFile sf = new StoreFile(TEST_UTIL.getTestFileSystem(), path, conf, cacheConf,
559 BloomType.NONE);
560 HFile.Reader reader = sf.createReader().getHFileReader();
561 byte[] encryptionKey = reader.getTrailer().getEncryptionKey();
562 Assert.assertTrue(null != encryptionKey);
563 Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName()
564 .equals(HConstants.CIPHER_AES));
565 }
566 return hasFiles;
567 }
568
569
570
571
572
573
574 private int countHFileLinks(String familyName) throws IOException {
575 Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName);
576 int count = 0;
577 if (fs.exists(mobDirPath)) {
578 FileStatus[] files = fs.listStatus(mobDirPath);
579 for (FileStatus file : files) {
580 if (HFileLink.isHFileLink(file.getPath())) {
581 count++;
582 }
583 }
584 }
585 return count;
586 }
587
588
589
590
591
592
593
594
595 private int countLargeFiles(int size, TableName tableName, String familyName) throws IOException {
596 Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName);
597 int count = 0;
598 if (fs.exists(mobDirPath)) {
599 FileStatus[] files = fs.listStatus(mobDirPath);
600 for (FileStatus file : files) {
601
602 if ((!StoreFileInfo.isDelFile(file.getPath())) && (file.getLen() > size)) {
603 count++;
604 }
605 }
606 }
607 return count;
608 }
609
610
611
612
613 private void loadData(Admin admin, BufferedMutator table, TableName tableName, int fileNum,
614 int rowNumPerFile) throws IOException, InterruptedException {
615 if (fileNum <= 0) {
616 throw new IllegalArgumentException();
617 }
618 for (int i = 0; i < fileNum * rowNumPerFile; i++) {
619 for (byte k0 : KEYS) {
620 byte[] k = new byte[] { k0 };
621 byte[] key = Bytes.add(k, Bytes.toBytes(i));
622 byte[] mobVal = makeDummyData(10 * (i + 1));
623 Put put = new Put(key);
624 put.setDurability(Durability.SKIP_WAL);
625 put.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), mobVal);
626 put.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf2), mobVal);
627 put.addColumn(Bytes.toBytes(family2), Bytes.toBytes(qf1), mobVal);
628 table.mutate(put);
629 }
630 if ((i + 1) % rowNumPerFile == 0) {
631 table.flush();
632 admin.flush(tableName);
633 }
634 }
635 }
636
637 private void loadData(Admin admin, BufferedMutator table, TableName tableName, Put[] puts)
638 throws IOException {
639 table.mutate(Arrays.asList(puts));
640 table.flush();
641 admin.flush(tableName);
642 }
643
644
645
646
647 private void createDelFile(Table table, TableName tableName, byte[] family, byte[] qf)
648 throws IOException, InterruptedException {
649 for (byte k0 : KEYS) {
650 byte[] k = new byte[] { k0 };
651
652 byte[] key1 = Bytes.add(k, Bytes.toBytes(0));
653 Delete delete1 = new Delete(key1);
654 delete1.addFamily(family);
655 table.delete(delete1);
656
657 byte[] key2 = Bytes.add(k, Bytes.toBytes(2));
658 Delete delete2 = new Delete(key2);
659 table.delete(delete2);
660
661 byte[] key3 = Bytes.add(k, Bytes.toBytes(4));
662 Delete delete3 = new Delete(key3);
663 delete3.addColumn(family, qf);
664 table.delete(delete3);
665 }
666 admin.flush(tableName);
667 List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
668 for (HRegion region : regions) {
669 region.waitForFlushesAndCompactions();
670 region.compact(true);
671 }
672 }
673
674
675
676
677
678 private byte[] makeDummyData(int size) {
679 byte[] dummyData = new byte[size];
680 new Random().nextBytes(dummyData);
681 return dummyData;
682 }
683
684
685
686
687 private byte[][] getSplitKeys() {
688 byte[][] splitKeys = new byte[KEYS.length - 1][];
689 for (int i = 0; i < splitKeys.length; ++i) {
690 splitKeys[i] = new byte[] { KEYS[i + 1] };
691 }
692 return splitKeys;
693 }
694
695 private static ExecutorService createThreadPool(Configuration conf) {
696 int maxThreads = 10;
697 long keepAliveTime = 60;
698 final SynchronousQueue<Runnable> queue = new SynchronousQueue<Runnable>();
699 ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads,
700 keepAliveTime, TimeUnit.SECONDS, queue,
701 Threads.newDaemonThreadFactory("MobFileCompactionChore"),
702 new RejectedExecutionHandler() {
703 @Override
704 public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
705 try {
706
707 queue.put(r);
708 } catch (InterruptedException e) {
709 throw new RejectedExecutionException(e);
710 }
711 }
712 });
713 ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true);
714 return pool;
715 }
716
717 private void assertRefFileNameEqual(String familyName) throws IOException {
718 Scan scan = new Scan();
719 scan.addFamily(Bytes.toBytes(familyName));
720
721 scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
722 ResultScanner results = table.getScanner(scan);
723 Path mobFamilyPath = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(),
724 tableName, familyName);
725 List<Path> actualFilePaths = new ArrayList<>();
726 List<Path> expectFilePaths = new ArrayList<>();
727 for (Result res : results) {
728 for (Cell cell : res.listCells()) {
729 byte[] referenceValue = CellUtil.cloneValue(cell);
730 String fileName = Bytes.toString(referenceValue, Bytes.SIZEOF_INT,
731 referenceValue.length - Bytes.SIZEOF_INT);
732 Path targetPath = new Path(mobFamilyPath, fileName);
733 if(!actualFilePaths.contains(targetPath)) {
734 actualFilePaths.add(targetPath);
735 }
736 }
737 }
738 results.close();
739 if (fs.exists(mobFamilyPath)) {
740 FileStatus[] files = fs.listStatus(mobFamilyPath);
741 for (FileStatus file : files) {
742 if (!StoreFileInfo.isDelFile(file.getPath())) {
743 expectFilePaths.add(file.getPath());
744 }
745 }
746 }
747 Collections.sort(actualFilePaths);
748 Collections.sort(expectFilePaths);
749 assertEquals(expectFilePaths, actualFilePaths);
750 }
751
752
753
754
755 private void resetConf() {
756 conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD,
757 MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD);
758 conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE,
759 MobConstants.DEFAULT_MOB_COMPACTION_BATCH_SIZE);
760 }
761 }