1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mob.compactions;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.Collections;
25 import java.util.Date;
26 import java.util.List;
27 import java.util.Random;
28 import java.util.UUID;
29 import java.util.concurrent.ExecutorService;
30 import java.util.concurrent.RejectedExecutionException;
31 import java.util.concurrent.RejectedExecutionHandler;
32 import java.util.concurrent.SynchronousQueue;
33 import java.util.concurrent.ThreadPoolExecutor;
34 import java.util.concurrent.TimeUnit;
35
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileStatus;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.*;
41 import org.apache.hadoop.hbase.KeyValue.Type;
42 import org.apache.hadoop.hbase.regionserver.*;
43 import org.apache.hadoop.hbase.testclassification.LargeTests;
44 import org.apache.hadoop.hbase.client.Scan;
45 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
46 import org.apache.hadoop.hbase.io.hfile.HFileContext;
47 import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
48 import org.apache.hadoop.hbase.mob.MobConstants;
49 import org.apache.hadoop.hbase.mob.MobFileName;
50 import org.apache.hadoop.hbase.mob.MobUtils;
51 import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactionRequest;
52 import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactor;
53 import org.apache.hadoop.hbase.mob.compactions.MobCompactionRequest.CompactionType;
54 import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactionRequest.CompactionPartition;
55 import org.apache.hadoop.hbase.util.Bytes;
56 import org.apache.hadoop.hbase.util.FSUtils;
57 import org.apache.hadoop.hbase.util.Threads;
58 import org.junit.AfterClass;
59 import org.junit.Assert;
60 import org.junit.BeforeClass;
61 import org.junit.Test;
62 import org.junit.experimental.categories.Category;
63
64 @Category(LargeTests.class)
65 public class TestPartitionedMobCompactor {
66 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
67 private final static String family = "family";
68 private final static String qf = "qf";
69 private HColumnDescriptor hcd = new HColumnDescriptor(family);
70 private Configuration conf = TEST_UTIL.getConfiguration();
71 private CacheConfig cacheConf = new CacheConfig(conf);
72 private FileSystem fs;
73 private List<FileStatus> mobFiles = new ArrayList<>();
74 private List<FileStatus> delFiles = new ArrayList<>();
75 private List<FileStatus> allFiles = new ArrayList<>();
76 private Path basePath;
77 private String mobSuffix;
78 private String delSuffix;
79 private static ExecutorService pool;
80
81 @BeforeClass
82 public static void setUpBeforeClass() throws Exception {
83 TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
84 TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
85 TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3);
86 TEST_UTIL.startMiniCluster(1);
87 pool = createThreadPool();
88 }
89
90 @AfterClass
91 public static void tearDownAfterClass() throws Exception {
92 pool.shutdown();
93 TEST_UTIL.shutdownMiniCluster();
94 }
95
96 private void init(String tableName) throws Exception {
97 fs = FileSystem.get(conf);
98 Path testDir = FSUtils.getRootDir(conf);
99 Path mobTestDir = new Path(testDir, MobConstants.MOB_DIR_NAME);
100 basePath = new Path(new Path(mobTestDir, tableName), family);
101 mobSuffix = UUID.randomUUID().toString().replaceAll("-", "");
102 delSuffix = UUID.randomUUID().toString().replaceAll("-", "") + "_del";
103 }
104
105 @Test
106 public void testCompactionSelectWithAllFiles() throws Exception {
107 resetConf();
108 String tableName = "testCompactionSelectWithAllFiles";
109 init(tableName);
110 int count = 10;
111
112 createStoreFiles(basePath, family, qf, count, Type.Put);
113
114 createStoreFiles(basePath, family, qf, count, Type.Delete);
115 listFiles();
116 long mergeSize = MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD;
117 List<String> expectedStartKeys = new ArrayList<>();
118 for(FileStatus file : mobFiles) {
119 if(file.getLen() < mergeSize) {
120 String fileName = file.getPath().getName();
121 String startKey = fileName.substring(0, 32);
122 expectedStartKeys.add(startKey);
123 }
124 }
125 testSelectFiles(tableName, CompactionType.ALL_FILES, false, expectedStartKeys);
126 }
127
128 @Test
129 public void testCompactionSelectWithPartFiles() throws Exception {
130 resetConf();
131 String tableName = "testCompactionSelectWithPartFiles";
132 init(tableName);
133 int count = 10;
134
135 createStoreFiles(basePath, family, qf, count, Type.Put);
136
137 createStoreFiles(basePath, family, qf, count, Type.Delete);
138 listFiles();
139 long mergeSize = 4000;
140 List<String> expectedStartKeys = new ArrayList<>();
141 for(FileStatus file : mobFiles) {
142 if(file.getLen() < 4000) {
143 String fileName = file.getPath().getName();
144 String startKey = fileName.substring(0, 32);
145 expectedStartKeys.add(startKey);
146 }
147 }
148
149 conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
150 testSelectFiles(tableName, CompactionType.PART_FILES, false, expectedStartKeys);
151 }
152
153 @Test
154 public void testCompactionSelectWithForceAllFiles() throws Exception {
155 resetConf();
156 String tableName = "testCompactionSelectWithForceAllFiles";
157 init(tableName);
158 int count = 10;
159
160 createStoreFiles(basePath, family, qf, count, Type.Put);
161
162 createStoreFiles(basePath, family, qf, count, Type.Delete);
163 listFiles();
164 long mergeSize = 4000;
165 List<String> expectedStartKeys = new ArrayList<>();
166 for(FileStatus file : mobFiles) {
167 String fileName = file.getPath().getName();
168 String startKey = fileName.substring(0, 32);
169 expectedStartKeys.add(startKey);
170 }
171
172 conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
173 testSelectFiles(tableName, CompactionType.ALL_FILES, true, expectedStartKeys);
174 }
175
176 @Test
177 public void testCompactDelFilesWithDefaultBatchSize() throws Exception {
178 resetConf();
179 String tableName = "testCompactDelFilesWithDefaultBatchSize";
180 init(tableName);
181
182 createStoreFiles(basePath, family, qf, 20, Type.Put);
183
184 createStoreFiles(basePath, family, qf, 13, Type.Delete);
185 listFiles();
186 testCompactDelFiles(tableName, 1, 13, false);
187 }
188
189 @Test
190 public void testCompactDelFilesWithSmallBatchSize() throws Exception {
191 resetConf();
192 String tableName = "testCompactDelFilesWithSmallBatchSize";
193 init(tableName);
194
195 createStoreFiles(basePath, family, qf, 20, Type.Put);
196
197 createStoreFiles(basePath, family, qf, 13, Type.Delete);
198 listFiles();
199
200
201 conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, 4);
202 testCompactDelFiles(tableName, 1, 13, false);
203 }
204
205 @Test
206 public void testCompactDelFilesChangeMaxDelFileCount() throws Exception {
207 resetConf();
208 String tableName = "testCompactDelFilesWithSmallBatchSize";
209 init(tableName);
210
211 createStoreFiles(basePath, family, qf, 20, Type.Put);
212
213 createStoreFiles(basePath, family, qf, 13, Type.Delete);
214 listFiles();
215
216
217 conf.setInt(MobConstants.MOB_DELFILE_MAX_COUNT, 5);
218
219 conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, 2);
220 testCompactDelFiles(tableName, 4, 13, false);
221 }
222
223
224
225
226
227
228
229
230 private void testSelectFiles(String tableName, final CompactionType type,
231 final boolean isForceAllFiles, final List<String> expected) throws IOException {
232 PartitionedMobCompactor compactor = new PartitionedMobCompactor(conf, fs,
233 TableName.valueOf(tableName), hcd, pool) {
234 @Override
235 public List<Path> compact(List<FileStatus> files, boolean isForceAllFiles)
236 throws IOException {
237 if (files == null || files.isEmpty()) {
238 return null;
239 }
240 PartitionedMobCompactionRequest request = select(files, isForceAllFiles);
241
242 Assert.assertEquals(type, request.type);
243
244 compareCompactedPartitions(expected, request.compactionPartitions);
245
246 compareDelFiles(request.delFiles);
247 return null;
248 }
249 };
250 compactor.compact(allFiles, isForceAllFiles);
251 }
252
253
254
255
256
257
258
259
260 private void testCompactDelFiles(String tableName, final int expectedFileCount,
261 final int expectedCellCount, boolean isForceAllFiles) throws IOException {
262 PartitionedMobCompactor compactor = new PartitionedMobCompactor(conf, fs,
263 TableName.valueOf(tableName), hcd, pool) {
264 @Override
265 protected List<Path> performCompaction(PartitionedMobCompactionRequest request)
266 throws IOException {
267 List<Path> delFilePaths = new ArrayList<Path>();
268 for (FileStatus delFile : request.delFiles) {
269 delFilePaths.add(delFile.getPath());
270 }
271 List<Path> newDelPaths = compactDelFiles(request, delFilePaths);
272
273 Assert.assertEquals(expectedFileCount, newDelPaths.size());
274 Assert.assertEquals(expectedCellCount, countDelCellsInDelFiles(newDelPaths));
275 return null;
276 }
277 };
278 compactor.compact(allFiles, isForceAllFiles);
279 }
280
281
282
283
284 private void listFiles() throws IOException {
285 for (FileStatus file : fs.listStatus(basePath)) {
286 allFiles.add(file);
287 if (file.getPath().getName().endsWith("_del")) {
288 delFiles.add(file);
289 } else {
290 mobFiles.add(file);
291 }
292 }
293 }
294
295
296
297
298
299 private void compareCompactedPartitions(List<String> expected,
300 Collection<CompactionPartition> partitions) {
301 List<String> actualKeys = new ArrayList<>();
302 for (CompactionPartition partition : partitions) {
303 actualKeys.add(partition.getPartitionId().getStartKey());
304 }
305 Collections.sort(expected);
306 Collections.sort(actualKeys);
307 Assert.assertEquals(expected.size(), actualKeys.size());
308 for (int i = 0; i < expected.size(); i++) {
309 Assert.assertEquals(expected.get(i), actualKeys.get(i));
310 }
311 }
312
313
314
315
316
317 private void compareDelFiles(Collection<FileStatus> allDelFiles) {
318 int i = 0;
319 for (FileStatus file : allDelFiles) {
320 Assert.assertEquals(delFiles.get(i), file);
321 i++;
322 }
323 }
324
325
326
327
328
329
330
331
332
333 private void createStoreFiles(Path basePath, String family, String qualifier, int count,
334 Type type) throws IOException {
335 HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
336 String startKey = "row_";
337 MobFileName mobFileName = null;
338 for (int i = 0; i < count; i++) {
339 byte[] startRow = Bytes.toBytes(startKey + i) ;
340 if(type.equals(Type.Delete)) {
341 mobFileName = MobFileName.create(startRow, MobUtils.formatDate(
342 new Date()), delSuffix);
343 }
344 if(type.equals(Type.Put)){
345 mobFileName = MobFileName.create(Bytes.toBytes(startKey + i), MobUtils.formatDate(
346 new Date()), mobSuffix);
347 }
348 StoreFile.Writer mobFileWriter = new StoreFile.WriterBuilder(conf, cacheConf, fs)
349 .withFileContext(meta).withFilePath(new Path(basePath, mobFileName.getFileName())).build();
350 writeStoreFile(mobFileWriter, startRow, Bytes.toBytes(family), Bytes.toBytes(qualifier),
351 type, (i+1)*1000);
352 }
353 }
354
355
356
357
358
359
360
361
362
363
364 private static void writeStoreFile(final StoreFile.Writer writer, byte[]row, byte[] family,
365 byte[] qualifier, Type type, int size) throws IOException {
366 long now = System.currentTimeMillis();
367 try {
368 byte[] dummyData = new byte[size];
369 new Random().nextBytes(dummyData);
370 writer.append(new KeyValue(row, family, qualifier, now, type, dummyData));
371 } finally {
372 writer.close();
373 }
374 }
375
376
377
378
379
380
381 private int countDelCellsInDelFiles(List<Path> paths) throws IOException {
382 List<StoreFile> sfs = new ArrayList<StoreFile>();
383 int size = 0;
384 for(Path path : paths) {
385 StoreFile sf = new StoreFile(fs, path, conf, cacheConf, BloomType.NONE);
386 sfs.add(sf);
387 }
388 List scanners = StoreFileScanner.getScannersForStoreFiles(sfs, false, true,
389 false, null, HConstants.LATEST_TIMESTAMP);
390 Scan scan = new Scan();
391 scan.setMaxVersions(hcd.getMaxVersions());
392 long timeToPurgeDeletes = Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
393 long ttl = HStore.determineTTLFromFamily(hcd);
394 ScanInfo scanInfo = new ScanInfo(hcd, ttl, timeToPurgeDeletes, KeyValue.COMPARATOR);
395 StoreScanner scanner = new StoreScanner(scan, scanInfo, ScanType.COMPACT_RETAIN_DELETES, null,
396 scanners, 0L, HConstants.LATEST_TIMESTAMP);
397 List<Cell> results = new ArrayList<>();
398 boolean hasMore = true;
399
400 while (hasMore) {
401 hasMore = scanner.next(results);
402 size += results.size();
403 results.clear();
404 }
405 scanner.close();
406 return size;
407 }
408
409 private static ExecutorService createThreadPool() {
410 int maxThreads = 10;
411 long keepAliveTime = 60;
412 final SynchronousQueue<Runnable> queue = new SynchronousQueue<Runnable>();
413 ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads, keepAliveTime,
414 TimeUnit.SECONDS, queue, Threads.newDaemonThreadFactory("MobFileCompactionChore"),
415 new RejectedExecutionHandler() {
416 @Override
417 public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
418 try {
419
420 queue.put(r);
421 } catch (InterruptedException e) {
422 throw new RejectedExecutionException(e);
423 }
424 }
425 });
426 ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true);
427 return pool;
428 }
429
430
431
432
433 private void resetConf() {
434 conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD,
435 MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD);
436 conf.setInt(MobConstants.MOB_DELFILE_MAX_COUNT, MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT);
437 conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE,
438 MobConstants.DEFAULT_MOB_COMPACTION_BATCH_SIZE);
439 }
440 }