View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.mob.compactions;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.Collection;
24  import java.util.Collections;
25  import java.util.Date;
26  import java.util.List;
27  import java.util.Random;
28  import java.util.UUID;
29  import java.util.concurrent.ExecutorService;
30  import java.util.concurrent.RejectedExecutionException;
31  import java.util.concurrent.RejectedExecutionHandler;
32  import java.util.concurrent.SynchronousQueue;
33  import java.util.concurrent.ThreadPoolExecutor;
34  import java.util.concurrent.TimeUnit;
35  
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FileStatus;
38  import org.apache.hadoop.fs.FileSystem;
39  import org.apache.hadoop.fs.Path;
40  import org.apache.hadoop.hbase.*;
41  import org.apache.hadoop.hbase.KeyValue.Type;
42  import org.apache.hadoop.hbase.regionserver.*;
43  import org.apache.hadoop.hbase.testclassification.LargeTests;
44  import org.apache.hadoop.hbase.client.Scan;
45  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
46  import org.apache.hadoop.hbase.io.hfile.HFileContext;
47  import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
48  import org.apache.hadoop.hbase.mob.MobConstants;
49  import org.apache.hadoop.hbase.mob.MobFileName;
50  import org.apache.hadoop.hbase.mob.MobUtils;
51  import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactionRequest;
52  import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactor;
53  import org.apache.hadoop.hbase.mob.compactions.MobCompactionRequest.CompactionType;
54  import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactionRequest.CompactionPartition;
55  import org.apache.hadoop.hbase.util.Bytes;
56  import org.apache.hadoop.hbase.util.FSUtils;
57  import org.apache.hadoop.hbase.util.Threads;
58  import org.junit.AfterClass;
59  import org.junit.Assert;
60  import org.junit.BeforeClass;
61  import org.junit.Test;
62  import org.junit.experimental.categories.Category;
63  
64  @Category(LargeTests.class)
65  public class TestPartitionedMobCompactor {
66    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
67    private final static String family = "family";
68    private final static String qf = "qf";
69    private HColumnDescriptor hcd = new HColumnDescriptor(family);
70    private Configuration conf = TEST_UTIL.getConfiguration();
71    private CacheConfig cacheConf = new CacheConfig(conf);
72    private FileSystem fs;
73    private List<FileStatus> mobFiles = new ArrayList<>();
74    private List<FileStatus> delFiles = new ArrayList<>();
75    private List<FileStatus> allFiles = new ArrayList<>();
76    private Path basePath;
77    private String mobSuffix;
78    private String delSuffix;
79    private static ExecutorService pool;
80  
81    @BeforeClass
82    public static void setUpBeforeClass() throws Exception {
83      TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
84      TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
85      TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3);
86      TEST_UTIL.startMiniCluster(1);
87      pool = createThreadPool();
88    }
89  
90    @AfterClass
91    public static void tearDownAfterClass() throws Exception {
92      pool.shutdown();
93      TEST_UTIL.shutdownMiniCluster();
94    }
95  
96    private void init(String tableName) throws Exception {
97      fs = FileSystem.get(conf);
98      Path testDir = FSUtils.getRootDir(conf);
99      Path mobTestDir = new Path(testDir, MobConstants.MOB_DIR_NAME);
100     basePath = new Path(new Path(mobTestDir, tableName), family);
101     mobSuffix = UUID.randomUUID().toString().replaceAll("-", "");
102     delSuffix = UUID.randomUUID().toString().replaceAll("-", "") + "_del";
103   }
104 
105   @Test
106   public void testCompactionSelectWithAllFiles() throws Exception {
107     resetConf();
108     String tableName = "testCompactionSelectWithAllFiles";
109     init(tableName);
110     int count = 10;
111     // create 10 mob files.
112     createStoreFiles(basePath, family, qf, count, Type.Put);
113     // create 10 del files
114     createStoreFiles(basePath, family, qf, count, Type.Delete);
115     listFiles();
116     long mergeSize = MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD;
117     List<String> expectedStartKeys = new ArrayList<>();
118     for(FileStatus file : mobFiles) {
119       if(file.getLen() < mergeSize) {
120         String fileName = file.getPath().getName();
121         String startKey = fileName.substring(0, 32);
122         expectedStartKeys.add(startKey);
123       }
124     }
125     testSelectFiles(tableName, CompactionType.ALL_FILES, false, expectedStartKeys);
126   }
127 
128   @Test
129   public void testCompactionSelectWithPartFiles() throws Exception {
130     resetConf();
131     String tableName = "testCompactionSelectWithPartFiles";
132     init(tableName);
133     int count = 10;
134     // create 10 mob files.
135     createStoreFiles(basePath, family, qf, count, Type.Put);
136     // create 10 del files
137     createStoreFiles(basePath, family, qf, count, Type.Delete);
138     listFiles();
139     long mergeSize = 4000;
140     List<String> expectedStartKeys = new ArrayList<>();
141     for(FileStatus file : mobFiles) {
142       if(file.getLen() < 4000) {
143         String fileName = file.getPath().getName();
144         String startKey = fileName.substring(0, 32);
145         expectedStartKeys.add(startKey);
146       }
147     }
148     // set the mob compaction mergeable threshold
149     conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
150     testSelectFiles(tableName, CompactionType.PART_FILES, false, expectedStartKeys);
151   }
152 
153   @Test
154   public void testCompactionSelectWithForceAllFiles() throws Exception {
155     resetConf();
156     String tableName = "testCompactionSelectWithForceAllFiles";
157     init(tableName);
158     int count = 10;
159     // create 10 mob files.
160     createStoreFiles(basePath, family, qf, count, Type.Put);
161     // create 10 del files
162     createStoreFiles(basePath, family, qf, count, Type.Delete);
163     listFiles();
164     long mergeSize = 4000;
165     List<String> expectedStartKeys = new ArrayList<>();
166     for(FileStatus file : mobFiles) {
167       String fileName = file.getPath().getName();
168       String startKey = fileName.substring(0, 32);
169       expectedStartKeys.add(startKey);
170     }
171     // set the mob compaction mergeable threshold
172     conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
173     testSelectFiles(tableName, CompactionType.ALL_FILES, true, expectedStartKeys);
174   }
175 
176   @Test
177   public void testCompactDelFilesWithDefaultBatchSize() throws Exception {
178     resetConf();
179     String tableName = "testCompactDelFilesWithDefaultBatchSize";
180     init(tableName);
181     // create 20 mob files.
182     createStoreFiles(basePath, family, qf, 20, Type.Put);
183     // create 13 del files
184     createStoreFiles(basePath, family, qf, 13, Type.Delete);
185     listFiles();
186     testCompactDelFiles(tableName, 1, 13, false);
187   }
188 
189   @Test
190   public void testCompactDelFilesWithSmallBatchSize() throws Exception {
191     resetConf();
192     String tableName = "testCompactDelFilesWithSmallBatchSize";
193     init(tableName);
194     // create 20 mob files.
195     createStoreFiles(basePath, family, qf, 20, Type.Put);
196     // create 13 del files
197     createStoreFiles(basePath, family, qf, 13, Type.Delete);
198     listFiles();
199 
200     // set the mob compaction batch size
201     conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, 4);
202     testCompactDelFiles(tableName, 1, 13, false);
203   }
204 
205   @Test
206   public void testCompactDelFilesChangeMaxDelFileCount() throws Exception {
207     resetConf();
208     String tableName = "testCompactDelFilesWithSmallBatchSize";
209     init(tableName);
210     // create 20 mob files.
211     createStoreFiles(basePath, family, qf, 20, Type.Put);
212     // create 13 del files
213     createStoreFiles(basePath, family, qf, 13, Type.Delete);
214     listFiles();
215 
216     // set the max del file count
217     conf.setInt(MobConstants.MOB_DELFILE_MAX_COUNT, 5);
218     // set the mob compaction batch size
219     conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, 2);
220     testCompactDelFiles(tableName, 4, 13, false);
221   }
222 
223   /**
224    * Tests the selectFiles
225    * @param tableName the table name
226    * @param type the expected compaction type
227    * @param isForceAllFiles whether all the mob files are selected
228    * @param expected the expected start keys
229    */
230   private void testSelectFiles(String tableName, final CompactionType type,
231     final boolean isForceAllFiles, final List<String> expected) throws IOException {
232     PartitionedMobCompactor compactor = new PartitionedMobCompactor(conf, fs,
233       TableName.valueOf(tableName), hcd, pool) {
234       @Override
235       public List<Path> compact(List<FileStatus> files, boolean isForceAllFiles)
236         throws IOException {
237         if (files == null || files.isEmpty()) {
238           return null;
239         }
240         PartitionedMobCompactionRequest request = select(files, isForceAllFiles);
241         // assert the compaction type
242         Assert.assertEquals(type, request.type);
243         // assert get the right partitions
244         compareCompactedPartitions(expected, request.compactionPartitions);
245         // assert get the right del files
246         compareDelFiles(request.delFiles);
247         return null;
248       }
249     };
250     compactor.compact(allFiles, isForceAllFiles);
251   }
252 
253   /**
254    * Tests the compacteDelFile
255    * @param tableName the table name
256    * @param expectedFileCount the expected file count
257    * @param expectedCellCount the expected cell count
258    * @param isForceAllFiles whether all the mob files are selected
259    */
260   private void testCompactDelFiles(String tableName, final int expectedFileCount,
261       final int expectedCellCount, boolean isForceAllFiles) throws IOException {
262     PartitionedMobCompactor compactor = new PartitionedMobCompactor(conf, fs,
263       TableName.valueOf(tableName), hcd, pool) {
264       @Override
265       protected List<Path> performCompaction(PartitionedMobCompactionRequest request)
266           throws IOException {
267         List<Path> delFilePaths = new ArrayList<Path>();
268         for (FileStatus delFile : request.delFiles) {
269           delFilePaths.add(delFile.getPath());
270         }
271         List<Path> newDelPaths = compactDelFiles(request, delFilePaths);
272         // assert the del files are merged.
273         Assert.assertEquals(expectedFileCount, newDelPaths.size());
274         Assert.assertEquals(expectedCellCount, countDelCellsInDelFiles(newDelPaths));
275         return null;
276       }
277     };
278     compactor.compact(allFiles, isForceAllFiles);
279   }
280 
281   /**
282    * Lists the files in the path
283    */
284   private void listFiles() throws IOException {
285     for (FileStatus file : fs.listStatus(basePath)) {
286       allFiles.add(file);
287       if (file.getPath().getName().endsWith("_del")) {
288         delFiles.add(file);
289       } else {
290         mobFiles.add(file);
291       }
292     }
293   }
294 
295   /**
296    * Compares the compacted partitions.
297    * @param partitions the collection of CompactedPartitions
298    */
299   private void compareCompactedPartitions(List<String> expected,
300       Collection<CompactionPartition> partitions) {
301     List<String> actualKeys = new ArrayList<>();
302     for (CompactionPartition partition : partitions) {
303       actualKeys.add(partition.getPartitionId().getStartKey());
304     }
305     Collections.sort(expected);
306     Collections.sort(actualKeys);
307     Assert.assertEquals(expected.size(), actualKeys.size());
308     for (int i = 0; i < expected.size(); i++) {
309       Assert.assertEquals(expected.get(i), actualKeys.get(i));
310     }
311   }
312 
313   /**
314    * Compares the del files.
315    * @param allDelFiles all the del files
316    */
317   private void compareDelFiles(Collection<FileStatus> allDelFiles) {
318     int i = 0;
319     for (FileStatus file : allDelFiles) {
320       Assert.assertEquals(delFiles.get(i), file);
321       i++;
322     }
323   }
324 
325   /**
326    * Creates store files.
327    * @param basePath the path to create file
328    * @family the family name
329    * @qualifier the column qualifier
330    * @count the store file number
331    * @type the key type
332    */
333   private void createStoreFiles(Path basePath, String family, String qualifier, int count,
334       Type type) throws IOException {
335     HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
336     String startKey = "row_";
337     MobFileName mobFileName = null;
338     for (int i = 0; i < count; i++) {
339       byte[] startRow = Bytes.toBytes(startKey + i) ;
340       if(type.equals(Type.Delete)) {
341         mobFileName = MobFileName.create(startRow, MobUtils.formatDate(
342             new Date()), delSuffix);
343       }
344       if(type.equals(Type.Put)){
345         mobFileName = MobFileName.create(Bytes.toBytes(startKey + i), MobUtils.formatDate(
346             new Date()), mobSuffix);
347       }
348       StoreFile.Writer mobFileWriter = new StoreFile.WriterBuilder(conf, cacheConf, fs)
349       .withFileContext(meta).withFilePath(new Path(basePath, mobFileName.getFileName())).build();
350       writeStoreFile(mobFileWriter, startRow, Bytes.toBytes(family), Bytes.toBytes(qualifier),
351           type, (i+1)*1000);
352     }
353   }
354 
355   /**
356    * Writes data to store file.
357    * @param writer the store file writer
358    * @param row the row key
359    * @param family the family name
360    * @param qualifier the column qualifier
361    * @param type the key type
362    * @param size the size of value
363    */
364   private static void writeStoreFile(final StoreFile.Writer writer, byte[]row, byte[] family,
365       byte[] qualifier, Type type, int size) throws IOException {
366     long now = System.currentTimeMillis();
367     try {
368       byte[] dummyData = new byte[size];
369       new Random().nextBytes(dummyData);
370       writer.append(new KeyValue(row, family, qualifier, now, type, dummyData));
371     } finally {
372       writer.close();
373     }
374   }
375 
376   /**
377    * Gets the number of del cell in the del files
378    * @param paths the del file paths
379    * @return the cell size
380    */
381   private int countDelCellsInDelFiles(List<Path> paths) throws IOException {
382     List<StoreFile> sfs = new ArrayList<StoreFile>();
383     int size = 0;
384     for(Path path : paths) {
385       StoreFile sf = new StoreFile(fs, path, conf, cacheConf, BloomType.NONE);
386       sfs.add(sf);
387     }
388     List scanners = StoreFileScanner.getScannersForStoreFiles(sfs, false, true,
389         false, null, HConstants.LATEST_TIMESTAMP);
390     Scan scan = new Scan();
391     scan.setMaxVersions(hcd.getMaxVersions());
392     long timeToPurgeDeletes = Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
393     long ttl = HStore.determineTTLFromFamily(hcd);
394     ScanInfo scanInfo = new ScanInfo(hcd, ttl, timeToPurgeDeletes, KeyValue.COMPARATOR);
395     StoreScanner scanner = new StoreScanner(scan, scanInfo, ScanType.COMPACT_RETAIN_DELETES, null,
396         scanners, 0L, HConstants.LATEST_TIMESTAMP);
397     List<Cell> results = new ArrayList<>();
398     boolean hasMore = true;
399 
400     while (hasMore) {
401       hasMore = scanner.next(results);
402       size += results.size();
403       results.clear();
404     }
405     scanner.close();
406     return size;
407   }
408 
409   private static ExecutorService createThreadPool() {
410     int maxThreads = 10;
411     long keepAliveTime = 60;
412     final SynchronousQueue<Runnable> queue = new SynchronousQueue<Runnable>();
413     ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads, keepAliveTime,
414       TimeUnit.SECONDS, queue, Threads.newDaemonThreadFactory("MobFileCompactionChore"),
415       new RejectedExecutionHandler() {
416         @Override
417         public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
418           try {
419             // waiting for a thread to pick up instead of throwing exceptions.
420             queue.put(r);
421           } catch (InterruptedException e) {
422             throw new RejectedExecutionException(e);
423           }
424         }
425       });
426     ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true);
427     return pool;
428   }
429 
430   /**
431    * Resets the configuration.
432    */
433   private void resetConf() {
434     conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD,
435       MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD);
436     conf.setInt(MobConstants.MOB_DELFILE_MAX_COUNT, MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT);
437     conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE,
438       MobConstants.DEFAULT_MOB_COMPACTION_BATCH_SIZE);
439   }
440 }