1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import com.google.common.annotations.VisibleForTesting;
22 import com.google.common.base.Function;
23 import com.google.common.base.Preconditions;
24 import com.google.common.collect.ImmutableList;
25 import com.google.common.collect.Ordering;
26
27 import java.io.DataInput;
28 import java.io.IOException;
29 import java.net.InetSocketAddress;
30 import java.nio.ByteBuffer;
31 import java.util.Arrays;
32 import java.util.Collection;
33 import java.util.Collections;
34 import java.util.Comparator;
35 import java.util.Map;
36 import java.util.SortedSet;
37 import java.util.UUID;
38 import java.util.concurrent.atomic.AtomicBoolean;
39
40 import org.apache.commons.logging.Log;
41 import org.apache.commons.logging.LogFactory;
42 import org.apache.hadoop.conf.Configuration;
43 import org.apache.hadoop.fs.FileSystem;
44 import org.apache.hadoop.fs.Path;
45 import org.apache.hadoop.hbase.Cell;
46 import org.apache.hadoop.hbase.CellUtil;
47 import org.apache.hadoop.hbase.HConstants;
48 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
49 import org.apache.hadoop.hbase.KeyValue;
50 import org.apache.hadoop.hbase.KeyValue.KVComparator;
51 import org.apache.hadoop.hbase.KeyValueUtil;
52 import org.apache.hadoop.hbase.classification.InterfaceAudience;
53 import org.apache.hadoop.hbase.client.Scan;
54 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
55 import org.apache.hadoop.hbase.io.hfile.BlockType;
56 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
57 import org.apache.hadoop.hbase.io.hfile.HFile;
58 import org.apache.hadoop.hbase.io.hfile.HFileContext;
59 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
60 import org.apache.hadoop.hbase.io.hfile.HFileWriterV2;
61 import org.apache.hadoop.hbase.regionserver.compactions.Compactor;
62 import org.apache.hadoop.hbase.util.BloomFilter;
63 import org.apache.hadoop.hbase.util.BloomFilterFactory;
64 import org.apache.hadoop.hbase.util.BloomFilterWriter;
65 import org.apache.hadoop.hbase.util.Bytes;
66 import org.apache.hadoop.hbase.util.Writables;
67 import org.apache.hadoop.io.WritableUtils;
68
69 import com.google.common.base.Function;
70 import com.google.common.base.Preconditions;
71 import com.google.common.collect.ImmutableList;
72 import com.google.common.collect.Ordering;
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87 @InterfaceAudience.LimitedPrivate("Coprocessor")
88 public class StoreFile {
89 static final Log LOG = LogFactory.getLog(StoreFile.class.getName());
90
91
92
93
94 public static final byte [] MAX_SEQ_ID_KEY = Bytes.toBytes("MAX_SEQ_ID_KEY");
95
96
97 public static final byte[] MAJOR_COMPACTION_KEY =
98 Bytes.toBytes("MAJOR_COMPACTION_KEY");
99
100
101 public static final byte[] EXCLUDE_FROM_MINOR_COMPACTION_KEY =
102 Bytes.toBytes("EXCLUDE_FROM_MINOR_COMPACTION");
103
104
105 public static final byte[] BLOOM_FILTER_TYPE_KEY =
106 Bytes.toBytes("BLOOM_FILTER_TYPE");
107
108
109 public static final byte[] DELETE_FAMILY_COUNT =
110 Bytes.toBytes("DELETE_FAMILY_COUNT");
111
112
113 private static final byte[] LAST_BLOOM_KEY = Bytes.toBytes("LAST_BLOOM_KEY");
114
115
116 public static final byte[] TIMERANGE_KEY = Bytes.toBytes("TIMERANGE");
117
118
119 public static final byte[] EARLIEST_PUT_TS = Bytes.toBytes("EARLIEST_PUT_TS");
120
121
122 public static final byte[] MOB_CELLS_COUNT = Bytes.toBytes("MOB_CELLS_COUNT");
123
124 private final StoreFileInfo fileInfo;
125 private final FileSystem fs;
126
127
128 private final CacheConfig cacheConf;
129
130
131
132 private long sequenceid = -1;
133
134
135
136 private long maxMemstoreTS = -1;
137
138 CacheConfig getCacheConf() {
139 return cacheConf;
140 }
141
142 public long getMaxMemstoreTS() {
143 return maxMemstoreTS;
144 }
145
146 public void setMaxMemstoreTS(long maxMemstoreTS) {
147 this.maxMemstoreTS = maxMemstoreTS;
148 }
149
150
151
152 private AtomicBoolean majorCompaction = null;
153
154
155
156 private boolean excludeFromMinorCompaction = false;
157
158
159 public static final byte[] BULKLOAD_TASK_KEY =
160 Bytes.toBytes("BULKLOAD_SOURCE_TASK");
161 public static final byte[] BULKLOAD_TIME_KEY =
162 Bytes.toBytes("BULKLOAD_TIMESTAMP");
163
164
165
166
167 private Map<byte[], byte[]> metadataMap;
168
169
170 private volatile Reader reader;
171
172
173
174
175
176 private final BloomType cfBloomType;
177
178
179
180
181
182
183 public static final byte[] SKIP_RESET_SEQ_ID = Bytes.toBytes("SKIP_RESET_SEQ_ID");
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200 public StoreFile(final FileSystem fs, final Path p, final Configuration conf,
201 final CacheConfig cacheConf, final BloomType cfBloomType) throws IOException {
202 this(fs, new StoreFileInfo(conf, fs, p), conf, cacheConf, cfBloomType);
203 }
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221 public StoreFile(final FileSystem fs, final StoreFileInfo fileInfo, final Configuration conf,
222 final CacheConfig cacheConf, final BloomType cfBloomType) throws IOException {
223 this.fs = fs;
224 this.fileInfo = fileInfo;
225 this.cacheConf = cacheConf;
226
227 if (BloomFilterFactory.isGeneralBloomEnabled(conf)) {
228 this.cfBloomType = cfBloomType;
229 } else {
230 LOG.info("Ignoring bloom filter check for file " + this.getPath() + ": " +
231 "cfBloomType=" + cfBloomType + " (disabled in config)");
232 this.cfBloomType = BloomType.NONE;
233 }
234 }
235
236
237
238
239
240 public StoreFile(final StoreFile other) {
241 this.fs = other.fs;
242 this.fileInfo = other.fileInfo;
243 this.cacheConf = other.cacheConf;
244 this.cfBloomType = other.cfBloomType;
245 }
246
247
248
249
250 public StoreFile cloneForReader() {
251 return new StoreFile(this);
252 }
253
254
255
256
257
258 public StoreFileInfo getFileInfo() {
259 return this.fileInfo;
260 }
261
262
263
264
265 public Path getPath() {
266 return this.fileInfo.getPath();
267 }
268
269
270
271
272 public Path getQualifiedPath() {
273 return this.fileInfo.getPath().makeQualified(fs);
274 }
275
276
277
278
279
280 public boolean isReference() {
281 return this.fileInfo.isReference();
282 }
283
284
285
286
287 public boolean isHFile() {
288 return this.fileInfo.isHFile(this.fileInfo.getPath());
289 }
290
291
292
293
294 public boolean isMajorCompaction() {
295 if (this.majorCompaction == null) {
296 throw new NullPointerException("This has not been set yet");
297 }
298 return this.majorCompaction.get();
299 }
300
301
302
303
304 public boolean excludeFromMinorCompaction() {
305 return this.excludeFromMinorCompaction;
306 }
307
308
309
310
311 public long getMaxSequenceId() {
312 return this.sequenceid;
313 }
314
315 public long getModificationTimeStamp() throws IOException {
316 return (fileInfo == null) ? 0 : fileInfo.getModificationTime();
317 }
318
319
320
321
322
323
324 public byte[] getMetadataValue(byte[] key) {
325 return metadataMap.get(key);
326 }
327
328
329
330
331
332
333
334
335
336 public static long getMaxMemstoreTSInList(Collection<StoreFile> sfs) {
337 long max = 0;
338 for (StoreFile sf : sfs) {
339 if (!sf.isBulkLoadResult()) {
340 max = Math.max(max, sf.getMaxMemstoreTS());
341 }
342 }
343 return max;
344 }
345
346
347
348
349
350
351
352
353 public static long getMaxSequenceIdInList(Collection<StoreFile> sfs) {
354 long max = 0;
355 for (StoreFile sf : sfs) {
356 max = Math.max(max, sf.getMaxSequenceId());
357 }
358 return max;
359 }
360
361
362
363
364
365
366
367
368
369
370
371 public boolean isBulkLoadResult() {
372 boolean bulkLoadedHFile = false;
373 String fileName = this.getPath().getName();
374 int startPos = fileName.indexOf("SeqId_");
375 if (startPos != -1) {
376 bulkLoadedHFile = true;
377 }
378 return bulkLoadedHFile || metadataMap.containsKey(BULKLOAD_TIME_KEY);
379 }
380
381
382
383
384 public long getBulkLoadTimestamp() {
385 byte[] bulkLoadTimestamp = metadataMap.get(BULKLOAD_TIME_KEY);
386 return (bulkLoadTimestamp == null) ? 0 : Bytes.toLong(bulkLoadTimestamp);
387 }
388
389
390
391
392
393 public HDFSBlocksDistribution getHDFSBlockDistribution() {
394 return this.fileInfo.getHDFSBlockDistribution();
395 }
396
397
398
399
400
401
402
403 private Reader open() throws IOException {
404 if (this.reader != null) {
405 throw new IllegalAccessError("Already open");
406 }
407
408
409 this.reader = fileInfo.open(this.fs, this.cacheConf);
410
411
412 metadataMap = Collections.unmodifiableMap(this.reader.loadFileInfo());
413
414
415 byte [] b = metadataMap.get(MAX_SEQ_ID_KEY);
416 if (b != null) {
417
418
419
420
421
422 this.sequenceid = Bytes.toLong(b);
423 if (fileInfo.isTopReference()) {
424 this.sequenceid += 1;
425 }
426 }
427
428 if (isBulkLoadResult()){
429
430
431 String fileName = this.getPath().getName();
432
433 int startPos = fileName.lastIndexOf("SeqId_");
434 if (startPos != -1) {
435 this.sequenceid = Long.parseLong(fileName.substring(startPos + 6,
436 fileName.indexOf('_', startPos + 6)));
437
438 if (fileInfo.isTopReference()) {
439 this.sequenceid += 1;
440 }
441 }
442
443
444
445
446
447 this.reader.setSkipResetSeqId(isSkipResetSeqId(metadataMap.get(SKIP_RESET_SEQ_ID)));
448 this.reader.setBulkLoaded(true);
449 }
450 this.reader.setSequenceID(this.sequenceid);
451
452 b = metadataMap.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY);
453 if (b != null) {
454 this.maxMemstoreTS = Bytes.toLong(b);
455 }
456
457 b = metadataMap.get(MAJOR_COMPACTION_KEY);
458 if (b != null) {
459 boolean mc = Bytes.toBoolean(b);
460 if (this.majorCompaction == null) {
461 this.majorCompaction = new AtomicBoolean(mc);
462 } else {
463 this.majorCompaction.set(mc);
464 }
465 } else {
466
467
468 this.majorCompaction = new AtomicBoolean(false);
469 }
470
471 b = metadataMap.get(EXCLUDE_FROM_MINOR_COMPACTION_KEY);
472 this.excludeFromMinorCompaction = (b != null && Bytes.toBoolean(b));
473
474 BloomType hfileBloomType = reader.getBloomFilterType();
475 if (cfBloomType != BloomType.NONE) {
476 reader.loadBloomfilter(BlockType.GENERAL_BLOOM_META);
477 if (hfileBloomType != cfBloomType) {
478 LOG.info("HFile Bloom filter type for "
479 + reader.getHFileReader().getName() + ": " + hfileBloomType
480 + ", but " + cfBloomType + " specified in column family "
481 + "configuration");
482 }
483 } else if (hfileBloomType != BloomType.NONE) {
484 LOG.info("Bloom filter turned off by CF config for "
485 + reader.getHFileReader().getName());
486 }
487
488
489 reader.loadBloomfilter(BlockType.DELETE_FAMILY_BLOOM_META);
490
491 try {
492 byte [] timerangeBytes = metadataMap.get(TIMERANGE_KEY);
493 if (timerangeBytes != null) {
494 this.reader.timeRangeTracker = new TimeRangeTracker();
495 Writables.copyWritable(timerangeBytes, this.reader.timeRangeTracker);
496 }
497 } catch (IllegalArgumentException e) {
498 LOG.error("Error reading timestamp range data from meta -- " +
499 "proceeding without", e);
500 this.reader.timeRangeTracker = null;
501 }
502 return this.reader;
503 }
504
505
506
507
508
509 public Reader createReader() throws IOException {
510 if (this.reader == null) {
511 try {
512 this.reader = open();
513 } catch (IOException e) {
514 try {
515 boolean evictOnClose =
516 cacheConf != null? cacheConf.shouldEvictOnClose(): true;
517 this.closeReader(evictOnClose);
518 } catch (IOException ee) {
519 }
520 throw e;
521 }
522
523 }
524 return this.reader;
525 }
526
527
528
529
530
531 public Reader getReader() {
532 return this.reader;
533 }
534
535
536
537
538
539 public synchronized void closeReader(boolean evictOnClose)
540 throws IOException {
541 if (this.reader != null) {
542 this.reader.close(evictOnClose);
543 this.reader = null;
544 }
545 }
546
547
548
549
550
551 public void deleteReader() throws IOException {
552 boolean evictOnClose =
553 cacheConf != null? cacheConf.shouldEvictOnClose(): true;
554 closeReader(evictOnClose);
555 this.fs.delete(getPath(), true);
556 }
557
558 @Override
559 public String toString() {
560 return this.fileInfo.toString();
561 }
562
563
564
565
566 public String toStringDetailed() {
567 StringBuilder sb = new StringBuilder();
568 sb.append(this.getPath().toString());
569 sb.append(", isReference=").append(isReference());
570 sb.append(", isBulkLoadResult=").append(isBulkLoadResult());
571 if (isBulkLoadResult()) {
572 sb.append(", bulkLoadTS=").append(getBulkLoadTimestamp());
573 } else {
574 sb.append(", seqid=").append(getMaxSequenceId());
575 }
576 sb.append(", majorCompaction=").append(isMajorCompaction());
577
578 return sb.toString();
579 }
580
581
582
583
584
585
586 private boolean isSkipResetSeqId(byte[] skipResetSeqId) {
587 if (skipResetSeqId != null && skipResetSeqId.length == 1) {
588 return Bytes.toBoolean(skipResetSeqId);
589 }
590 return false;
591 }
592
593 public static class WriterBuilder {
594 private final Configuration conf;
595 private final CacheConfig cacheConf;
596 private final FileSystem fs;
597
598 private KeyValue.KVComparator comparator = KeyValue.COMPARATOR;
599 private BloomType bloomType = BloomType.NONE;
600 private long maxKeyCount = 0;
601 private Path dir;
602 private Path filePath;
603 private InetSocketAddress[] favoredNodes;
604 private HFileContext fileContext;
605 public WriterBuilder(Configuration conf, CacheConfig cacheConf,
606 FileSystem fs) {
607 this.conf = conf;
608 this.cacheConf = cacheConf;
609 this.fs = fs;
610 }
611
612
613
614
615
616
617
618
619 public WriterBuilder withOutputDir(Path dir) {
620 Preconditions.checkNotNull(dir);
621 this.dir = dir;
622 return this;
623 }
624
625
626
627
628
629
630 public WriterBuilder withFilePath(Path filePath) {
631 Preconditions.checkNotNull(filePath);
632 this.filePath = filePath;
633 return this;
634 }
635
636
637
638
639
640 public WriterBuilder withFavoredNodes(InetSocketAddress[] favoredNodes) {
641 this.favoredNodes = favoredNodes;
642 return this;
643 }
644
645 public WriterBuilder withComparator(KeyValue.KVComparator comparator) {
646 Preconditions.checkNotNull(comparator);
647 this.comparator = comparator;
648 return this;
649 }
650
651 public WriterBuilder withBloomType(BloomType bloomType) {
652 Preconditions.checkNotNull(bloomType);
653 this.bloomType = bloomType;
654 return this;
655 }
656
657
658
659
660
661 public WriterBuilder withMaxKeyCount(long maxKeyCount) {
662 this.maxKeyCount = maxKeyCount;
663 return this;
664 }
665
666 public WriterBuilder withFileContext(HFileContext fileContext) {
667 this.fileContext = fileContext;
668 return this;
669 }
670
671
672
673
674
675 public Writer build() throws IOException {
676 if ((dir == null ? 0 : 1) + (filePath == null ? 0 : 1) != 1) {
677 throw new IllegalArgumentException("Either specify parent directory " +
678 "or file path");
679 }
680
681 if (dir == null) {
682 dir = filePath.getParent();
683 }
684
685 if (!fs.exists(dir)) {
686 fs.mkdirs(dir);
687 }
688
689 if (filePath == null) {
690 filePath = getUniqueFile(fs, dir);
691 if (!BloomFilterFactory.isGeneralBloomEnabled(conf)) {
692 bloomType = BloomType.NONE;
693 }
694 }
695
696 if (comparator == null) {
697 comparator = KeyValue.COMPARATOR;
698 }
699 return new Writer(fs, filePath,
700 conf, cacheConf, comparator, bloomType, maxKeyCount, favoredNodes, fileContext);
701 }
702 }
703
704
705
706
707
708
709 public static Path getUniqueFile(final FileSystem fs, final Path dir)
710 throws IOException {
711 if (!fs.getFileStatus(dir).isDirectory()) {
712 throw new IOException("Expecting " + dir.toString() +
713 " to be a directory");
714 }
715 return new Path(dir, UUID.randomUUID().toString().replaceAll("-", ""));
716 }
717
718 public Long getMinimumTimestamp() {
719 return (getReader().timeRangeTracker == null) ?
720 null :
721 getReader().timeRangeTracker.getMinimumTimestamp();
722 }
723
724 public Long getMaximumTimestamp() {
725 return (getReader().timeRangeTracker == null) ?
726 null :
727 getReader().timeRangeTracker.getMaximumTimestamp();
728 }
729
730
731
732
733
734
735
736 @SuppressWarnings("deprecation")
737 byte[] getFileSplitPoint(KVComparator comparator) throws IOException {
738 if (this.reader == null) {
739 LOG.warn("Storefile " + this + " Reader is null; cannot get split point");
740 return null;
741 }
742
743
744
745 byte [] midkey = this.reader.midkey();
746 if (midkey != null) {
747 KeyValue mk = KeyValue.createKeyValueFromKey(midkey, 0, midkey.length);
748 byte [] fk = this.reader.getFirstKey();
749 KeyValue firstKey = KeyValue.createKeyValueFromKey(fk, 0, fk.length);
750 byte [] lk = this.reader.getLastKey();
751 KeyValue lastKey = KeyValue.createKeyValueFromKey(lk, 0, lk.length);
752
753 if (comparator.compareRows(mk, firstKey) == 0 || comparator.compareRows(mk, lastKey) == 0) {
754 if (LOG.isDebugEnabled()) {
755 LOG.debug("cannot split because midkey is the same as first or last row");
756 }
757 return null;
758 }
759 return mk.getRow();
760 }
761 return null;
762 }
763
764
765
766
767
768 public static class Writer implements Compactor.CellSink {
769 private final BloomFilterWriter generalBloomFilterWriter;
770 private final BloomFilterWriter deleteFamilyBloomFilterWriter;
771 private final BloomType bloomType;
772 private byte[] lastBloomKey;
773 private int lastBloomKeyOffset, lastBloomKeyLen;
774 private KVComparator kvComparator;
775 private Cell lastCell = null;
776 private long earliestPutTs = HConstants.LATEST_TIMESTAMP;
777 private Cell lastDeleteFamilyCell = null;
778 private long deleteFamilyCnt = 0;
779
780
781 protected int bytesPerChecksum;
782
783 TimeRangeTracker timeRangeTracker = new TimeRangeTracker();
784
785
786
787
788
789
790 boolean isTimeRangeTrackerSet = false;
791
792 protected HFile.Writer writer;
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807 private Writer(FileSystem fs, Path path,
808 final Configuration conf,
809 CacheConfig cacheConf,
810 final KVComparator comparator, BloomType bloomType, long maxKeys,
811 InetSocketAddress[] favoredNodes, HFileContext fileContext)
812 throws IOException {
813 writer = HFile.getWriterFactory(conf, cacheConf)
814 .withPath(fs, path)
815 .withComparator(comparator)
816 .withFavoredNodes(favoredNodes)
817 .withFileContext(fileContext)
818 .create();
819
820 this.kvComparator = comparator;
821
822 generalBloomFilterWriter = BloomFilterFactory.createGeneralBloomAtWrite(
823 conf, cacheConf, bloomType,
824 (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);
825
826 if (generalBloomFilterWriter != null) {
827 this.bloomType = bloomType;
828 if (LOG.isTraceEnabled()) LOG.trace("Bloom filter type for " + path + ": " +
829 this.bloomType + ", " + generalBloomFilterWriter.getClass().getSimpleName());
830 } else {
831
832 this.bloomType = BloomType.NONE;
833 }
834
835
836
837 if (this.bloomType != BloomType.ROWCOL) {
838 this.deleteFamilyBloomFilterWriter = BloomFilterFactory
839 .createDeleteBloomAtWrite(conf, cacheConf,
840 (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);
841 } else {
842 deleteFamilyBloomFilterWriter = null;
843 }
844 if (deleteFamilyBloomFilterWriter != null) {
845 if (LOG.isTraceEnabled()) LOG.trace("Delete Family Bloom filter type for " + path + ": "
846 + deleteFamilyBloomFilterWriter.getClass().getSimpleName());
847 }
848 }
849
850
851
852
853
854
855
856
857 public void appendMetadata(final long maxSequenceId, final boolean majorCompaction)
858 throws IOException {
859 writer.appendFileInfo(MAX_SEQ_ID_KEY, Bytes.toBytes(maxSequenceId));
860 writer.appendFileInfo(MAJOR_COMPACTION_KEY,
861 Bytes.toBytes(majorCompaction));
862 appendTrackedTimestampsToMetadata();
863 }
864
865
866
867
868
869
870
871
872
873 public void appendMetadata(final long maxSequenceId, final boolean majorCompaction,
874 final long mobCellsCount) throws IOException {
875 writer.appendFileInfo(MAX_SEQ_ID_KEY, Bytes.toBytes(maxSequenceId));
876 writer.appendFileInfo(MAJOR_COMPACTION_KEY, Bytes.toBytes(majorCompaction));
877 writer.appendFileInfo(MOB_CELLS_COUNT, Bytes.toBytes(mobCellsCount));
878 appendTrackedTimestampsToMetadata();
879 }
880
881
882
883
884 public void appendTrackedTimestampsToMetadata() throws IOException {
885 appendFileInfo(TIMERANGE_KEY,WritableUtils.toByteArray(timeRangeTracker));
886 appendFileInfo(EARLIEST_PUT_TS, Bytes.toBytes(earliestPutTs));
887 }
888
889
890
891
892
893 public void setTimeRangeTracker(final TimeRangeTracker trt) {
894 this.timeRangeTracker = trt;
895 isTimeRangeTrackerSet = true;
896 }
897
898
899
900
901
902
903
904
905 public void trackTimestamps(final Cell cell) {
906 if (KeyValue.Type.Put.getCode() == cell.getTypeByte()) {
907 earliestPutTs = Math.min(earliestPutTs, cell.getTimestamp());
908 }
909 if (!isTimeRangeTrackerSet) {
910 timeRangeTracker.includeTimestamp(cell);
911 }
912 }
913
914 private void appendGeneralBloomfilter(final Cell cell) throws IOException {
915 if (this.generalBloomFilterWriter != null) {
916
917 boolean newKey = true;
918 if (this.lastCell != null) {
919 switch(bloomType) {
920 case ROW:
921 newKey = ! kvComparator.matchingRows(cell, lastCell);
922 break;
923 case ROWCOL:
924 newKey = ! kvComparator.matchingRowColumn(cell, lastCell);
925 break;
926 case NONE:
927 newKey = false;
928 break;
929 default:
930 throw new IOException("Invalid Bloom filter type: " + bloomType +
931 " (ROW or ROWCOL expected)");
932 }
933 }
934 if (newKey) {
935
936
937
938
939
940
941
942
943 byte[] bloomKey;
944 int bloomKeyOffset, bloomKeyLen;
945
946 switch (bloomType) {
947 case ROW:
948 bloomKey = cell.getRowArray();
949 bloomKeyOffset = cell.getRowOffset();
950 bloomKeyLen = cell.getRowLength();
951 break;
952 case ROWCOL:
953
954
955
956 bloomKey = generalBloomFilterWriter.createBloomKey(cell.getRowArray(),
957 cell.getRowOffset(), cell.getRowLength(), cell.getQualifierArray(),
958 cell.getQualifierOffset(), cell.getQualifierLength());
959 bloomKeyOffset = 0;
960 bloomKeyLen = bloomKey.length;
961 break;
962 default:
963 throw new IOException("Invalid Bloom filter type: " + bloomType +
964 " (ROW or ROWCOL expected)");
965 }
966 generalBloomFilterWriter.add(bloomKey, bloomKeyOffset, bloomKeyLen);
967 if (lastBloomKey != null
968 && generalBloomFilterWriter.getComparator().compareFlatKey(bloomKey,
969 bloomKeyOffset, bloomKeyLen, lastBloomKey,
970 lastBloomKeyOffset, lastBloomKeyLen) <= 0) {
971 throw new IOException("Non-increasing Bloom keys: "
972 + Bytes.toStringBinary(bloomKey, bloomKeyOffset, bloomKeyLen)
973 + " after "
974 + Bytes.toStringBinary(lastBloomKey, lastBloomKeyOffset,
975 lastBloomKeyLen));
976 }
977 lastBloomKey = bloomKey;
978 lastBloomKeyOffset = bloomKeyOffset;
979 lastBloomKeyLen = bloomKeyLen;
980 this.lastCell = cell;
981 }
982 }
983 }
984
985 private void appendDeleteFamilyBloomFilter(final Cell cell)
986 throws IOException {
987 if (!CellUtil.isDeleteFamily(cell) && !CellUtil.isDeleteFamilyVersion(cell)) {
988 return;
989 }
990
991
992 deleteFamilyCnt++;
993 if (null != this.deleteFamilyBloomFilterWriter) {
994 boolean newKey = true;
995 if (lastDeleteFamilyCell != null) {
996 newKey = !kvComparator.matchingRows(cell, lastDeleteFamilyCell);
997 }
998 if (newKey) {
999 this.deleteFamilyBloomFilterWriter.add(cell.getRowArray(),
1000 cell.getRowOffset(), cell.getRowLength());
1001 this.lastDeleteFamilyCell = cell;
1002 }
1003 }
1004 }
1005
1006 public void append(final Cell cell) throws IOException {
1007 appendGeneralBloomfilter(cell);
1008 appendDeleteFamilyBloomFilter(cell);
1009 writer.append(cell);
1010 trackTimestamps(cell);
1011 }
1012
1013 public Path getPath() {
1014 return this.writer.getPath();
1015 }
1016
1017 public boolean hasGeneralBloom() {
1018 return this.generalBloomFilterWriter != null;
1019 }
1020
1021
1022
1023
1024
1025
1026 BloomFilterWriter getGeneralBloomWriter() {
1027 return generalBloomFilterWriter;
1028 }
1029
1030 private boolean closeBloomFilter(BloomFilterWriter bfw) throws IOException {
1031 boolean haveBloom = (bfw != null && bfw.getKeyCount() > 0);
1032 if (haveBloom) {
1033 bfw.compactBloom();
1034 }
1035 return haveBloom;
1036 }
1037
1038 private boolean closeGeneralBloomFilter() throws IOException {
1039 boolean hasGeneralBloom = closeBloomFilter(generalBloomFilterWriter);
1040
1041
1042 if (hasGeneralBloom) {
1043 writer.addGeneralBloomFilter(generalBloomFilterWriter);
1044 writer.appendFileInfo(BLOOM_FILTER_TYPE_KEY,
1045 Bytes.toBytes(bloomType.toString()));
1046 if (lastBloomKey != null) {
1047 writer.appendFileInfo(LAST_BLOOM_KEY, Arrays.copyOfRange(
1048 lastBloomKey, lastBloomKeyOffset, lastBloomKeyOffset
1049 + lastBloomKeyLen));
1050 }
1051 }
1052 return hasGeneralBloom;
1053 }
1054
1055 private boolean closeDeleteFamilyBloomFilter() throws IOException {
1056 boolean hasDeleteFamilyBloom = closeBloomFilter(deleteFamilyBloomFilterWriter);
1057
1058
1059 if (hasDeleteFamilyBloom) {
1060 writer.addDeleteFamilyBloomFilter(deleteFamilyBloomFilterWriter);
1061 }
1062
1063
1064
1065 writer.appendFileInfo(DELETE_FAMILY_COUNT,
1066 Bytes.toBytes(this.deleteFamilyCnt));
1067
1068 return hasDeleteFamilyBloom;
1069 }
1070
1071 public void close() throws IOException {
1072 boolean hasGeneralBloom = this.closeGeneralBloomFilter();
1073 boolean hasDeleteFamilyBloom = this.closeDeleteFamilyBloomFilter();
1074
1075 writer.close();
1076
1077
1078
1079 if (StoreFile.LOG.isTraceEnabled()) {
1080 StoreFile.LOG.trace((hasGeneralBloom ? "" : "NO ") + "General Bloom and " +
1081 (hasDeleteFamilyBloom ? "" : "NO ") + "DeleteFamily" + " was added to HFile " +
1082 getPath());
1083 }
1084
1085 }
1086
1087 public void appendFileInfo(byte[] key, byte[] value) throws IOException {
1088 writer.appendFileInfo(key, value);
1089 }
1090
1091
1092
1093 HFile.Writer getHFileWriter() {
1094 return writer;
1095 }
1096 }
1097
1098
1099
1100
1101 public static class Reader {
1102 static final Log LOG = LogFactory.getLog(Reader.class.getName());
1103
1104 protected BloomFilter generalBloomFilter = null;
1105 protected BloomFilter deleteFamilyBloomFilter = null;
1106 protected BloomType bloomFilterType;
1107 private final HFile.Reader reader;
1108 protected TimeRangeTracker timeRangeTracker = null;
1109 protected long sequenceID = -1;
1110 private byte[] lastBloomKey;
1111 private long deleteFamilyCnt = -1;
1112 private boolean bulkLoadResult = false;
1113 private boolean skipResetSeqId = true;
1114
1115 public Reader(FileSystem fs, Path path, CacheConfig cacheConf, Configuration conf)
1116 throws IOException {
1117 reader = HFile.createReader(fs, path, cacheConf, conf);
1118 bloomFilterType = BloomType.NONE;
1119 }
1120
1121 public Reader(FileSystem fs, Path path, FSDataInputStreamWrapper in, long size,
1122 CacheConfig cacheConf, Configuration conf) throws IOException {
1123 reader = HFile.createReader(fs, path, in, size, cacheConf, conf);
1124 bloomFilterType = BloomType.NONE;
1125 }
1126
1127 public void setReplicaStoreFile(boolean isPrimaryReplicaStoreFile) {
1128 reader.setPrimaryReplicaReader(isPrimaryReplicaStoreFile);
1129 }
1130 public boolean isPrimaryReplicaReader() {
1131 return reader.isPrimaryReplicaReader();
1132 }
1133
1134
1135
1136
1137 Reader() {
1138 this.reader = null;
1139 }
1140
1141 public KVComparator getComparator() {
1142 return reader.getComparator();
1143 }
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153 public StoreFileScanner getStoreFileScanner(boolean cacheBlocks,
1154 boolean pread) {
1155 return getStoreFileScanner(cacheBlocks, pread, false,
1156
1157
1158 0);
1159 }
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169 public StoreFileScanner getStoreFileScanner(boolean cacheBlocks,
1170 boolean pread,
1171 boolean isCompaction, long readPt) {
1172 return new StoreFileScanner(this,
1173 getScanner(cacheBlocks, pread, isCompaction),
1174 !isCompaction, reader.hasMVCCInfo(), readPt);
1175 }
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186 @Deprecated
1187 public HFileScanner getScanner(boolean cacheBlocks, boolean pread) {
1188 return getScanner(cacheBlocks, pread, false);
1189 }
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204 @Deprecated
1205 public HFileScanner getScanner(boolean cacheBlocks, boolean pread,
1206 boolean isCompaction) {
1207 return reader.getScanner(cacheBlocks, pread, isCompaction);
1208 }
1209
1210 public void close(boolean evictOnClose) throws IOException {
1211 reader.close(evictOnClose);
1212 }
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222 boolean passesTimerangeFilter(Scan scan, long oldestUnexpiredTS) {
1223 if (timeRangeTracker == null) {
1224 return true;
1225 } else {
1226 return timeRangeTracker.includesTimeRange(scan.getTimeRange()) &&
1227 timeRangeTracker.getMaximumTimestamp() >= oldestUnexpiredTS;
1228 }
1229 }
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247 boolean passesBloomFilter(Scan scan,
1248 final SortedSet<byte[]> columns) {
1249
1250
1251 if (!scan.isGetScan()) {
1252 return true;
1253 }
1254
1255 byte[] row = scan.getStartRow();
1256 switch (this.bloomFilterType) {
1257 case ROW:
1258 return passesGeneralBloomFilter(row, 0, row.length, null, 0, 0);
1259
1260 case ROWCOL:
1261 if (columns != null && columns.size() == 1) {
1262 byte[] column = columns.first();
1263 return passesGeneralBloomFilter(row, 0, row.length, column, 0,
1264 column.length);
1265 }
1266
1267
1268
1269 return true;
1270
1271 default:
1272 return true;
1273 }
1274 }
1275
1276 public boolean passesDeleteFamilyBloomFilter(byte[] row, int rowOffset,
1277 int rowLen) {
1278
1279
1280 BloomFilter bloomFilter = this.deleteFamilyBloomFilter;
1281
1282
1283 if (reader.getTrailer().getEntryCount() == 0 || deleteFamilyCnt == 0) {
1284 return false;
1285 }
1286
1287 if (bloomFilter == null) {
1288 return true;
1289 }
1290
1291 try {
1292 if (!bloomFilter.supportsAutoLoading()) {
1293 return true;
1294 }
1295 return bloomFilter.contains(row, rowOffset, rowLen, null);
1296 } catch (IllegalArgumentException e) {
1297 LOG.error("Bad Delete Family bloom filter data -- proceeding without",
1298 e);
1299 setDeleteFamilyBloomFilterFaulty();
1300 }
1301
1302 return true;
1303 }
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317 public boolean passesGeneralBloomFilter(byte[] row, int rowOffset,
1318 int rowLen, byte[] col, int colOffset, int colLen) {
1319
1320
1321 BloomFilter bloomFilter = this.generalBloomFilter;
1322 if (bloomFilter == null) {
1323 return true;
1324 }
1325
1326 byte[] key;
1327 switch (bloomFilterType) {
1328 case ROW:
1329 if (col != null) {
1330 throw new RuntimeException("Row-only Bloom filter called with " +
1331 "column specified");
1332 }
1333 if (rowOffset != 0 || rowLen != row.length) {
1334 throw new AssertionError("For row-only Bloom filters the row "
1335 + "must occupy the whole array");
1336 }
1337 key = row;
1338 break;
1339
1340 case ROWCOL:
1341 key = bloomFilter.createBloomKey(row, rowOffset, rowLen, col,
1342 colOffset, colLen);
1343 break;
1344
1345 default:
1346 return true;
1347 }
1348
1349
1350 if (reader.getTrailer().getEntryCount() == 0)
1351 return false;
1352
1353 try {
1354 boolean shouldCheckBloom;
1355 ByteBuffer bloom;
1356 if (bloomFilter.supportsAutoLoading()) {
1357 bloom = null;
1358 shouldCheckBloom = true;
1359 } else {
1360 bloom = reader.getMetaBlock(HFile.BLOOM_FILTER_DATA_KEY,
1361 true);
1362 shouldCheckBloom = bloom != null;
1363 }
1364
1365 if (shouldCheckBloom) {
1366 boolean exists;
1367
1368
1369
1370
1371 boolean keyIsAfterLast = lastBloomKey != null
1372 && bloomFilter.getComparator().compareFlatKey(key, lastBloomKey) > 0;
1373
1374 if (bloomFilterType == BloomType.ROWCOL) {
1375
1376
1377
1378
1379 byte[] rowBloomKey = bloomFilter.createBloomKey(row, rowOffset, rowLen,
1380 null, 0, 0);
1381
1382 if (keyIsAfterLast
1383 && bloomFilter.getComparator().compareFlatKey(rowBloomKey,
1384 lastBloomKey) > 0) {
1385 exists = false;
1386 } else {
1387 exists =
1388 bloomFilter.contains(key, 0, key.length, bloom) ||
1389 bloomFilter.contains(rowBloomKey, 0, rowBloomKey.length,
1390 bloom);
1391 }
1392 } else {
1393 exists = !keyIsAfterLast
1394 && bloomFilter.contains(key, 0, key.length, bloom);
1395 }
1396
1397 return exists;
1398 }
1399 } catch (IOException e) {
1400 LOG.error("Error reading bloom filter data -- proceeding without",
1401 e);
1402 setGeneralBloomFilterFaulty();
1403 } catch (IllegalArgumentException e) {
1404 LOG.error("Bad bloom filter data -- proceeding without", e);
1405 setGeneralBloomFilterFaulty();
1406 }
1407
1408 return true;
1409 }
1410
1411
1412
1413
1414
1415
1416 public boolean passesKeyRangeFilter(Scan scan) {
1417 if (this.getFirstKey() == null || this.getLastKey() == null) {
1418
1419 return false;
1420 }
1421 if (Bytes.equals(scan.getStartRow(), HConstants.EMPTY_START_ROW)
1422 && Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) {
1423 return true;
1424 }
1425 KeyValue smallestScanKeyValue = scan.isReversed() ? KeyValueUtil
1426 .createFirstOnRow(scan.getStopRow()) : KeyValueUtil.createFirstOnRow(scan
1427 .getStartRow());
1428 KeyValue largestScanKeyValue = scan.isReversed() ? KeyValueUtil
1429 .createLastOnRow(scan.getStartRow()) : KeyValueUtil.createLastOnRow(scan
1430 .getStopRow());
1431 boolean nonOverLapping = (getComparator().compareFlatKey(
1432 this.getFirstKey(), largestScanKeyValue.getKey()) > 0 && !Bytes
1433 .equals(scan.isReversed() ? scan.getStartRow() : scan.getStopRow(),
1434 HConstants.EMPTY_END_ROW))
1435 || getComparator().compareFlatKey(this.getLastKey(),
1436 smallestScanKeyValue.getKey()) < 0;
1437 return !nonOverLapping;
1438 }
1439
1440 public Map<byte[], byte[]> loadFileInfo() throws IOException {
1441 Map<byte [], byte []> fi = reader.loadFileInfo();
1442
1443 byte[] b = fi.get(BLOOM_FILTER_TYPE_KEY);
1444 if (b != null) {
1445 bloomFilterType = BloomType.valueOf(Bytes.toString(b));
1446 }
1447
1448 lastBloomKey = fi.get(LAST_BLOOM_KEY);
1449 byte[] cnt = fi.get(DELETE_FAMILY_COUNT);
1450 if (cnt != null) {
1451 deleteFamilyCnt = Bytes.toLong(cnt);
1452 }
1453
1454 return fi;
1455 }
1456
1457 public void loadBloomfilter() {
1458 this.loadBloomfilter(BlockType.GENERAL_BLOOM_META);
1459 this.loadBloomfilter(BlockType.DELETE_FAMILY_BLOOM_META);
1460 }
1461
1462 private void loadBloomfilter(BlockType blockType) {
1463 try {
1464 if (blockType == BlockType.GENERAL_BLOOM_META) {
1465 if (this.generalBloomFilter != null)
1466 return;
1467
1468 DataInput bloomMeta = reader.getGeneralBloomFilterMetadata();
1469 if (bloomMeta != null) {
1470
1471 if (bloomFilterType == BloomType.NONE) {
1472 throw new IOException(
1473 "valid bloom filter type not found in FileInfo");
1474 } else {
1475 generalBloomFilter = BloomFilterFactory.createFromMeta(bloomMeta,
1476 reader);
1477 if (LOG.isTraceEnabled()) {
1478 LOG.trace("Loaded " + bloomFilterType.toString() + " "
1479 + generalBloomFilter.getClass().getSimpleName()
1480 + " metadata for " + reader.getName());
1481 }
1482 }
1483 }
1484 } else if (blockType == BlockType.DELETE_FAMILY_BLOOM_META) {
1485 if (this.deleteFamilyBloomFilter != null)
1486 return;
1487
1488 DataInput bloomMeta = reader.getDeleteBloomFilterMetadata();
1489 if (bloomMeta != null) {
1490 deleteFamilyBloomFilter = BloomFilterFactory.createFromMeta(
1491 bloomMeta, reader);
1492 LOG.info("Loaded Delete Family Bloom ("
1493 + deleteFamilyBloomFilter.getClass().getSimpleName()
1494 + ") metadata for " + reader.getName());
1495 }
1496 } else {
1497 throw new RuntimeException("Block Type: " + blockType.toString()
1498 + "is not supported for Bloom filter");
1499 }
1500 } catch (IOException e) {
1501 LOG.error("Error reading bloom filter meta for " + blockType
1502 + " -- proceeding without", e);
1503 setBloomFilterFaulty(blockType);
1504 } catch (IllegalArgumentException e) {
1505 LOG.error("Bad bloom filter meta " + blockType
1506 + " -- proceeding without", e);
1507 setBloomFilterFaulty(blockType);
1508 }
1509 }
1510
1511 private void setBloomFilterFaulty(BlockType blockType) {
1512 if (blockType == BlockType.GENERAL_BLOOM_META) {
1513 setGeneralBloomFilterFaulty();
1514 } else if (blockType == BlockType.DELETE_FAMILY_BLOOM_META) {
1515 setDeleteFamilyBloomFilterFaulty();
1516 }
1517 }
1518
1519
1520
1521
1522
1523
1524
1525
1526 public long getFilterEntries() {
1527 return generalBloomFilter != null ? generalBloomFilter.getKeyCount()
1528 : reader.getEntries();
1529 }
1530
1531 public void setGeneralBloomFilterFaulty() {
1532 generalBloomFilter = null;
1533 }
1534
1535 public void setDeleteFamilyBloomFilterFaulty() {
1536 this.deleteFamilyBloomFilter = null;
1537 }
1538
1539 public byte[] getLastKey() {
1540 return reader.getLastKey();
1541 }
1542
1543 public byte[] getLastRowKey() {
1544 return reader.getLastRowKey();
1545 }
1546
1547 public byte[] midkey() throws IOException {
1548 return reader.midkey();
1549 }
1550
1551 public long length() {
1552 return reader.length();
1553 }
1554
1555 public long getTotalUncompressedBytes() {
1556 return reader.getTrailer().getTotalUncompressedBytes();
1557 }
1558
1559 public long getEntries() {
1560 return reader.getEntries();
1561 }
1562
1563 public long getDeleteFamilyCnt() {
1564 return deleteFamilyCnt;
1565 }
1566
1567 public byte[] getFirstKey() {
1568 return reader.getFirstKey();
1569 }
1570
1571 public long indexSize() {
1572 return reader.indexSize();
1573 }
1574
1575 public BloomType getBloomFilterType() {
1576 return this.bloomFilterType;
1577 }
1578
1579 public long getSequenceID() {
1580 return sequenceID;
1581 }
1582
1583 public void setSequenceID(long sequenceID) {
1584 this.sequenceID = sequenceID;
1585 }
1586
1587 public void setBulkLoaded(boolean bulkLoadResult) {
1588 this.bulkLoadResult = bulkLoadResult;
1589 }
1590
1591 public boolean isBulkLoaded() {
1592 return this.bulkLoadResult;
1593 }
1594
1595 BloomFilter getGeneralBloomFilter() {
1596 return generalBloomFilter;
1597 }
1598
1599 long getUncompressedDataIndexSize() {
1600 return reader.getTrailer().getUncompressedDataIndexSize();
1601 }
1602
1603 public long getTotalBloomSize() {
1604 if (generalBloomFilter == null)
1605 return 0;
1606 return generalBloomFilter.getByteSize();
1607 }
1608
1609 public int getHFileVersion() {
1610 return reader.getTrailer().getMajorVersion();
1611 }
1612
1613 public int getHFileMinorVersion() {
1614 return reader.getTrailer().getMinorVersion();
1615 }
1616
1617 public HFile.Reader getHFileReader() {
1618 return reader;
1619 }
1620
1621 void disableBloomFilterForTesting() {
1622 generalBloomFilter = null;
1623 this.deleteFamilyBloomFilter = null;
1624 }
1625
1626 public long getMaxTimestamp() {
1627 return timeRangeTracker == null ? Long.MAX_VALUE : timeRangeTracker.getMaximumTimestamp();
1628 }
1629
1630 boolean isSkipResetSeqId() {
1631 return skipResetSeqId;
1632 }
1633
1634 void setSkipResetSeqId(boolean skipResetSeqId) {
1635 this.skipResetSeqId = skipResetSeqId;
1636 }
1637 }
1638
1639
1640
1641
1642 public abstract static class Comparators {
1643
1644
1645
1646
1647
1648
1649
1650
1651 public static final Comparator<StoreFile> SEQ_ID =
1652 Ordering.compound(ImmutableList.of(
1653 Ordering.natural().onResultOf(new GetSeqId()),
1654 Ordering.natural().onResultOf(new GetFileSize()).reverse(),
1655 Ordering.natural().onResultOf(new GetBulkTime()),
1656 Ordering.natural().onResultOf(new GetPathName())
1657 ));
1658
1659
1660
1661
1662
1663 public static final Comparator<StoreFile> SEQ_ID_MAX_TIMESTAMP =
1664 Ordering.compound(ImmutableList.of(
1665 Ordering.natural().onResultOf(new GetSeqId()),
1666 Ordering.natural().onResultOf(new GetMaxTimestamp()),
1667 Ordering.natural().onResultOf(new GetFileSize()).reverse(),
1668 Ordering.natural().onResultOf(new GetBulkTime()),
1669 Ordering.natural().onResultOf(new GetPathName())
1670 ));
1671
1672 private static class GetSeqId implements Function<StoreFile, Long> {
1673 @Override
1674 public Long apply(StoreFile sf) {
1675 return sf.getMaxSequenceId();
1676 }
1677 }
1678
1679 private static class GetFileSize implements Function<StoreFile, Long> {
1680 @Override
1681 public Long apply(StoreFile sf) {
1682 return sf.getReader().length();
1683 }
1684 }
1685
1686 private static class GetBulkTime implements Function<StoreFile, Long> {
1687 @Override
1688 public Long apply(StoreFile sf) {
1689 if (!sf.isBulkLoadResult()) return Long.MAX_VALUE;
1690 return sf.getBulkLoadTimestamp();
1691 }
1692 }
1693
1694 private static class GetPathName implements Function<StoreFile, String> {
1695 @Override
1696 public String apply(StoreFile sf) {
1697 return sf.getPath().getName();
1698 }
1699 }
1700
1701 private static class GetMaxTimestamp implements Function<StoreFile, Long> {
1702 @Override
1703 public Long apply(StoreFile sf) {
1704 return sf.getMaximumTimestamp() == null? (Long)Long.MAX_VALUE : sf.getMaximumTimestamp();
1705 }
1706 }
1707 }
1708 }