1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.mob;
19
20 import java.io.IOException;
21 import java.util.ArrayList;
22 import java.util.Date;
23 import java.util.List;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.hbase.classification.InterfaceAudience;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.hbase.Cell;
31 import org.apache.hadoop.hbase.CellUtil;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.KeyValueUtil;
34 import org.apache.hadoop.hbase.Tag;
35 import org.apache.hadoop.hbase.TagType;
36 import org.apache.hadoop.hbase.client.Scan;
37 import org.apache.hadoop.hbase.regionserver.*;
38 import org.apache.hadoop.hbase.regionserver.StoreFile.Writer;
39 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
40 import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputController;
41 import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
42 import org.apache.hadoop.hbase.security.User;
43 import org.apache.hadoop.hbase.util.Bytes;
44
45
46
47
48 @InterfaceAudience.Private
49 public class DefaultMobStoreCompactor extends DefaultCompactor {
50
51 private static final Log LOG = LogFactory.getLog(DefaultMobStoreCompactor.class);
52 private long mobSizeThreshold;
53 private HMobStore mobStore;
54
55 private final InternalScannerFactory scannerFactory = new InternalScannerFactory() {
56
57 @Override
58 public ScanType getScanType(CompactionRequest request) {
59 return request.isRetainDeleteMarkers() ? ScanType.COMPACT_RETAIN_DELETES
60 : ScanType.COMPACT_DROP_DELETES;
61 }
62
63 @Override
64 public InternalScanner createScanner(List<StoreFileScanner> scanners,
65 ScanType scanType, FileDetails fd, long smallestReadPoint) throws IOException {
66 Scan scan = new Scan();
67 scan.setMaxVersions(store.getFamily().getMaxVersions());
68 if (scanType == ScanType.COMPACT_DROP_DELETES) {
69
70
71 scanType = ScanType.COMPACT_RETAIN_DELETES;
72 return new MobCompactionStoreScanner(store, store.getScanInfo(), scan, scanners,
73 scanType, smallestReadPoint, fd.earliestPutTs, true);
74 } else {
75 return new MobCompactionStoreScanner(store, store.getScanInfo(), scan, scanners,
76 scanType, smallestReadPoint, fd.earliestPutTs, false);
77 }
78 }
79 };
80
81 private final CellSinkFactory<Writer> writerFactory = new CellSinkFactory<Writer>() {
82
83 @Override
84 public Writer createWriter(InternalScanner scanner,
85 org.apache.hadoop.hbase.regionserver.compactions.Compactor.FileDetails fd,
86 boolean shouldDropBehind) throws IOException {
87
88 return store.createWriterInTmp(fd.maxKeyCount, compactionCompression, true, true, true);
89 }
90 };
91
92 public DefaultMobStoreCompactor(Configuration conf, Store store) {
93 super(conf, store);
94
95
96
97
98 if (!(store instanceof HMobStore)) {
99 throw new IllegalArgumentException("The store " + store + " is not a HMobStore");
100 }
101 mobStore = (HMobStore) store;
102 mobSizeThreshold = store.getFamily().getMobThreshold();
103 }
104
105 @Override
106 public List<Path> compact(CompactionRequest request,
107 CompactionThroughputController throughputController, User user) throws IOException {
108 return compact(request, scannerFactory, writerFactory, throughputController, user);
109 }
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156 @Override
157 protected boolean performCompaction(FileDetails fd, InternalScanner scanner, CellSink writer,
158 long smallestReadPoint, boolean cleanSeqId,
159 CompactionThroughputController throughputController, boolean major) throws IOException {
160 if (!(scanner instanceof MobCompactionStoreScanner)) {
161 throw new IllegalArgumentException(
162 "The scanner should be an instance of MobCompactionStoreScanner");
163 }
164 MobCompactionStoreScanner compactionScanner = (MobCompactionStoreScanner) scanner;
165 int bytesWritten = 0;
166
167
168 List<Cell> cells = new ArrayList<Cell>();
169
170 int closeCheckInterval = HStore.getCloseCheckInterval();
171 boolean hasMore;
172 Path path = MobUtils.getMobFamilyPath(conf, store.getTableName(), store.getColumnFamilyName());
173 byte[] fileName = null;
174 Writer mobFileWriter = null, delFileWriter = null;
175 long mobCells = 0;
176 long deleteMarkersCount = 0;
177 Tag tableNameTag = new Tag(TagType.MOB_TABLE_NAME_TAG_TYPE, store.getTableName()
178 .getName());
179 long cellsCountCompactedToMob = 0;
180 long cellsCountCompactedFromMob = 0;
181 long cellsSizeCompactedToMob = 0;
182 long cellsSizeCompactedFromMob = 0;
183 try {
184 try {
185
186 mobFileWriter = mobStore.createWriterInTmp(new Date(fd.latestPutTs), fd.maxKeyCount,
187 store.getFamily().getCompression(), store.getRegionInfo().getStartKey());
188 fileName = Bytes.toBytes(mobFileWriter.getPath().getName());
189 } catch (IOException e) {
190 LOG.error(
191 "Failed to create mob writer, "
192 + "we will continue the compaction by writing MOB cells directly in store files",
193 e);
194 }
195 delFileWriter = mobStore.createDelFileWriterInTmp(new Date(fd.latestPutTs), fd.maxKeyCount,
196 store.getFamily().getCompression(), store.getRegionInfo().getStartKey());
197 ScannerContext scannerContext =
198 ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build();
199 do {
200 hasMore = compactionScanner.next(cells, scannerContext);
201
202 for (Cell c : cells) {
203 if (compactionScanner.isOutputDeleteMarkers() && CellUtil.isDelete(c)) {
204 delFileWriter.append(c);
205 deleteMarkersCount++;
206 } else if (mobFileWriter == null || c.getTypeByte() != KeyValue.Type.Put.getCode()) {
207
208
209 writer.append(c);
210 } else if (MobUtils.isMobReferenceCell(c)) {
211 if (MobUtils.hasValidMobRefCellValue(c)) {
212 int size = MobUtils.getMobValueLength(c);
213 if (size > mobSizeThreshold) {
214
215
216 writer.append(c);
217 } else {
218
219
220 Cell mobCell = mobStore.resolve(c, false);
221 if (mobCell.getValueLength() != 0) {
222
223 CellUtil.setSequenceId(mobCell, c.getSequenceId());
224 writer.append(mobCell);
225 cellsCountCompactedFromMob++;
226 cellsSizeCompactedFromMob += mobCell.getValueLength();
227 } else {
228
229
230
231 writer.append(c);
232 }
233 }
234 } else {
235 LOG.warn("The value format of the KeyValue " + c
236 + " is wrong, its length is less than " + Bytes.SIZEOF_INT);
237 writer.append(c);
238 }
239 } else if (c.getValueLength() <= mobSizeThreshold) {
240
241
242 writer.append(c);
243 } else {
244
245
246 mobCells++;
247
248 mobFileWriter.append(c);
249 KeyValue reference = MobUtils.createMobRefKeyValue(c, fileName, tableNameTag);
250
251 writer.append(reference);
252 cellsCountCompactedToMob++;
253 cellsSizeCompactedToMob += c.getValueLength();
254 }
255 ++progress.currentCompactedKVs;
256
257
258 if (closeCheckInterval > 0) {
259 bytesWritten += KeyValueUtil.length(c);
260 if (bytesWritten > closeCheckInterval) {
261 bytesWritten = 0;
262 if (!store.areWritesEnabled()) {
263 progress.cancel();
264 return false;
265 }
266 }
267 }
268 }
269 cells.clear();
270 } while (hasMore);
271 } finally {
272 if (mobFileWriter != null) {
273 mobFileWriter.appendMetadata(fd.maxSeqId, major, mobCells);
274 mobFileWriter.close();
275 }
276 if (delFileWriter != null) {
277 delFileWriter.appendMetadata(fd.maxSeqId, major, deleteMarkersCount);
278 delFileWriter.close();
279 }
280 }
281 if (mobFileWriter != null) {
282 if (mobCells > 0) {
283
284 mobStore.commitFile(mobFileWriter.getPath(), path);
285 } else {
286 try {
287
288 store.getFileSystem().delete(mobFileWriter.getPath(), true);
289 } catch (IOException e) {
290 LOG.error("Failed to delete the temp mob file", e);
291 }
292 }
293 }
294 if (delFileWriter != null) {
295 if (deleteMarkersCount > 0) {
296
297
298 mobStore.commitFile(delFileWriter.getPath(), path);
299 } else {
300 try {
301
302 store.getFileSystem().delete(delFileWriter.getPath(), true);
303 } catch (IOException e) {
304 LOG.error("Failed to delete the temp del file", e);
305 }
306 }
307 }
308 mobStore.updateCellsCountCompactedFromMob(cellsCountCompactedFromMob);
309 mobStore.updateCellsCountCompactedToMob(cellsCountCompactedToMob);
310 mobStore.updateCellsSizeCompactedFromMob(cellsSizeCompactedFromMob);
311 mobStore.updateCellsSizeCompactedToMob(cellsSizeCompactedToMob);
312 progress.complete();
313 return true;
314 }
315 }