1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.procedure2.store.wal;
20
21 import java.io.IOException;
22 import java.util.Iterator;
23 import java.util.Map;
24 import java.util.HashMap;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.fs.FSDataInputStream;
29 import org.apache.hadoop.hbase.classification.InterfaceAudience;
30 import org.apache.hadoop.hbase.classification.InterfaceStability;
31 import org.apache.hadoop.hbase.ProcedureInfo;
32 import org.apache.hadoop.hbase.procedure2.Procedure;
33 import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker;
34 import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos;
35 import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALEntry;
36
37 import com.google.protobuf.InvalidProtocolBufferException;
38
39
40
41
42 @InterfaceAudience.Private
43 @InterfaceStability.Evolving
44 public class ProcedureWALFormatReader {
45 private static final Log LOG = LogFactory.getLog(ProcedureWALFormatReader.class);
46
47 private final ProcedureStoreTracker tracker;
48
49
50 private final Map<Long, Procedure> procedures = new HashMap<Long, Procedure>();
51 private final Map<Long, ProcedureProtos.Procedure> localProcedures =
52 new HashMap<Long, ProcedureProtos.Procedure>();
53
54 private long maxProcId = 0;
55
56 public ProcedureWALFormatReader(final ProcedureStoreTracker tracker) {
57 this.tracker = tracker;
58 }
59
60 public void read(ProcedureWALFile log, ProcedureWALFormat.Loader loader) throws IOException {
61 FSDataInputStream stream = log.getStream();
62 try {
63 boolean hasMore = true;
64 while (hasMore) {
65 ProcedureWALEntry entry = ProcedureWALFormat.readEntry(stream);
66 if (entry == null) {
67 LOG.warn("nothing left to decode. exiting with missing EOF");
68 hasMore = false;
69 break;
70 }
71 switch (entry.getType()) {
72 case INIT:
73 readInitEntry(entry);
74 break;
75 case INSERT:
76 readInsertEntry(entry);
77 break;
78 case UPDATE:
79 case COMPACT:
80 readUpdateEntry(entry);
81 break;
82 case DELETE:
83 readDeleteEntry(entry);
84 break;
85 case EOF:
86 hasMore = false;
87 break;
88 default:
89 throw new CorruptedWALProcedureStoreException("Invalid entry: " + entry);
90 }
91 }
92 } catch (InvalidProtocolBufferException e) {
93 LOG.error("got an exception while reading the procedure WAL: " + log, e);
94 loader.markCorruptedWAL(log, e);
95 }
96
97 if (!localProcedures.isEmpty()) {
98 Iterator<Map.Entry<Long, ProcedureProtos.Procedure>> itd =
99 localProcedures.entrySet().iterator();
100 long minProcId = Long.MAX_VALUE;
101 long maxProcId = Long.MIN_VALUE;
102 while (itd.hasNext()) {
103 Map.Entry<Long, ProcedureProtos.Procedure> entry = itd.next();
104 itd.remove();
105
106 long procId = entry.getKey();
107 minProcId = Math.min(minProcId, procId);
108 maxProcId = Math.max(maxProcId, procId);
109
110
111 Procedure proc = Procedure.convert(entry.getValue());
112 procedures.put(procId, proc);
113 }
114
115
116
117 log.setProcIds(minProcId, maxProcId);
118 }
119 }
120
121 public Iterator<Procedure> getProcedures() {
122 return procedures.values().iterator();
123 }
124
125 private void loadEntries(final ProcedureWALEntry entry) {
126 for (ProcedureProtos.Procedure proc: entry.getProcedureList()) {
127 maxProcId = Math.max(maxProcId, proc.getProcId());
128 if (isRequired(proc.getProcId())) {
129 if (LOG.isTraceEnabled()) {
130 LOG.trace("read " + entry.getType() + " entry " + proc.getProcId());
131 }
132 localProcedures.put(proc.getProcId(), proc);
133 tracker.setDeleted(proc.getProcId(), false);
134 }
135 }
136 }
137
138 private void readInitEntry(final ProcedureWALEntry entry)
139 throws IOException {
140 assert entry.getProcedureCount() == 1 : "Expected only one procedure";
141
142 loadEntries(entry);
143 }
144
145 private void readInsertEntry(final ProcedureWALEntry entry) throws IOException {
146 assert entry.getProcedureCount() >= 1 : "Expected one or more procedures";
147 loadEntries(entry);
148 }
149
150 private void readUpdateEntry(final ProcedureWALEntry entry) throws IOException {
151 assert entry.getProcedureCount() == 1 : "Expected only one procedure";
152 loadEntries(entry);
153 }
154
155 private void readDeleteEntry(final ProcedureWALEntry entry) throws IOException {
156 assert entry.getProcedureCount() == 0 : "Expected no procedures";
157 assert entry.hasProcId() : "expected ProcID";
158 if (LOG.isTraceEnabled()) {
159 LOG.trace("read delete entry " + entry.getProcId());
160 }
161 maxProcId = Math.max(maxProcId, entry.getProcId());
162 localProcedures.remove(entry.getProcId());
163 assert !procedures.containsKey(entry.getProcId());
164 tracker.setDeleted(entry.getProcId(), true);
165 }
166
167 private boolean isDeleted(final long procId) {
168 return tracker.isDeleted(procId) == ProcedureStoreTracker.DeleteState.YES;
169 }
170
171 private boolean isRequired(final long procId) {
172 return !isDeleted(procId) && !procedures.containsKey(procId);
173 }
174 }