View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.procedure2.store.wal;
20  
21  import java.io.IOException;
22  import java.util.Iterator;
23  import java.util.Map;
24  import java.util.HashMap;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.fs.FSDataInputStream;
29  import org.apache.hadoop.hbase.classification.InterfaceAudience;
30  import org.apache.hadoop.hbase.classification.InterfaceStability;
31  import org.apache.hadoop.hbase.ProcedureInfo;
32  import org.apache.hadoop.hbase.procedure2.Procedure;
33  import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker;
34  import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos;
35  import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALEntry;
36  
37  import com.google.protobuf.InvalidProtocolBufferException;
38  
39  /**
40   * Helper class that loads the procedures stored in a WAL
41   */
42  @InterfaceAudience.Private
43  @InterfaceStability.Evolving
44  public class ProcedureWALFormatReader {
45    private static final Log LOG = LogFactory.getLog(ProcedureWALFormatReader.class);
46  
47    private final ProcedureStoreTracker tracker;
48    //private final long compactionLogId;
49  
50    private final Map<Long, Procedure> procedures = new HashMap<Long, Procedure>();
51    private final Map<Long, ProcedureProtos.Procedure> localProcedures =
52      new HashMap<Long, ProcedureProtos.Procedure>();
53  
54    private long maxProcId = 0;
55  
56    public ProcedureWALFormatReader(final ProcedureStoreTracker tracker) {
57      this.tracker = tracker;
58    }
59  
60    public void read(ProcedureWALFile log, ProcedureWALFormat.Loader loader) throws IOException {
61      FSDataInputStream stream = log.getStream();
62      try {
63        boolean hasMore = true;
64        while (hasMore) {
65          ProcedureWALEntry entry = ProcedureWALFormat.readEntry(stream);
66          if (entry == null) {
67            LOG.warn("nothing left to decode. exiting with missing EOF");
68            hasMore = false;
69            break;
70          }
71          switch (entry.getType()) {
72            case INIT:
73              readInitEntry(entry);
74              break;
75            case INSERT:
76              readInsertEntry(entry);
77              break;
78            case UPDATE:
79            case COMPACT:
80              readUpdateEntry(entry);
81              break;
82            case DELETE:
83              readDeleteEntry(entry);
84              break;
85            case EOF:
86              hasMore = false;
87              break;
88            default:
89              throw new CorruptedWALProcedureStoreException("Invalid entry: " + entry);
90          }
91        }
92      } catch (InvalidProtocolBufferException e) {
93        LOG.error("got an exception while reading the procedure WAL: " + log, e);
94        loader.markCorruptedWAL(log, e);
95      }
96  
97      if (!localProcedures.isEmpty()) {
98        Iterator<Map.Entry<Long, ProcedureProtos.Procedure>> itd =
99          localProcedures.entrySet().iterator();
100       long minProcId = Long.MAX_VALUE;
101       long maxProcId = Long.MIN_VALUE;
102       while (itd.hasNext()) {
103         Map.Entry<Long, ProcedureProtos.Procedure> entry = itd.next();
104         itd.remove();
105 
106         long procId = entry.getKey();
107         minProcId = Math.min(minProcId, procId);
108         maxProcId = Math.max(maxProcId, procId);
109 
110         // Deserialize the procedure
111         Procedure proc = Procedure.convert(entry.getValue());
112         procedures.put(procId, proc);
113       }
114 
115       // TODO: Some procedure may be already runnables (see readInitEntry())
116       //       (we can also check the "update map" in the log trackers)
117       log.setProcIds(minProcId, maxProcId);
118     }
119   }
120 
121   public Iterator<Procedure> getProcedures() {
122     return procedures.values().iterator();
123   }
124 
125   private void loadEntries(final ProcedureWALEntry entry) {
126     for (ProcedureProtos.Procedure proc: entry.getProcedureList()) {
127       maxProcId = Math.max(maxProcId, proc.getProcId());
128       if (isRequired(proc.getProcId())) {
129         if (LOG.isTraceEnabled()) {
130           LOG.trace("read " + entry.getType() + " entry " + proc.getProcId());
131         }
132         localProcedures.put(proc.getProcId(), proc);
133         tracker.setDeleted(proc.getProcId(), false);
134       }
135     }
136   }
137 
138   private void readInitEntry(final ProcedureWALEntry entry)
139       throws IOException {
140     assert entry.getProcedureCount() == 1 : "Expected only one procedure";
141     // TODO: Make it runnable, before reading other files
142     loadEntries(entry);
143   }
144 
145   private void readInsertEntry(final ProcedureWALEntry entry) throws IOException {
146     assert entry.getProcedureCount() >= 1 : "Expected one or more procedures";
147     loadEntries(entry);
148   }
149 
150   private void readUpdateEntry(final ProcedureWALEntry entry) throws IOException {
151     assert entry.getProcedureCount() == 1 : "Expected only one procedure";
152     loadEntries(entry);
153   }
154 
155   private void readDeleteEntry(final ProcedureWALEntry entry) throws IOException {
156     assert entry.getProcedureCount() == 0 : "Expected no procedures";
157     assert entry.hasProcId() : "expected ProcID";
158     if (LOG.isTraceEnabled()) {
159       LOG.trace("read delete entry " + entry.getProcId());
160     }
161     maxProcId = Math.max(maxProcId, entry.getProcId());
162     localProcedures.remove(entry.getProcId());
163     assert !procedures.containsKey(entry.getProcId());
164     tracker.setDeleted(entry.getProcId(), true);
165   }
166 
167   private boolean isDeleted(final long procId) {
168     return tracker.isDeleted(procId) == ProcedureStoreTracker.DeleteState.YES;
169   }
170 
171   private boolean isRequired(final long procId) {
172     return !isDeleted(procId) && !procedures.containsKey(procId);
173   }
174 }