View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.io.InterruptedIOException;
23  import java.net.InetSocketAddress;
24  import java.net.UnknownHostException;
25  import java.util.ArrayList;
26  import java.util.Collections;
27  import java.util.HashMap;
28  import java.util.Iterator;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.NavigableMap;
32  import java.util.Set;
33  import java.util.TreeSet;
34  import java.util.concurrent.ConcurrentHashMap;
35  import java.util.concurrent.atomic.AtomicLong;
36  
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.conf.Configuration;
40  import org.apache.hadoop.hbase.Cell;
41  import org.apache.hadoop.hbase.CellScannable;
42  import org.apache.hadoop.hbase.CellScanner;
43  import org.apache.hadoop.hbase.CellUtil;
44  import org.apache.hadoop.hbase.DoNotRetryIOException;
45  import org.apache.hadoop.hbase.DroppedSnapshotException;
46  import org.apache.hadoop.hbase.HBaseIOException;
47  import org.apache.hadoop.hbase.HConstants;
48  import org.apache.hadoop.hbase.HRegionInfo;
49  import org.apache.hadoop.hbase.HTableDescriptor;
50  import org.apache.hadoop.hbase.MetaTableAccessor;
51  import org.apache.hadoop.hbase.NotServingRegionException;
52  import org.apache.hadoop.hbase.ServerName;
53  import org.apache.hadoop.hbase.TableName;
54  import org.apache.hadoop.hbase.UnknownScannerException;
55  import org.apache.hadoop.hbase.classification.InterfaceAudience;
56  import org.apache.hadoop.hbase.client.Append;
57  import org.apache.hadoop.hbase.client.ConnectionUtils;
58  import org.apache.hadoop.hbase.client.Delete;
59  import org.apache.hadoop.hbase.client.Durability;
60  import org.apache.hadoop.hbase.client.Get;
61  import org.apache.hadoop.hbase.client.Increment;
62  import org.apache.hadoop.hbase.client.Mutation;
63  import org.apache.hadoop.hbase.client.Put;
64  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
65  import org.apache.hadoop.hbase.client.Result;
66  import org.apache.hadoop.hbase.client.RowMutations;
67  import org.apache.hadoop.hbase.client.Scan;
68  import org.apache.hadoop.hbase.coordination.CloseRegionCoordination;
69  import org.apache.hadoop.hbase.coordination.OpenRegionCoordination;
70  import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
71  import org.apache.hadoop.hbase.exceptions.MergeRegionException;
72  import org.apache.hadoop.hbase.exceptions.OperationConflictException;
73  import org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException;
74  import org.apache.hadoop.hbase.filter.ByteArrayComparable;
75  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
76  import org.apache.hadoop.hbase.ipc.HBaseRPCErrorHandler;
77  import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
78  import org.apache.hadoop.hbase.ipc.PriorityFunction;
79  import org.apache.hadoop.hbase.ipc.QosPriority;
80  import org.apache.hadoop.hbase.ipc.RpcCallContext;
81  import org.apache.hadoop.hbase.ipc.RpcServer;
82  import org.apache.hadoop.hbase.ipc.RpcServer.BlockingServiceAndInterface;
83  import org.apache.hadoop.hbase.ipc.RpcServerInterface;
84  import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
85  import org.apache.hadoop.hbase.ipc.ServerRpcController;
86  import org.apache.hadoop.hbase.master.MasterRpcServices;
87  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
88  import org.apache.hadoop.hbase.protobuf.RequestConverter;
89  import org.apache.hadoop.hbase.protobuf.ResponseConverter;
90  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
91  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
92  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
93  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest;
94  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionResponse;
95  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionRequest;
96  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionResponse;
97  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
98  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
99  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
100 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
101 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest;
102 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
103 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest;
104 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileResponse;
105 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest;
106 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsResponse;
107 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
108 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest.RegionOpenInfo;
109 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
110 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse.RegionOpeningState;
111 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryRequest;
112 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryResponse;
113 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest;
114 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterResponse;
115 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest;
116 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionResponse;
117 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerRequest;
118 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerResponse;
119 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateConfigurationRequest;
120 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateConfigurationResponse;
121 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesRequest;
122 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesResponse;
123 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WALEntry;
124 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionRequest;
125 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionResponse;
126 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
127 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
128 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest.FamilyPath;
129 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse;
130 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
131 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Condition;
132 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
133 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
134 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
135 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
136 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest;
137 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiResponse;
138 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateRequest;
139 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateResponse;
140 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
141 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
142 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionAction;
143 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
144 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultOrException;
145 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
146 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
147 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo;
148 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
149 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
150 import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.RequestHeader;
151 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.BulkLoadDescriptor;
152 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
153 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
154 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
155 import org.apache.hadoop.hbase.quotas.OperationQuota;
156 import org.apache.hadoop.hbase.quotas.RegionServerQuotaManager;
157 import org.apache.hadoop.hbase.regionserver.Leases.LeaseStillHeldException;
158 import org.apache.hadoop.hbase.regionserver.Region.Operation;
159 import org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope;
160 import org.apache.hadoop.hbase.regionserver.handler.OpenMetaHandler;
161 import org.apache.hadoop.hbase.regionserver.handler.OpenPriorityRegionHandler;
162 import org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler;
163 import org.apache.hadoop.hbase.wal.WAL;
164 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
165 import org.apache.hadoop.hbase.util.Bytes;
166 import org.apache.hadoop.hbase.util.Counter;
167 import org.apache.hadoop.hbase.util.DNS;
168 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
169 import org.apache.hadoop.hbase.util.Pair;
170 import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
171 import org.apache.hadoop.hbase.util.Strings;
172 import org.apache.hadoop.hbase.wal.WALKey;
173 import org.apache.hadoop.hbase.wal.WALSplitter;
174 import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
175 import org.apache.zookeeper.KeeperException;
176 
177 import com.google.common.annotations.VisibleForTesting;
178 import com.google.protobuf.ByteString;
179 import com.google.protobuf.Message;
180 import com.google.protobuf.RpcController;
181 import com.google.protobuf.ServiceException;
182 import com.google.protobuf.TextFormat;
183 
184 /**
185  * Implements the regionserver RPC services.
186  */
187 @InterfaceAudience.Private
188 @SuppressWarnings("deprecation")
189 public class RSRpcServices implements HBaseRPCErrorHandler,
190     AdminService.BlockingInterface, ClientService.BlockingInterface, PriorityFunction {
191   protected static final Log LOG = LogFactory.getLog(RSRpcServices.class);
192 
193   /** RPC scheduler to use for the region server. */
194   public static final String REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS =
195     "hbase.region.server.rpc.scheduler.factory.class";
196 
197   /**
198    * Minimum allowable time limit delta (in milliseconds) that can be enforced during scans. This
199    * configuration exists to prevent the scenario where a time limit is specified to be so
200    * restrictive that the time limit is reached immediately (before any cells are scanned).
201    */
202   private static final String REGION_SERVER_RPC_MINIMUM_SCAN_TIME_LIMIT_DELTA =
203       "hbase.region.server.rpc.minimum.scan.time.limit.delta";
204   /**
205    * Default value of {@link RSRpcServices#REGION_SERVER_RPC_MINIMUM_SCAN_TIME_LIMIT_DELTA}
206    */
207   private static final long DEFAULT_REGION_SERVER_RPC_MINIMUM_SCAN_TIME_LIMIT_DELTA = 10;
208 
209   // Request counter. (Includes requests that are not serviced by regions.)
210   final Counter requestCount = new Counter();
211 
212   // Request counter for rpc get
213   final Counter rpcGetRequestCount = new Counter();
214 
215   // Request counter for rpc scan
216   final Counter rpcScanRequestCount = new Counter();
217 
218   // Request counter for rpc multi
219   final Counter rpcMultiRequestCount = new Counter();
220 
221   // Request counter for rpc mutate
222   final Counter rpcMutateRequestCount = new Counter();
223 
224   // Server to handle client requests.
225   final RpcServerInterface rpcServer;
226   final InetSocketAddress isa;
227 
228   private final HRegionServer regionServer;
229   private final long maxScannerResultSize;
230 
231   // The reference to the priority extraction function
232   private final PriorityFunction priority;
233 
234   private final AtomicLong scannerIdGen = new AtomicLong(0L);
235   private final ConcurrentHashMap<String, RegionScannerHolder> scanners =
236     new ConcurrentHashMap<String, RegionScannerHolder>();
237 
238   /**
239    * The lease timeout period for client scanners (milliseconds).
240    */
241   private final int scannerLeaseTimeoutPeriod;
242 
243   /**
244    * The RPC timeout period (milliseconds)
245    */
246   private final int rpcTimeout;
247 
248   /**
249    * The minimum allowable delta to use for the scan limit
250    */
251   private final long minimumScanTimeLimitDelta;
252 
253   /**
254    * Holder class which holds the RegionScanner and nextCallSeq together.
255    */
256   private static class RegionScannerHolder {
257     private AtomicLong nextCallSeq = new AtomicLong(0);
258     private RegionScanner s;
259     private Region r;
260 
261     public RegionScannerHolder(RegionScanner s, Region r) {
262       this.s = s;
263       this.r = r;
264     }
265 
266     private long getNextCallSeq() {
267       return nextCallSeq.get();
268     }
269 
270     private void incNextCallSeq() {
271       nextCallSeq.incrementAndGet();
272     }
273 
274     private void rollbackNextCallSeq() {
275       nextCallSeq.decrementAndGet();
276     }
277   }
278 
279   /**
280    * Instantiated as a scanner lease. If the lease times out, the scanner is
281    * closed
282    */
283   private class ScannerListener implements LeaseListener {
284     private final String scannerName;
285 
286     ScannerListener(final String n) {
287       this.scannerName = n;
288     }
289 
290     @Override
291     public void leaseExpired() {
292       RegionScannerHolder rsh = scanners.remove(this.scannerName);
293       if (rsh != null) {
294         RegionScanner s = rsh.s;
295         LOG.info("Scanner " + this.scannerName + " lease expired on region "
296           + s.getRegionInfo().getRegionNameAsString());
297         Region region = null;
298         try {
299           region = regionServer.getRegion(s.getRegionInfo().getRegionName());
300           if (region != null && region.getCoprocessorHost() != null) {
301             region.getCoprocessorHost().preScannerClose(s);
302           }
303 
304           s.close();
305           if (region != null && region.getCoprocessorHost() != null) {
306             region.getCoprocessorHost().postScannerClose(s);
307           }
308         } catch (IOException e) {
309           LOG.error("Closing scanner for "
310               + s.getRegionInfo().getRegionNameAsString(), e);
311         } finally {
312           try {
313             s.close();
314             if (region != null && region.getCoprocessorHost() != null) {
315               region.getCoprocessorHost().postScannerClose(s);
316             }
317           } catch (IOException e) {
318             LOG.error("Closing scanner for " + s.getRegionInfo().getRegionNameAsString(), e);
319           }
320         }
321       } else {
322         LOG.warn("Scanner " + this.scannerName + " lease expired, but no related" +
323           " scanner found, hence no chance to close that related scanner!");
324       }
325     }
326   }
327 
328   private static ResultOrException getResultOrException(
329       final ClientProtos.Result r, final int index, final ClientProtos.RegionLoadStats stats) {
330     return getResultOrException(ResponseConverter.buildActionResult(r, stats), index);
331   }
332 
333   private static ResultOrException getResultOrException(final Exception e, final int index) {
334     return getResultOrException(ResponseConverter.buildActionResult(e), index);
335   }
336 
337   private static ResultOrException getResultOrException(
338       final ResultOrException.Builder builder, final int index) {
339     return builder.setIndex(index).build();
340   }
341 
342   /**
343    * Starts the nonce operation for a mutation, if needed.
344    * @param mutation Mutation.
345    * @param nonceGroup Nonce group from the request.
346    * @returns Nonce used (can be NO_NONCE).
347    */
348   private long startNonceOperation(final MutationProto mutation, long nonceGroup)
349       throws IOException, OperationConflictException {
350     if (regionServer.nonceManager == null || !mutation.hasNonce()) return HConstants.NO_NONCE;
351     boolean canProceed = false;
352     try {
353       canProceed = regionServer.nonceManager.startOperation(
354         nonceGroup, mutation.getNonce(), regionServer);
355     } catch (InterruptedException ex) {
356       throw new InterruptedIOException("Nonce start operation interrupted");
357     }
358     if (!canProceed) {
359       // TODO: instead, we could convert append/increment to get w/mvcc
360       String message = "The operation with nonce {" + nonceGroup + ", " + mutation.getNonce()
361         + "} on row [" + Bytes.toString(mutation.getRow().toByteArray())
362         + "] may have already completed";
363       throw new OperationConflictException(message);
364     }
365     return mutation.getNonce();
366   }
367 
368   /**
369    * Ends nonce operation for a mutation, if needed.
370    * @param mutation Mutation.
371    * @param nonceGroup Nonce group from the request. Always 0 in initial implementation.
372    * @param success Whether the operation for this nonce has succeeded.
373    */
374   private void endNonceOperation(final MutationProto mutation,
375       long nonceGroup, boolean success) {
376     if (regionServer.nonceManager != null && mutation.hasNonce()) {
377       regionServer.nonceManager.endOperation(nonceGroup, mutation.getNonce(), success);
378     }
379   }
380 
381   /**
382    * @return True if current call supports cellblocks
383    */
384   private boolean isClientCellBlockSupport() {
385     RpcCallContext context = RpcServer.getCurrentCall();
386     return context != null && context.isClientCellBlockSupport();
387   }
388 
389   private void addResult(final MutateResponse.Builder builder,
390       final Result result, final PayloadCarryingRpcController rpcc) {
391     if (result == null) return;
392     if (isClientCellBlockSupport()) {
393       builder.setResult(ProtobufUtil.toResultNoData(result));
394       rpcc.setCellScanner(result.cellScanner());
395     } else {
396       ClientProtos.Result pbr = ProtobufUtil.toResult(result);
397       builder.setResult(pbr);
398     }
399   }
400 
401   private void addResults(final ScanResponse.Builder builder, final List<Result> results,
402       final RpcController controller, boolean isDefaultRegion) {
403     builder.setStale(!isDefaultRegion);
404     if (results == null || results.isEmpty()) return;
405     if (isClientCellBlockSupport()) {
406       for (Result res : results) {
407         builder.addCellsPerResult(res.size());
408         builder.addPartialFlagPerResult(res.isPartial());
409       }
410       ((PayloadCarryingRpcController)controller).
411         setCellScanner(CellUtil.createCellScanner(results));
412     } else {
413       for (Result res: results) {
414         ClientProtos.Result pbr = ProtobufUtil.toResult(res);
415         builder.addResults(pbr);
416       }
417     }
418   }
419 
420   /**
421    * Mutate a list of rows atomically.
422    *
423    * @param region
424    * @param actions
425    * @param cellScanner if non-null, the mutation data -- the Cell content.
426    * @throws IOException
427    */
428   private ClientProtos.RegionLoadStats mutateRows(final Region region,
429       final List<ClientProtos.Action> actions,
430       final CellScanner cellScanner) throws IOException {
431     if (!region.getRegionInfo().isMetaTable()) {
432       regionServer.cacheFlusher.reclaimMemStoreMemory();
433     }
434     RowMutations rm = null;
435     for (ClientProtos.Action action: actions) {
436       if (action.hasGet()) {
437         throw new DoNotRetryIOException("Atomic put and/or delete only, not a Get=" +
438           action.getGet());
439       }
440       MutationType type = action.getMutation().getMutateType();
441       if (rm == null) {
442         rm = new RowMutations(action.getMutation().getRow().toByteArray());
443       }
444       switch (type) {
445       case PUT:
446         rm.add(ProtobufUtil.toPut(action.getMutation(), cellScanner));
447         break;
448       case DELETE:
449         rm.add(ProtobufUtil.toDelete(action.getMutation(), cellScanner));
450         break;
451       default:
452           throw new DoNotRetryIOException("Atomic put and/or delete only, not " + type.name());
453       }
454     }
455     region.mutateRow(rm);
456     return ((HRegion)region).getRegionStats();
457   }
458 
459   /**
460    * Mutate a list of rows atomically.
461    *
462    * @param region
463    * @param actions
464    * @param cellScanner if non-null, the mutation data -- the Cell content.
465    * @param row
466    * @param family
467    * @param qualifier
468    * @param compareOp
469    * @param comparator @throws IOException
470    */
471   private boolean checkAndRowMutate(final Region region, final List<ClientProtos.Action> actions,
472       final CellScanner cellScanner, byte[] row, byte[] family, byte[] qualifier,
473       CompareOp compareOp, ByteArrayComparable comparator) throws IOException {
474     if (!region.getRegionInfo().isMetaTable()) {
475       regionServer.cacheFlusher.reclaimMemStoreMemory();
476     }
477     RowMutations rm = null;
478     for (ClientProtos.Action action: actions) {
479       if (action.hasGet()) {
480         throw new DoNotRetryIOException("Atomic put and/or delete only, not a Get=" +
481             action.getGet());
482       }
483       MutationType type = action.getMutation().getMutateType();
484       if (rm == null) {
485         rm = new RowMutations(action.getMutation().getRow().toByteArray());
486       }
487       switch (type) {
488       case PUT:
489         rm.add(ProtobufUtil.toPut(action.getMutation(), cellScanner));
490         break;
491       case DELETE:
492         rm.add(ProtobufUtil.toDelete(action.getMutation(), cellScanner));
493         break;
494       default:
495         throw new DoNotRetryIOException("Atomic put and/or delete only, not " + type.name());
496       }
497     }
498     return region.checkAndRowMutate(row, family, qualifier, compareOp, comparator, rm, Boolean.TRUE);
499   }
500 
501   /**
502    * Execute an append mutation.
503    *
504    * @param region
505    * @param m
506    * @param cellScanner
507    * @return result to return to client if default operation should be
508    * bypassed as indicated by RegionObserver, null otherwise
509    * @throws IOException
510    */
511   private Result append(final Region region, final OperationQuota quota, final MutationProto m,
512       final CellScanner cellScanner, long nonceGroup) throws IOException {
513     long before = EnvironmentEdgeManager.currentTime();
514     Append append = ProtobufUtil.toAppend(m, cellScanner);
515     quota.addMutation(append);
516     Result r = null;
517     if (region.getCoprocessorHost() != null) {
518       r = region.getCoprocessorHost().preAppend(append);
519     }
520     if (r == null) {
521       long nonce = startNonceOperation(m, nonceGroup);
522       boolean success = false;
523       try {
524         r = region.append(append, nonceGroup, nonce);
525         success = true;
526       } finally {
527         endNonceOperation(m, nonceGroup, success);
528       }
529       if (region.getCoprocessorHost() != null) {
530         region.getCoprocessorHost().postAppend(append, r);
531       }
532     }
533     if (regionServer.metricsRegionServer != null) {
534       regionServer.metricsRegionServer.updateAppend(
535         EnvironmentEdgeManager.currentTime() - before);
536     }
537     return r;
538   }
539 
540   /**
541    * Execute an increment mutation.
542    *
543    * @param region
544    * @param mutation
545    * @return the Result
546    * @throws IOException
547    */
548   private Result increment(final Region region, final OperationQuota quota,
549       final MutationProto mutation, final CellScanner cells, long nonceGroup) throws IOException {
550     long before = EnvironmentEdgeManager.currentTime();
551     Increment increment = ProtobufUtil.toIncrement(mutation, cells);
552     quota.addMutation(increment);
553     Result r = null;
554     if (region.getCoprocessorHost() != null) {
555       r = region.getCoprocessorHost().preIncrement(increment);
556     }
557     if (r == null) {
558       long nonce = startNonceOperation(mutation, nonceGroup);
559       boolean success = false;
560       try {
561         r = region.increment(increment, nonceGroup, nonce);
562         success = true;
563       } finally {
564         endNonceOperation(mutation, nonceGroup, success);
565       }
566       if (region.getCoprocessorHost() != null) {
567         r = region.getCoprocessorHost().postIncrement(increment, r);
568       }
569     }
570     if (regionServer.metricsRegionServer != null) {
571       regionServer.metricsRegionServer.updateIncrement(
572         EnvironmentEdgeManager.currentTime() - before);
573     }
574     return r;
575   }
576 
577   /**
578    * Run through the regionMutation <code>rm</code> and per Mutation, do the work, and then when
579    * done, add an instance of a {@link ResultOrException} that corresponds to each Mutation.
580    * @param region
581    * @param actions
582    * @param cellScanner
583    * @param builder
584    * @param cellsToReturn  Could be null. May be allocated in this method.  This is what this
585    * method returns as a 'result'.
586    * @return Return the <code>cellScanner</code> passed
587    */
588   private List<CellScannable> doNonAtomicRegionMutation(final Region region,
589       final OperationQuota quota, final RegionAction actions, final CellScanner cellScanner,
590       final RegionActionResult.Builder builder, List<CellScannable> cellsToReturn, long nonceGroup) {
591     // Gather up CONTIGUOUS Puts and Deletes in this mutations List.  Idea is that rather than do
592     // one at a time, we instead pass them in batch.  Be aware that the corresponding
593     // ResultOrException instance that matches each Put or Delete is then added down in the
594     // doBatchOp call.  We should be staying aligned though the Put and Delete are deferred/batched
595     List<ClientProtos.Action> mutations = null;
596     for (ClientProtos.Action action: actions.getActionList()) {
597       ClientProtos.ResultOrException.Builder resultOrExceptionBuilder = null;
598       try {
599         Result r = null;
600         if (action.hasGet()) {
601           Get get = ProtobufUtil.toGet(action.getGet());
602           r = region.get(get);
603         } else if (action.hasServiceCall()) {
604           resultOrExceptionBuilder = ResultOrException.newBuilder();
605           try {
606             Message result = execServiceOnRegion(region, action.getServiceCall());
607             ClientProtos.CoprocessorServiceResult.Builder serviceResultBuilder =
608                 ClientProtos.CoprocessorServiceResult.newBuilder();
609             resultOrExceptionBuilder.setServiceResult(
610                 serviceResultBuilder.setValue(
611                   serviceResultBuilder.getValueBuilder()
612                     .setName(result.getClass().getName())
613                     .setValue(result.toByteString())));
614           } catch (IOException ioe) {
615             rpcServer.getMetrics().exception(ioe);
616             resultOrExceptionBuilder.setException(ResponseConverter.buildException(ioe));
617           }
618         } else if (action.hasMutation()) {
619           MutationType type = action.getMutation().getMutateType();
620           if (type != MutationType.PUT && type != MutationType.DELETE && mutations != null &&
621               !mutations.isEmpty()) {
622             // Flush out any Puts or Deletes already collected.
623             doBatchOp(builder, region, quota, mutations, cellScanner);
624             mutations.clear();
625           }
626           switch (type) {
627           case APPEND:
628             r = append(region, quota, action.getMutation(), cellScanner, nonceGroup);
629             break;
630           case INCREMENT:
631             r = increment(region, quota, action.getMutation(), cellScanner,  nonceGroup);
632             break;
633           case PUT:
634           case DELETE:
635             // Collect the individual mutations and apply in a batch
636             if (mutations == null) {
637               mutations = new ArrayList<ClientProtos.Action>(actions.getActionCount());
638             }
639             mutations.add(action);
640             break;
641           default:
642             throw new DoNotRetryIOException("Unsupported mutate type: " + type.name());
643           }
644         } else {
645           throw new HBaseIOException("Unexpected Action type");
646         }
647         if (r != null) {
648           ClientProtos.Result pbResult = null;
649           if (isClientCellBlockSupport()) {
650             pbResult = ProtobufUtil.toResultNoData(r);
651             //  Hard to guess the size here.  Just make a rough guess.
652             if (cellsToReturn == null) cellsToReturn = new ArrayList<CellScannable>();
653             cellsToReturn.add(r);
654           } else {
655             pbResult = ProtobufUtil.toResult(r);
656           }
657           resultOrExceptionBuilder =
658             ClientProtos.ResultOrException.newBuilder().setResult(pbResult);
659         }
660         // Could get to here and there was no result and no exception.  Presumes we added
661         // a Put or Delete to the collecting Mutations List for adding later.  In this
662         // case the corresponding ResultOrException instance for the Put or Delete will be added
663         // down in the doBatchOp method call rather than up here.
664       } catch (IOException ie) {
665         rpcServer.getMetrics().exception(ie);
666         resultOrExceptionBuilder = ResultOrException.newBuilder().
667           setException(ResponseConverter.buildException(ie));
668       }
669       if (resultOrExceptionBuilder != null) {
670         // Propagate index.
671         resultOrExceptionBuilder.setIndex(action.getIndex());
672         builder.addResultOrException(resultOrExceptionBuilder.build());
673       }
674     }
675     // Finish up any outstanding mutations
676     if (mutations != null && !mutations.isEmpty()) {
677       doBatchOp(builder, region, quota, mutations, cellScanner);
678     }
679     return cellsToReturn;
680   }
681 
682   /**
683    * Execute a list of Put/Delete mutations.
684    *
685    * @param builder
686    * @param region
687    * @param mutations
688    */
689   private void doBatchOp(final RegionActionResult.Builder builder, final Region region,
690       final OperationQuota quota,
691       final List<ClientProtos.Action> mutations, final CellScanner cells) {
692     Mutation[] mArray = new Mutation[mutations.size()];
693     long before = EnvironmentEdgeManager.currentTime();
694     boolean batchContainsPuts = false, batchContainsDelete = false;
695     try {
696       int i = 0;
697       for (ClientProtos.Action action: mutations) {
698         MutationProto m = action.getMutation();
699         Mutation mutation;
700         if (m.getMutateType() == MutationType.PUT) {
701           mutation = ProtobufUtil.toPut(m, cells);
702           batchContainsPuts = true;
703         } else {
704           mutation = ProtobufUtil.toDelete(m, cells);
705           batchContainsDelete = true;
706         }
707         mArray[i++] = mutation;
708         quota.addMutation(mutation);
709       }
710 
711       if (!region.getRegionInfo().isMetaTable()) {
712         regionServer.cacheFlusher.reclaimMemStoreMemory();
713       }
714 
715       OperationStatus codes[] = region.batchMutate(mArray, HConstants.NO_NONCE,
716         HConstants.NO_NONCE);
717       for (i = 0; i < codes.length; i++) {
718         int index = mutations.get(i).getIndex();
719         Exception e = null;
720         switch (codes[i].getOperationStatusCode()) {
721           case BAD_FAMILY:
722             e = new NoSuchColumnFamilyException(codes[i].getExceptionMsg());
723             builder.addResultOrException(getResultOrException(e, index));
724             break;
725 
726           case SANITY_CHECK_FAILURE:
727             e = new FailedSanityCheckException(codes[i].getExceptionMsg());
728             builder.addResultOrException(getResultOrException(e, index));
729             break;
730 
731           default:
732             e = new DoNotRetryIOException(codes[i].getExceptionMsg());
733             builder.addResultOrException(getResultOrException(e, index));
734             break;
735 
736           case SUCCESS:
737             builder.addResultOrException(getResultOrException(
738               ClientProtos.Result.getDefaultInstance(), index,
739                 ((HRegion)region).getRegionStats()));
740             break;
741         }
742       }
743     } catch (IOException ie) {
744       for (int i = 0; i < mutations.size(); i++) {
745         builder.addResultOrException(getResultOrException(ie, mutations.get(i).getIndex()));
746       }
747     }
748     if (regionServer.metricsRegionServer != null) {
749       long after = EnvironmentEdgeManager.currentTime();
750       if (batchContainsPuts) {
751         regionServer.metricsRegionServer.updatePut(after - before);
752       }
753       if (batchContainsDelete) {
754         regionServer.metricsRegionServer.updateDelete(after - before);
755       }
756     }
757   }
758 
759   /**
760    * Execute a list of Put/Delete mutations. The function returns OperationStatus instead of
761    * constructing MultiResponse to save a possible loop if caller doesn't need MultiResponse.
762    * @param region
763    * @param mutations
764    * @param replaySeqId
765    * @return an array of OperationStatus which internally contains the OperationStatusCode and the
766    *         exceptionMessage if any
767    * @throws IOException
768    */
769   private OperationStatus [] doReplayBatchOp(final Region region,
770       final List<WALSplitter.MutationReplay> mutations, long replaySeqId) throws IOException {
771     long before = EnvironmentEdgeManager.currentTime();
772     boolean batchContainsPuts = false, batchContainsDelete = false;
773     try {
774       for (Iterator<WALSplitter.MutationReplay> it = mutations.iterator(); it.hasNext();) {
775         WALSplitter.MutationReplay m = it.next();
776 
777         if (m.type == MutationType.PUT) {
778           batchContainsPuts = true;
779         } else {
780           batchContainsDelete = true;
781         }
782 
783         NavigableMap<byte[], List<Cell>> map = m.mutation.getFamilyCellMap();
784         List<Cell> metaCells = map.get(WALEdit.METAFAMILY);
785         if (metaCells != null && !metaCells.isEmpty()) {
786           for (Cell metaCell : metaCells) {
787             CompactionDescriptor compactionDesc = WALEdit.getCompaction(metaCell);
788             boolean isDefaultReplica = RegionReplicaUtil.isDefaultReplica(region.getRegionInfo());
789             HRegion hRegion = (HRegion)region;
790             if (compactionDesc != null) {
791               // replay the compaction. Remove the files from stores only if we are the primary
792               // region replica (thus own the files)
793               hRegion.replayWALCompactionMarker(compactionDesc, !isDefaultReplica, isDefaultReplica,
794                 replaySeqId);
795               continue;
796             }
797             FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(metaCell);
798             if (flushDesc != null && !isDefaultReplica) {
799               hRegion.replayWALFlushMarker(flushDesc, replaySeqId);
800               continue;
801             }
802             RegionEventDescriptor regionEvent = WALEdit.getRegionEventDescriptor(metaCell);
803             if (regionEvent != null && !isDefaultReplica) {
804               hRegion.replayWALRegionEventMarker(regionEvent);
805               continue;
806             }
807             BulkLoadDescriptor bulkLoadEvent = WALEdit.getBulkLoadDescriptor(metaCell);
808             if (bulkLoadEvent != null) {
809               hRegion.replayWALBulkLoadEventMarker(bulkLoadEvent);
810               continue;
811             }
812           }
813           it.remove();
814         }
815       }
816       requestCount.add(mutations.size());
817       if (!region.getRegionInfo().isMetaTable()) {
818         regionServer.cacheFlusher.reclaimMemStoreMemory();
819       }
820       return region.batchReplay(mutations.toArray(
821         new WALSplitter.MutationReplay[mutations.size()]), replaySeqId);
822     } finally {
823       if (regionServer.metricsRegionServer != null) {
824         long after = EnvironmentEdgeManager.currentTime();
825           if (batchContainsPuts) {
826           regionServer.metricsRegionServer.updatePut(after - before);
827         }
828         if (batchContainsDelete) {
829           regionServer.metricsRegionServer.updateDelete(after - before);
830         }
831       }
832     }
833   }
834 
835   private void closeAllScanners() {
836     // Close any outstanding scanners. Means they'll get an UnknownScanner
837     // exception next time they come in.
838     for (Map.Entry<String, RegionScannerHolder> e : scanners.entrySet()) {
839       try {
840         e.getValue().s.close();
841       } catch (IOException ioe) {
842         LOG.warn("Closing scanner " + e.getKey(), ioe);
843       }
844     }
845   }
846 
847   public RSRpcServices(HRegionServer rs) throws IOException {
848     regionServer = rs;
849 
850     RpcSchedulerFactory rpcSchedulerFactory;
851     try {
852       Class<?> rpcSchedulerFactoryClass = rs.conf.getClass(
853           REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS,
854           SimpleRpcSchedulerFactory.class);
855       rpcSchedulerFactory = ((RpcSchedulerFactory) rpcSchedulerFactoryClass.newInstance());
856     } catch (InstantiationException e) {
857       throw new IllegalArgumentException(e);
858     } catch (IllegalAccessException e) {
859       throw new IllegalArgumentException(e);
860     }
861     // Server to handle client requests.
862     InetSocketAddress initialIsa;
863     InetSocketAddress bindAddress;
864     if(this instanceof MasterRpcServices) {
865       String hostname = getHostname(rs.conf, true);
866       int port = rs.conf.getInt(HConstants.MASTER_PORT, HConstants.DEFAULT_MASTER_PORT);
867       // Creation of a HSA will force a resolve.
868       initialIsa = new InetSocketAddress(hostname, port);
869       bindAddress = new InetSocketAddress(rs.conf.get("hbase.master.ipc.address", hostname), port);
870     } else {
871       String hostname = getHostname(rs.conf, false);
872       int port = rs.conf.getInt(HConstants.REGIONSERVER_PORT,
873         HConstants.DEFAULT_REGIONSERVER_PORT);
874       // Creation of a HSA will force a resolve.
875       initialIsa = new InetSocketAddress(hostname, port);
876       bindAddress = new InetSocketAddress(
877         rs.conf.get("hbase.regionserver.ipc.address", hostname), port);
878     }
879     if (initialIsa.getAddress() == null) {
880       throw new IllegalArgumentException("Failed resolve of " + initialIsa);
881     }
882     priority = new AnnotationReadingPriorityFunction(this);
883     String name = rs.getProcessName() + "/" + initialIsa.toString();
884     // Set how many times to retry talking to another server over HConnection.
885     ConnectionUtils.setServerSideHConnectionRetriesConfig(rs.conf, name, LOG);
886     rpcServer = new RpcServer(rs, name, getServices(),
887       bindAddress, // use final bindAddress for this server.
888       rs.conf,
889       rpcSchedulerFactory.create(rs.conf, this, rs));
890 
891     scannerLeaseTimeoutPeriod = rs.conf.getInt(
892       HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
893       HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
894     maxScannerResultSize = rs.conf.getLong(
895       HConstants.HBASE_SERVER_SCANNER_MAX_RESULT_SIZE_KEY,
896       HConstants.DEFAULT_HBASE_SERVER_SCANNER_MAX_RESULT_SIZE);
897     rpcTimeout = rs.conf.getInt(
898       HConstants.HBASE_RPC_TIMEOUT_KEY,
899       HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
900     minimumScanTimeLimitDelta = rs.conf.getLong(
901       REGION_SERVER_RPC_MINIMUM_SCAN_TIME_LIMIT_DELTA,
902       DEFAULT_REGION_SERVER_RPC_MINIMUM_SCAN_TIME_LIMIT_DELTA);
903 
904     InetSocketAddress address = rpcServer.getListenerAddress();
905     if (address == null) {
906       throw new IOException("Listener channel is closed");
907     }
908     // Set our address, however we need the final port that was given to rpcServer
909     isa = new InetSocketAddress(initialIsa.getHostName(), address.getPort());
910     rpcServer.setErrorHandler(this);
911     rs.setName(name);
912   }
913 
914   public static String getHostname(Configuration conf, boolean isMaster)
915       throws UnknownHostException {
916     String hostname = conf.get(isMaster? HRegionServer.MASTER_HOSTNAME_KEY :
917       HRegionServer.RS_HOSTNAME_KEY);
918     if (hostname == null || hostname.isEmpty()) {
919       String masterOrRS = isMaster ? "master" : "regionserver";
920       return Strings.domainNamePointerToHostName(DNS.getDefaultHost(
921         conf.get("hbase." + masterOrRS + ".dns.interface", "default"),
922         conf.get("hbase." + masterOrRS + ".dns.nameserver", "default")));
923     } else {
924       LOG.info("hostname is configured to be " + hostname);
925       return hostname;
926     }
927   }
928 
929   RegionScanner getScanner(long scannerId) {
930     String scannerIdString = Long.toString(scannerId);
931     RegionScannerHolder scannerHolder = scanners.get(scannerIdString);
932     if (scannerHolder != null) {
933       return scannerHolder.s;
934     }
935     return null;
936   }
937 
938   /**
939    * Get the vtime associated with the scanner.
940    * Currently the vtime is the number of "next" calls.
941    */
942   long getScannerVirtualTime(long scannerId) {
943     String scannerIdString = Long.toString(scannerId);
944     RegionScannerHolder scannerHolder = scanners.get(scannerIdString);
945     if (scannerHolder != null) {
946       return scannerHolder.getNextCallSeq();
947     }
948     return 0L;
949   }
950 
951   long addScanner(RegionScanner s, Region r) throws LeaseStillHeldException {
952     long scannerId = this.scannerIdGen.incrementAndGet();
953     String scannerName = String.valueOf(scannerId);
954 
955     RegionScannerHolder existing =
956       scanners.putIfAbsent(scannerName, new RegionScannerHolder(s, r));
957     assert existing == null : "scannerId must be unique within regionserver's whole lifecycle!";
958 
959     regionServer.leases.createLease(scannerName, this.scannerLeaseTimeoutPeriod,
960         new ScannerListener(scannerName));
961     return scannerId;
962   }
963 
964   /**
965    * Find the HRegion based on a region specifier
966    *
967    * @param regionSpecifier the region specifier
968    * @return the corresponding region
969    * @throws IOException if the specifier is not null,
970    *    but failed to find the region
971    */
972   Region getRegion(
973       final RegionSpecifier regionSpecifier) throws IOException {
974     return regionServer.getRegionByEncodedName(regionSpecifier.getValue().toByteArray(),
975         ProtobufUtil.getRegionEncodedName(regionSpecifier));
976   }
977 
978   @VisibleForTesting
979   public PriorityFunction getPriority() {
980     return priority;
981   }
982 
983   Configuration getConfiguration() {
984     return regionServer.getConfiguration();
985   }
986 
987   private RegionServerQuotaManager getQuotaManager() {
988     return regionServer.getRegionServerQuotaManager();
989   }
990 
991   void start() {
992     rpcServer.start();
993   }
994 
995   void stop() {
996     closeAllScanners();
997     rpcServer.stop();
998   }
999 
1000   /**
1001    * Called to verify that this server is up and running.
1002    *
1003    * @throws IOException
1004    */
1005   protected void checkOpen() throws IOException {
1006     if (regionServer.isAborted()) {
1007       throw new RegionServerAbortedException("Server " + regionServer.serverName + " aborting");
1008     }
1009     if (regionServer.isStopped()) {
1010       throw new RegionServerStoppedException("Server " + regionServer.serverName + " stopping");
1011     }
1012     if (!regionServer.fsOk) {
1013       throw new RegionServerStoppedException("File system not available");
1014     }
1015     if (!regionServer.isOnline()) {
1016       throw new ServerNotRunningYetException("Server is not running yet");
1017     }
1018   }
1019 
1020   /**
1021    * @return list of blocking services and their security info classes that this server supports
1022    */
1023   protected List<BlockingServiceAndInterface> getServices() {
1024     List<BlockingServiceAndInterface> bssi = new ArrayList<BlockingServiceAndInterface>(2);
1025     bssi.add(new BlockingServiceAndInterface(
1026       ClientService.newReflectiveBlockingService(this),
1027       ClientService.BlockingInterface.class));
1028     bssi.add(new BlockingServiceAndInterface(
1029       AdminService.newReflectiveBlockingService(this),
1030       AdminService.BlockingInterface.class));
1031     return bssi;
1032   }
1033 
1034   public InetSocketAddress getSocketAddress() {
1035     return isa;
1036   }
1037 
1038   @Override
1039   public int getPriority(RequestHeader header, Message param) {
1040     return priority.getPriority(header, param);
1041   }
1042 
1043   @Override
1044   public long getDeadline(RequestHeader header, Message param) {
1045     return priority.getDeadline(header, param);
1046   }
1047 
1048   /*
1049    * Check if an OOME and, if so, abort immediately to avoid creating more objects.
1050    *
1051    * @param e
1052    *
1053    * @return True if we OOME'd and are aborting.
1054    */
1055   @Override
1056   public boolean checkOOME(final Throwable e) {
1057     boolean stop = false;
1058     try {
1059       if (e instanceof OutOfMemoryError
1060           || (e.getCause() != null && e.getCause() instanceof OutOfMemoryError)
1061           || (e.getMessage() != null && e.getMessage().contains(
1062               "java.lang.OutOfMemoryError"))) {
1063         stop = true;
1064         LOG.fatal("Run out of memory; " + getClass().getSimpleName()
1065           + " will abort itself immediately", e);
1066       }
1067     } finally {
1068       if (stop) {
1069         Runtime.getRuntime().halt(1);
1070       }
1071     }
1072     return stop;
1073   }
1074 
1075   /**
1076    * Close a region on the region server.
1077    *
1078    * @param controller the RPC controller
1079    * @param request the request
1080    * @throws ServiceException
1081    */
1082   @Override
1083   @QosPriority(priority=HConstants.ADMIN_QOS)
1084   public CloseRegionResponse closeRegion(final RpcController controller,
1085       final CloseRegionRequest request) throws ServiceException {
1086     final ServerName sn = (request.hasDestinationServer() ?
1087       ProtobufUtil.toServerName(request.getDestinationServer()) : null);
1088 
1089     try {
1090       checkOpen();
1091       if (request.hasServerStartCode()) {
1092         // check that we are the same server that this RPC is intended for.
1093         long serverStartCode = request.getServerStartCode();
1094         if (regionServer.serverName.getStartcode() !=  serverStartCode) {
1095           throw new ServiceException(new DoNotRetryIOException("This RPC was intended for a " +
1096               "different server with startCode: " + serverStartCode + ", this server is: "
1097               + regionServer.serverName));
1098         }
1099       }
1100       final String encodedRegionName = ProtobufUtil.getRegionEncodedName(request.getRegion());
1101 
1102       // Can be null if we're calling close on a region that's not online
1103       final Region region = regionServer.getFromOnlineRegions(encodedRegionName);
1104       if ((region  != null) && (region .getCoprocessorHost() != null)) {
1105         region.getCoprocessorHost().preClose(false);
1106       }
1107 
1108       requestCount.increment();
1109       LOG.info("Close " + encodedRegionName + ", moving to " + sn);
1110       CloseRegionCoordination.CloseRegionDetails crd = regionServer.getCoordinatedStateManager()
1111         .getCloseRegionCoordination().parseFromProtoRequest(request);
1112 
1113       boolean closed = regionServer.closeRegion(encodedRegionName, false, crd, sn);
1114       CloseRegionResponse.Builder builder = CloseRegionResponse.newBuilder().setClosed(closed);
1115       return builder.build();
1116     } catch (IOException ie) {
1117       throw new ServiceException(ie);
1118     }
1119   }
1120 
1121   /**
1122    * Compact a region on the region server.
1123    *
1124    * @param controller the RPC controller
1125    * @param request the request
1126    * @throws ServiceException
1127    */
1128   @Override
1129   @QosPriority(priority=HConstants.ADMIN_QOS)
1130   public CompactRegionResponse compactRegion(final RpcController controller,
1131       final CompactRegionRequest request) throws ServiceException {
1132     try {
1133       checkOpen();
1134       requestCount.increment();
1135       Region region = getRegion(request.getRegion());
1136       region.startRegionOperation(Operation.COMPACT_REGION);
1137       LOG.info("Compacting " + region.getRegionInfo().getRegionNameAsString());
1138       boolean major = false;
1139       byte [] family = null;
1140       Store store = null;
1141       if (request.hasFamily()) {
1142         family = request.getFamily().toByteArray();
1143         store = region.getStore(family);
1144         if (store == null) {
1145           throw new ServiceException(new IOException("column family " + Bytes.toString(family)
1146             + " does not exist in region " + region.getRegionInfo().getRegionNameAsString()));
1147         }
1148       }
1149       if (request.hasMajor()) {
1150         major = request.getMajor();
1151       }
1152       if (major) {
1153         if (family != null) {
1154           store.triggerMajorCompaction();
1155         } else {
1156           region.triggerMajorCompaction();
1157         }
1158       }
1159 
1160       String familyLogMsg = (family != null)?" for column family: " + Bytes.toString(family):"";
1161       if (LOG.isTraceEnabled()) {
1162         LOG.trace("User-triggered compaction requested for region "
1163           + region.getRegionInfo().getRegionNameAsString() + familyLogMsg);
1164       }
1165       String log = "User-triggered " + (major ? "major " : "") + "compaction" + familyLogMsg;
1166       if(family != null) {
1167         regionServer.compactSplitThread.requestCompaction(region, store, log,
1168           Store.PRIORITY_USER, null, RpcServer.getRequestUser());
1169       } else {
1170         regionServer.compactSplitThread.requestCompaction(region, log,
1171           Store.PRIORITY_USER, null, RpcServer.getRequestUser());
1172       }
1173       return CompactRegionResponse.newBuilder().build();
1174     } catch (IOException ie) {
1175       throw new ServiceException(ie);
1176     }
1177   }
1178 
1179   /**
1180    * Flush a region on the region server.
1181    *
1182    * @param controller the RPC controller
1183    * @param request the request
1184    * @throws ServiceException
1185    */
1186   @Override
1187   @QosPriority(priority=HConstants.ADMIN_QOS)
1188   public FlushRegionResponse flushRegion(final RpcController controller,
1189       final FlushRegionRequest request) throws ServiceException {
1190     try {
1191       checkOpen();
1192       requestCount.increment();
1193       Region region = getRegion(request.getRegion());
1194       LOG.info("Flushing " + region.getRegionInfo().getRegionNameAsString());
1195       boolean shouldFlush = true;
1196       if (request.hasIfOlderThanTs()) {
1197         shouldFlush = region.getEarliestFlushTimeForAllStores() < request.getIfOlderThanTs();
1198       }
1199       FlushRegionResponse.Builder builder = FlushRegionResponse.newBuilder();
1200       if (shouldFlush) {
1201         boolean writeFlushWalMarker =  request.hasWriteFlushWalMarker() ?
1202             request.getWriteFlushWalMarker() : false;
1203         // Go behind the curtain so we can manage writing of the flush WAL marker
1204         HRegion.FlushResultImpl flushResult = (HRegion.FlushResultImpl)
1205             ((HRegion)region).flushcache(true, writeFlushWalMarker);
1206         boolean compactionNeeded = flushResult.isCompactionNeeded();
1207         if (compactionNeeded) {
1208           regionServer.compactSplitThread.requestSystemCompaction(region,
1209             "Compaction through user triggered flush");
1210         }
1211         builder.setFlushed(flushResult.isFlushSucceeded());
1212         builder.setWroteFlushWalMarker(flushResult.wroteFlushWalMarker);
1213       }
1214       builder.setLastFlushTime(region.getEarliestFlushTimeForAllStores());
1215       return builder.build();
1216     } catch (DroppedSnapshotException ex) {
1217       // Cache flush can fail in a few places. If it fails in a critical
1218       // section, we get a DroppedSnapshotException and a replay of wal
1219       // is required. Currently the only way to do this is a restart of
1220       // the server.
1221       regionServer.abort("Replay of WAL required. Forcing server shutdown", ex);
1222       throw new ServiceException(ex);
1223     } catch (IOException ie) {
1224       throw new ServiceException(ie);
1225     }
1226   }
1227 
1228   @Override
1229   @QosPriority(priority=HConstants.ADMIN_QOS)
1230   public GetOnlineRegionResponse getOnlineRegion(final RpcController controller,
1231       final GetOnlineRegionRequest request) throws ServiceException {
1232     try {
1233       checkOpen();
1234       requestCount.increment();
1235       Map<String, Region> onlineRegions = regionServer.onlineRegions;
1236       List<HRegionInfo> list = new ArrayList<HRegionInfo>(onlineRegions.size());
1237       for (Region region: onlineRegions.values()) {
1238         list.add(region.getRegionInfo());
1239       }
1240       Collections.sort(list);
1241       return ResponseConverter.buildGetOnlineRegionResponse(list);
1242     } catch (IOException ie) {
1243       throw new ServiceException(ie);
1244     }
1245   }
1246 
1247   @Override
1248   @QosPriority(priority=HConstants.ADMIN_QOS)
1249   public GetRegionInfoResponse getRegionInfo(final RpcController controller,
1250       final GetRegionInfoRequest request) throws ServiceException {
1251     try {
1252       checkOpen();
1253       requestCount.increment();
1254       Region region = getRegion(request.getRegion());
1255       HRegionInfo info = region.getRegionInfo();
1256       GetRegionInfoResponse.Builder builder = GetRegionInfoResponse.newBuilder();
1257       builder.setRegionInfo(HRegionInfo.convert(info));
1258       if (request.hasCompactionState() && request.getCompactionState()) {
1259         builder.setCompactionState(region.getCompactionState());
1260       }
1261       builder.setIsRecovering(region.isRecovering());
1262       return builder.build();
1263     } catch (IOException ie) {
1264       throw new ServiceException(ie);
1265     }
1266   }
1267 
1268   /**
1269    * Get some information of the region server.
1270    *
1271    * @param controller the RPC controller
1272    * @param request the request
1273    * @throws ServiceException
1274    */
1275   @Override
1276   @QosPriority(priority=HConstants.ADMIN_QOS)
1277   public GetServerInfoResponse getServerInfo(final RpcController controller,
1278       final GetServerInfoRequest request) throws ServiceException {
1279     try {
1280       checkOpen();
1281     } catch (IOException ie) {
1282       throw new ServiceException(ie);
1283     }
1284     requestCount.increment();
1285     int infoPort = regionServer.infoServer != null ? regionServer.infoServer.getPort() : -1;
1286     return ResponseConverter.buildGetServerInfoResponse(regionServer.serverName, infoPort);
1287   }
1288 
1289   @Override
1290   @QosPriority(priority=HConstants.ADMIN_QOS)
1291   public GetStoreFileResponse getStoreFile(final RpcController controller,
1292       final GetStoreFileRequest request) throws ServiceException {
1293     try {
1294       checkOpen();
1295       Region region = getRegion(request.getRegion());
1296       requestCount.increment();
1297       Set<byte[]> columnFamilies;
1298       if (request.getFamilyCount() == 0) {
1299         columnFamilies = region.getTableDesc().getFamiliesKeys();
1300       } else {
1301         columnFamilies = new TreeSet<byte[]>(Bytes.BYTES_RAWCOMPARATOR);
1302         for (ByteString cf: request.getFamilyList()) {
1303           columnFamilies.add(cf.toByteArray());
1304         }
1305       }
1306       int nCF = columnFamilies.size();
1307       List<String>  fileList = region.getStoreFileList(
1308         columnFamilies.toArray(new byte[nCF][]));
1309       GetStoreFileResponse.Builder builder = GetStoreFileResponse.newBuilder();
1310       builder.addAllStoreFile(fileList);
1311       return builder.build();
1312     } catch (IOException ie) {
1313       throw new ServiceException(ie);
1314     }
1315   }
1316 
1317   /**
1318    * Merge regions on the region server.
1319    *
1320    * @param controller the RPC controller
1321    * @param request the request
1322    * @return merge regions response
1323    * @throws ServiceException
1324    */
1325   @Override
1326   @QosPriority(priority = HConstants.ADMIN_QOS)
1327   public MergeRegionsResponse mergeRegions(final RpcController controller,
1328       final MergeRegionsRequest request) throws ServiceException {
1329     try {
1330       checkOpen();
1331       requestCount.increment();
1332       Region regionA = getRegion(request.getRegionA());
1333       Region regionB = getRegion(request.getRegionB());
1334       boolean forcible = request.getForcible();
1335       long masterSystemTime = request.hasMasterSystemTime() ? request.getMasterSystemTime() : -1;
1336       regionA.startRegionOperation(Operation.MERGE_REGION);
1337       regionB.startRegionOperation(Operation.MERGE_REGION);
1338       if (regionA.getRegionInfo().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID ||
1339           regionB.getRegionInfo().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1340         throw new ServiceException(new MergeRegionException("Can't merge non-default replicas"));
1341       }
1342       LOG.info("Receiving merging request for  " + regionA + ", " + regionB
1343           + ",forcible=" + forcible);
1344       regionA.flush(true);
1345       regionB.flush(true);
1346       regionServer.compactSplitThread.requestRegionsMerge(regionA, regionB, forcible,
1347           masterSystemTime, RpcServer.getRequestUser());
1348       return MergeRegionsResponse.newBuilder().build();
1349     } catch (DroppedSnapshotException ex) {
1350       regionServer.abort("Replay of WAL required. Forcing server shutdown", ex);
1351       throw new ServiceException(ex);
1352     } catch (IOException ie) {
1353       throw new ServiceException(ie);
1354     }
1355   }
1356 
1357   /**
1358    * Open asynchronously a region or a set of regions on the region server.
1359    *
1360    * The opening is coordinated by ZooKeeper, and this method requires the znode to be created
1361    *  before being called. As a consequence, this method should be called only from the master.
1362    * <p>
1363    * Different manages states for the region are:<ul>
1364    *  <li>region not opened: the region opening will start asynchronously.</li>
1365    *  <li>a close is already in progress: this is considered as an error.</li>
1366    *  <li>an open is already in progress: this new open request will be ignored. This is important
1367    *  because the Master can do multiple requests if it crashes.</li>
1368    *  <li>the region is already opened:  this new open request will be ignored./li>
1369    *  </ul>
1370    * </p>
1371    * <p>
1372    * Bulk assign: If there are more than 1 region to open, it will be considered as a bulk assign.
1373    * For a single region opening, errors are sent through a ServiceException. For bulk assign,
1374    * errors are put in the response as FAILED_OPENING.
1375    * </p>
1376    * @param controller the RPC controller
1377    * @param request the request
1378    * @throws ServiceException
1379    */
1380   @Override
1381   @QosPriority(priority=HConstants.ADMIN_QOS)
1382   public OpenRegionResponse openRegion(final RpcController controller,
1383       final OpenRegionRequest request) throws ServiceException {
1384     requestCount.increment();
1385     if (request.hasServerStartCode()) {
1386       // check that we are the same server that this RPC is intended for.
1387       long serverStartCode = request.getServerStartCode();
1388       if (regionServer.serverName.getStartcode() !=  serverStartCode) {
1389         throw new ServiceException(new DoNotRetryIOException("This RPC was intended for a " +
1390             "different server with startCode: " + serverStartCode + ", this server is: "
1391             + regionServer.serverName));
1392       }
1393     }
1394 
1395     OpenRegionResponse.Builder builder = OpenRegionResponse.newBuilder();
1396     final int regionCount = request.getOpenInfoCount();
1397     final Map<TableName, HTableDescriptor> htds =
1398         new HashMap<TableName, HTableDescriptor>(regionCount);
1399     final boolean isBulkAssign = regionCount > 1;
1400     try {
1401       checkOpen();
1402     } catch (IOException ie) {
1403       TableName tableName = null;
1404       if (regionCount == 1) {
1405         RegionInfo ri = request.getOpenInfo(0).getRegion();
1406         if (ri != null) {
1407           tableName = ProtobufUtil.toTableName(ri.getTableName());
1408         }
1409       }
1410       if (!TableName.META_TABLE_NAME.equals(tableName)) {
1411         throw new ServiceException(ie);
1412       }
1413       // We are assigning meta, wait a little for regionserver to finish initialization.
1414       int timeout = regionServer.conf.getInt(HConstants.HBASE_RPC_TIMEOUT_KEY,
1415         HConstants.DEFAULT_HBASE_RPC_TIMEOUT) >> 2; // Quarter of RPC timeout
1416       long endTime = System.currentTimeMillis() + timeout;
1417       synchronized (regionServer.online) {
1418         try {
1419           while (System.currentTimeMillis() <= endTime
1420               && !regionServer.isStopped() && !regionServer.isOnline()) {
1421             regionServer.online.wait(regionServer.msgInterval);
1422           }
1423           checkOpen();
1424         } catch (InterruptedException t) {
1425           Thread.currentThread().interrupt();
1426           throw new ServiceException(t);
1427         } catch (IOException e) {
1428           throw new ServiceException(e);
1429         }
1430       }
1431     }
1432 
1433     long masterSystemTime = request.hasMasterSystemTime() ? request.getMasterSystemTime() : -1;
1434 
1435     for (RegionOpenInfo regionOpenInfo : request.getOpenInfoList()) {
1436       final HRegionInfo region = HRegionInfo.convert(regionOpenInfo.getRegion());
1437       OpenRegionCoordination coordination = regionServer.getCoordinatedStateManager().
1438         getOpenRegionCoordination();
1439       OpenRegionCoordination.OpenRegionDetails ord =
1440         coordination.parseFromProtoRequest(regionOpenInfo);
1441 
1442       HTableDescriptor htd;
1443       try {
1444         final Region onlineRegion = regionServer.getFromOnlineRegions(region.getEncodedName());
1445         if (onlineRegion != null) {
1446           //Check if the region can actually be opened.
1447           if (onlineRegion.getCoprocessorHost() != null) {
1448             onlineRegion.getCoprocessorHost().preOpen();
1449           }
1450           // See HBASE-5094. Cross check with hbase:meta if still this RS is owning
1451           // the region.
1452           Pair<HRegionInfo, ServerName> p = MetaTableAccessor.getRegion(
1453             regionServer.getConnection(), region.getRegionName());
1454           if (regionServer.serverName.equals(p.getSecond())) {
1455             Boolean closing = regionServer.regionsInTransitionInRS.get(region.getEncodedNameAsBytes());
1456             // Map regionsInTransitionInRSOnly has an entry for a region only if the region
1457             // is in transition on this RS, so here closing can be null. If not null, it can
1458             // be true or false. True means the region is opening on this RS; while false
1459             // means the region is closing. Only return ALREADY_OPENED if not closing (i.e.
1460             // not in transition any more, or still transition to open.
1461             if (!Boolean.FALSE.equals(closing)
1462                 && regionServer.getFromOnlineRegions(region.getEncodedName()) != null) {
1463               LOG.warn("Attempted open of " + region.getEncodedName()
1464                 + " but already online on this server");
1465               builder.addOpeningState(RegionOpeningState.ALREADY_OPENED);
1466               continue;
1467             }
1468           } else {
1469             LOG.warn("The region " + region.getEncodedName() + " is online on this server"
1470               + " but hbase:meta does not have this server - continue opening.");
1471             regionServer.removeFromOnlineRegions(onlineRegion, null);
1472           }
1473         }
1474         LOG.info("Open " + region.getRegionNameAsString());
1475         htd = htds.get(region.getTable());
1476         if (htd == null) {
1477           htd = regionServer.tableDescriptors.get(region.getTable());
1478           htds.put(region.getTable(), htd);
1479         }
1480 
1481         final Boolean previous = regionServer.regionsInTransitionInRS.putIfAbsent(
1482           region.getEncodedNameAsBytes(), Boolean.TRUE);
1483 
1484         if (Boolean.FALSE.equals(previous)) {
1485           // There is a close in progress. We need to mark this open as failed in ZK.
1486 
1487           coordination.tryTransitionFromOfflineToFailedOpen(regionServer, region, ord);
1488 
1489           throw new RegionAlreadyInTransitionException("Received OPEN for the region:"
1490             + region.getRegionNameAsString() + " , which we are already trying to CLOSE ");
1491         }
1492 
1493         if (Boolean.TRUE.equals(previous)) {
1494           // An open is in progress. This is supported, but let's log this.
1495           LOG.info("Receiving OPEN for the region:" +
1496             region.getRegionNameAsString() + " , which we are already trying to OPEN"
1497               + " - ignoring this new request for this region.");
1498         }
1499 
1500         // We are opening this region. If it moves back and forth for whatever reason, we don't
1501         // want to keep returning the stale moved record while we are opening/if we close again.
1502         regionServer.removeFromMovedRegions(region.getEncodedName());
1503 
1504         if (previous == null) {
1505           // check if the region to be opened is marked in recovering state in ZK
1506           if (ZKSplitLog.isRegionMarkedRecoveringInZK(regionServer.getZooKeeper(),
1507               region.getEncodedName())) {
1508             // Check if current region open is for distributedLogReplay. This check is to support
1509             // rolling restart/upgrade where we want to Master/RS see same configuration
1510             if (!regionOpenInfo.hasOpenForDistributedLogReplay()
1511                   || regionOpenInfo.getOpenForDistributedLogReplay()) {
1512               regionServer.recoveringRegions.put(region.getEncodedName(), null);
1513             } else {
1514               // Remove stale recovery region from ZK when we open region not for recovering which
1515               // could happen when turn distributedLogReplay off from on.
1516               List<String> tmpRegions = new ArrayList<String>();
1517               tmpRegions.add(region.getEncodedName());
1518               ZKSplitLog.deleteRecoveringRegionZNodes(regionServer.getZooKeeper(),
1519                 tmpRegions);
1520             }
1521           }
1522           if (htd == null) {
1523             throw new IOException("Missing table descriptor for " + region.getEncodedName());
1524           }
1525           // If there is no action in progress, we can submit a specific handler.
1526           // Need to pass the expected version in the constructor.
1527           if (region.isMetaRegion()) {
1528             regionServer.service.submit(new OpenMetaHandler(
1529               regionServer, regionServer, region, htd, masterSystemTime, coordination, ord));
1530           } else {
1531             regionServer.updateRegionFavoredNodesMapping(region.getEncodedName(),
1532               regionOpenInfo.getFavoredNodesList());
1533             if (htd.getPriority() >= HConstants.ADMIN_QOS || region.getTable().isSystemTable()) {
1534               regionServer.service.submit(new OpenPriorityRegionHandler(
1535                 regionServer, regionServer, region, htd, masterSystemTime, coordination, ord));
1536             } else {
1537               regionServer.service.submit(new OpenRegionHandler(
1538                 regionServer, regionServer, region, htd, masterSystemTime, coordination, ord));
1539             }
1540           }
1541         }
1542 
1543         builder.addOpeningState(RegionOpeningState.OPENED);
1544 
1545       } catch (KeeperException zooKeeperEx) {
1546         LOG.error("Can't retrieve recovering state from zookeeper", zooKeeperEx);
1547         throw new ServiceException(zooKeeperEx);
1548       } catch (IOException ie) {
1549         LOG.warn("Failed opening region " + region.getRegionNameAsString(), ie);
1550         if (isBulkAssign) {
1551           builder.addOpeningState(RegionOpeningState.FAILED_OPENING);
1552         } else {
1553           throw new ServiceException(ie);
1554         }
1555       }
1556     }
1557     return builder.build();
1558   }
1559 
1560   /**
1561    *  Wamrmup a region on this server.
1562    *
1563    * This method should only be called by Master. It synchrnously opens the region and
1564    * closes the region bringing the most important pages in cache.
1565    * <p>
1566    *
1567    * @param controller the RPC controller
1568    * @param request the request
1569    * @throws ServiceException
1570    */
1571   @Override
1572   public WarmupRegionResponse warmupRegion(final RpcController controller,
1573       final WarmupRegionRequest request) throws ServiceException {
1574 
1575     RegionInfo regionInfo = request.getRegionInfo();
1576     final HRegionInfo region = HRegionInfo.convert(regionInfo);
1577     HTableDescriptor htd;
1578     WarmupRegionResponse response = WarmupRegionResponse.getDefaultInstance();
1579 
1580     try {
1581       checkOpen();
1582       String encodedName = region.getEncodedName();
1583       byte[] encodedNameBytes = region.getEncodedNameAsBytes();
1584       final Region onlineRegion = regionServer.getFromOnlineRegions(encodedName);
1585 
1586       if (onlineRegion != null) {
1587         LOG.info("Region already online. Skipping warming up " + region);
1588         return response;
1589       }
1590 
1591       if (LOG.isDebugEnabled()) {
1592         LOG.debug("Warming up Region " + region.getRegionNameAsString());
1593       }
1594 
1595       htd = regionServer.tableDescriptors.get(region.getTable());
1596 
1597       if (regionServer.getRegionsInTransitionInRS().containsKey(encodedNameBytes)) {
1598         LOG.info("Region is in transition. Skipping warmup " + region);
1599         return response;
1600       }
1601 
1602       HRegion.warmupHRegion(region, htd, regionServer.getWAL(region),
1603           regionServer.getConfiguration(), regionServer, null);
1604 
1605     } catch (IOException ie) {
1606       LOG.error("Failed warming up region " + region.getRegionNameAsString(), ie);
1607       throw new ServiceException(ie);
1608     }
1609 
1610     return response;
1611   }
1612 
1613   /**
1614    * Replay the given changes when distributedLogReplay WAL edits from a failed RS. The guarantee is
1615    * that the given mutations will be durable on the receiving RS if this method returns without any
1616    * exception.
1617    * @param controller the RPC controller
1618    * @param request the request
1619    * @throws ServiceException
1620    */
1621   @Override
1622   @QosPriority(priority = HConstants.REPLAY_QOS)
1623   public ReplicateWALEntryResponse replay(final RpcController controller,
1624       final ReplicateWALEntryRequest request) throws ServiceException {
1625     long before = EnvironmentEdgeManager.currentTime();
1626     CellScanner cells = ((PayloadCarryingRpcController) controller).cellScanner();
1627     try {
1628       checkOpen();
1629       List<WALEntry> entries = request.getEntryList();
1630       if (entries == null || entries.isEmpty()) {
1631         // empty input
1632         return ReplicateWALEntryResponse.newBuilder().build();
1633       }
1634       ByteString regionName = entries.get(0).getKey().getEncodedRegionName();
1635       Region region = regionServer.getRegionByEncodedName(regionName.toStringUtf8());
1636       RegionCoprocessorHost coprocessorHost =
1637           ServerRegionReplicaUtil.isDefaultReplica(region.getRegionInfo())
1638             ? region.getCoprocessorHost()
1639             : null; // do not invoke coprocessors if this is a secondary region replica
1640       List<Pair<WALKey, WALEdit>> walEntries = new ArrayList<Pair<WALKey, WALEdit>>();
1641 
1642       // Skip adding the edits to WAL if this is a secondary region replica
1643       boolean isPrimary = RegionReplicaUtil.isDefaultReplica(region.getRegionInfo());
1644       Durability durability = isPrimary ? Durability.USE_DEFAULT : Durability.SKIP_WAL;
1645 
1646       for (WALEntry entry : entries) {
1647         if (!regionName.equals(entry.getKey().getEncodedRegionName())) {
1648           throw new NotServingRegionException("Replay request contains entries from multiple " +
1649               "regions. First region:" + regionName.toStringUtf8() + " , other region:"
1650               + entry.getKey().getEncodedRegionName());
1651         }
1652         if (regionServer.nonceManager != null && isPrimary) {
1653           long nonceGroup = entry.getKey().hasNonceGroup()
1654             ? entry.getKey().getNonceGroup() : HConstants.NO_NONCE;
1655           long nonce = entry.getKey().hasNonce() ? entry.getKey().getNonce() : HConstants.NO_NONCE;
1656           regionServer.nonceManager.reportOperationFromWal(nonceGroup, nonce, entry.getKey().getWriteTime());
1657         }
1658         Pair<WALKey, WALEdit> walEntry = (coprocessorHost == null) ? null :
1659           new Pair<WALKey, WALEdit>();
1660         List<WALSplitter.MutationReplay> edits = WALSplitter.getMutationsFromWALEntry(entry,
1661           cells, walEntry, durability);
1662         if (coprocessorHost != null) {
1663           // Start coprocessor replay here. The coprocessor is for each WALEdit instead of a
1664           // KeyValue.
1665           if (coprocessorHost.preWALRestore(region.getRegionInfo(), walEntry.getFirst(),
1666             walEntry.getSecond())) {
1667             // if bypass this log entry, ignore it ...
1668             continue;
1669           }
1670           walEntries.add(walEntry);
1671         }
1672         if(edits!=null && !edits.isEmpty()) {
1673           long replaySeqId = (entry.getKey().hasOrigSequenceNumber()) ?
1674             entry.getKey().getOrigSequenceNumber() : entry.getKey().getLogSequenceNumber();
1675           OperationStatus[] result = doReplayBatchOp(region, edits, replaySeqId);
1676           // check if it's a partial success
1677           for (int i = 0; result != null && i < result.length; i++) {
1678             if (result[i] != OperationStatus.SUCCESS) {
1679               throw new IOException(result[i].getExceptionMsg());
1680             }
1681           }
1682         }
1683       }
1684 
1685       //sync wal at the end because ASYNC_WAL is used above
1686       WAL wal = getWAL(region);
1687       if (wal != null) {
1688         wal.sync();
1689       }
1690 
1691       if (coprocessorHost != null) {
1692         for (Pair<WALKey, WALEdit> entry : walEntries) {
1693           coprocessorHost.postWALRestore(region.getRegionInfo(), entry.getFirst(),
1694             entry.getSecond());
1695         }
1696       }
1697       return ReplicateWALEntryResponse.newBuilder().build();
1698     } catch (IOException ie) {
1699       throw new ServiceException(ie);
1700     } finally {
1701       if (regionServer.metricsRegionServer != null) {
1702         regionServer.metricsRegionServer.updateReplay(
1703           EnvironmentEdgeManager.currentTime() - before);
1704       }
1705     }
1706   }
1707 
1708   WAL getWAL(Region region) {
1709     return ((HRegion)region).getWAL();
1710   }
1711 
1712   /**
1713    * Replicate WAL entries on the region server.
1714    *
1715    * @param controller the RPC controller
1716    * @param request the request
1717    * @throws ServiceException
1718    */
1719   @Override
1720   @QosPriority(priority=HConstants.REPLICATION_QOS)
1721   public ReplicateWALEntryResponse replicateWALEntry(final RpcController controller,
1722       final ReplicateWALEntryRequest request) throws ServiceException {
1723     try {
1724       checkOpen();
1725       if (regionServer.replicationSinkHandler != null) {
1726         requestCount.increment();
1727         List<WALEntry> entries = request.getEntryList();
1728         CellScanner cellScanner = ((PayloadCarryingRpcController)controller).cellScanner();
1729         regionServer.getRegionServerCoprocessorHost().preReplicateLogEntries(entries, cellScanner);
1730         regionServer.replicationSinkHandler.replicateLogEntries(entries, cellScanner,
1731           request.getReplicationClusterId(), request.getSourceBaseNamespaceDirPath(),
1732           request.getSourceHFileArchiveDirPath());
1733         regionServer.getRegionServerCoprocessorHost().postReplicateLogEntries(entries, cellScanner);
1734         return ReplicateWALEntryResponse.newBuilder().build();
1735       } else {
1736         throw new ServiceException("Replication services are not initialized yet");
1737       }
1738     } catch (IOException ie) {
1739       throw new ServiceException(ie);
1740     }
1741   }
1742 
1743   /**
1744    * Roll the WAL writer of the region server.
1745    * @param controller the RPC controller
1746    * @param request the request
1747    * @throws ServiceException
1748    */
1749   @Override
1750   public RollWALWriterResponse rollWALWriter(final RpcController controller,
1751       final RollWALWriterRequest request) throws ServiceException {
1752     try {
1753       checkOpen();
1754       requestCount.increment();
1755       regionServer.getRegionServerCoprocessorHost().preRollWALWriterRequest();
1756       regionServer.walRoller.requestRollAll();
1757       regionServer.getRegionServerCoprocessorHost().postRollWALWriterRequest();
1758       RollWALWriterResponse.Builder builder = RollWALWriterResponse.newBuilder();
1759       return builder.build();
1760     } catch (IOException ie) {
1761       throw new ServiceException(ie);
1762     }
1763   }
1764 
1765   /**
1766    * Split a region on the region server.
1767    *
1768    * @param controller the RPC controller
1769    * @param request the request
1770    * @throws ServiceException
1771    */
1772   @Override
1773   @QosPriority(priority=HConstants.ADMIN_QOS)
1774   public SplitRegionResponse splitRegion(final RpcController controller,
1775       final SplitRegionRequest request) throws ServiceException {
1776     try {
1777       checkOpen();
1778       requestCount.increment();
1779       Region region = getRegion(request.getRegion());
1780       region.startRegionOperation(Operation.SPLIT_REGION);
1781       if (region.getRegionInfo().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1782         throw new IOException("Can't split replicas directly. "
1783             + "Replicas are auto-split when their primary is split.");
1784       }
1785       LOG.info("Splitting " + region.getRegionInfo().getRegionNameAsString());
1786       region.flush(true);
1787       byte[] splitPoint = null;
1788       if (request.hasSplitPoint()) {
1789         splitPoint = request.getSplitPoint().toByteArray();
1790       }
1791       ((HRegion)region).forceSplit(splitPoint);
1792       regionServer.compactSplitThread.requestSplit(region, ((HRegion)region).checkSplit(),
1793         RpcServer.getRequestUser());
1794       return SplitRegionResponse.newBuilder().build();
1795     } catch (DroppedSnapshotException ex) {
1796       regionServer.abort("Replay of WAL required. Forcing server shutdown", ex);
1797       throw new ServiceException(ex);
1798     } catch (IOException ie) {
1799       throw new ServiceException(ie);
1800     }
1801   }
1802 
1803   /**
1804    * Stop the region server.
1805    *
1806    * @param controller the RPC controller
1807    * @param request the request
1808    * @throws ServiceException
1809    */
1810   @Override
1811   @QosPriority(priority=HConstants.ADMIN_QOS)
1812   public StopServerResponse stopServer(final RpcController controller,
1813       final StopServerRequest request) throws ServiceException {
1814     requestCount.increment();
1815     String reason = request.getReason();
1816     regionServer.stop(reason);
1817     return StopServerResponse.newBuilder().build();
1818   }
1819 
1820   @Override
1821   public UpdateFavoredNodesResponse updateFavoredNodes(RpcController controller,
1822       UpdateFavoredNodesRequest request) throws ServiceException {
1823     List<UpdateFavoredNodesRequest.RegionUpdateInfo> openInfoList = request.getUpdateInfoList();
1824     UpdateFavoredNodesResponse.Builder respBuilder = UpdateFavoredNodesResponse.newBuilder();
1825     for (UpdateFavoredNodesRequest.RegionUpdateInfo regionUpdateInfo : openInfoList) {
1826       HRegionInfo hri = HRegionInfo.convert(regionUpdateInfo.getRegion());
1827       regionServer.updateRegionFavoredNodesMapping(hri.getEncodedName(),
1828         regionUpdateInfo.getFavoredNodesList());
1829     }
1830     respBuilder.setResponse(openInfoList.size());
1831     return respBuilder.build();
1832   }
1833 
1834   /**
1835    * Atomically bulk load several HFiles into an open region
1836    * @return true if successful, false is failed but recoverably (no action)
1837    * @throws IOException if failed unrecoverably
1838    */
1839   @Override
1840   public BulkLoadHFileResponse bulkLoadHFile(final RpcController controller,
1841       final BulkLoadHFileRequest request) throws ServiceException {
1842     try {
1843       checkOpen();
1844       requestCount.increment();
1845       Region region = getRegion(request.getRegion());
1846       List<Pair<byte[], String>> familyPaths = new ArrayList<Pair<byte[], String>>();
1847       for (FamilyPath familyPath: request.getFamilyPathList()) {
1848         familyPaths.add(new Pair<byte[], String>(familyPath.getFamily().toByteArray(),
1849           familyPath.getPath()));
1850       }
1851       boolean bypass = false;
1852       if (region.getCoprocessorHost() != null) {
1853         bypass = region.getCoprocessorHost().preBulkLoadHFile(familyPaths);
1854       }
1855       boolean loaded = false;
1856       if (!bypass) {
1857         loaded = region.bulkLoadHFiles(familyPaths, request.getAssignSeqNum(), null);
1858       }
1859       if (region.getCoprocessorHost() != null) {
1860         loaded = region.getCoprocessorHost().postBulkLoadHFile(familyPaths, loaded);
1861       }
1862       BulkLoadHFileResponse.Builder builder = BulkLoadHFileResponse.newBuilder();
1863       builder.setLoaded(loaded);
1864       return builder.build();
1865     } catch (IOException ie) {
1866       throw new ServiceException(ie);
1867     }
1868   }
1869 
1870   @Override
1871   public CoprocessorServiceResponse execService(final RpcController controller,
1872       final CoprocessorServiceRequest request) throws ServiceException {
1873     try {
1874       checkOpen();
1875       requestCount.increment();
1876       Region region = getRegion(request.getRegion());
1877       Message result = execServiceOnRegion(region, request.getCall());
1878       CoprocessorServiceResponse.Builder builder =
1879         CoprocessorServiceResponse.newBuilder();
1880       builder.setRegion(RequestConverter.buildRegionSpecifier(
1881         RegionSpecifierType.REGION_NAME, region.getRegionInfo().getRegionName()));
1882       builder.setValue(
1883         builder.getValueBuilder().setName(result.getClass().getName())
1884           .setValue(result.toByteString()));
1885       return builder.build();
1886     } catch (IOException ie) {
1887       throw new ServiceException(ie);
1888     }
1889   }
1890 
1891   private Message execServiceOnRegion(Region region,
1892       final ClientProtos.CoprocessorServiceCall serviceCall) throws IOException {
1893     // ignore the passed in controller (from the serialized call)
1894     ServerRpcController execController = new ServerRpcController();
1895     Message result = region.execService(execController, serviceCall);
1896     if (execController.getFailedOn() != null) {
1897       throw execController.getFailedOn();
1898     }
1899     return result;
1900   }
1901 
1902   /**
1903    * Get data from a table.
1904    *
1905    * @param controller the RPC controller
1906    * @param request the get request
1907    * @throws ServiceException
1908    */
1909   @Override
1910   public GetResponse get(final RpcController controller,
1911       final GetRequest request) throws ServiceException {
1912     long before = EnvironmentEdgeManager.currentTime();
1913     OperationQuota quota = null;
1914     try {
1915       checkOpen();
1916       requestCount.increment();
1917       rpcGetRequestCount.increment();
1918       Region region = getRegion(request.getRegion());
1919 
1920       GetResponse.Builder builder = GetResponse.newBuilder();
1921       ClientProtos.Get get = request.getGet();
1922       Boolean existence = null;
1923       Result r = null;
1924       quota = getQuotaManager().checkQuota(region, OperationQuota.OperationType.GET);
1925 
1926       if (get.hasClosestRowBefore() && get.getClosestRowBefore()) {
1927         if (get.getColumnCount() != 1) {
1928           throw new DoNotRetryIOException(
1929             "get ClosestRowBefore supports one and only one family now, not "
1930               + get.getColumnCount() + " families");
1931         }
1932         byte[] row = get.getRow().toByteArray();
1933         byte[] family = get.getColumn(0).getFamily().toByteArray();
1934         r = region.getClosestRowBefore(row, family);
1935       } else {
1936         Get clientGet = ProtobufUtil.toGet(get);
1937         if (get.getExistenceOnly() && region.getCoprocessorHost() != null) {
1938           existence = region.getCoprocessorHost().preExists(clientGet);
1939         }
1940         if (existence == null) {
1941           r = region.get(clientGet);
1942           if (get.getExistenceOnly()) {
1943             boolean exists = r.getExists();
1944             if (region.getCoprocessorHost() != null) {
1945               exists = region.getCoprocessorHost().postExists(clientGet, exists);
1946             }
1947             existence = exists;
1948           }
1949         }
1950       }
1951       if (existence != null){
1952         ClientProtos.Result pbr =
1953             ProtobufUtil.toResult(existence, region.getRegionInfo().getReplicaId() != 0);
1954         builder.setResult(pbr);
1955       } else  if (r != null) {
1956         ClientProtos.Result pbr = ProtobufUtil.toResult(r);
1957         builder.setResult(pbr);
1958       }
1959       if (r != null) {
1960         quota.addGetResult(r);
1961       }
1962       return builder.build();
1963     } catch (IOException ie) {
1964       throw new ServiceException(ie);
1965     } finally {
1966       if (regionServer.metricsRegionServer != null) {
1967         regionServer.metricsRegionServer.updateGet(
1968           EnvironmentEdgeManager.currentTime() - before);
1969       }
1970       if (quota != null) {
1971         quota.close();
1972       }
1973     }
1974   }
1975 
1976   /**
1977    * Execute multiple actions on a table: get, mutate, and/or execCoprocessor
1978    *
1979    * @param rpcc the RPC controller
1980    * @param request the multi request
1981    * @throws ServiceException
1982    */
1983   @Override
1984   public MultiResponse multi(final RpcController rpcc, final MultiRequest request)
1985   throws ServiceException {
1986     try {
1987       checkOpen();
1988     } catch (IOException ie) {
1989       throw new ServiceException(ie);
1990     }
1991 
1992     // rpc controller is how we bring in data via the back door;  it is unprotobuf'ed data.
1993     // It is also the conduit via which we pass back data.
1994     PayloadCarryingRpcController controller = (PayloadCarryingRpcController)rpcc;
1995     CellScanner cellScanner = controller != null ? controller.cellScanner(): null;
1996     if (controller != null) controller.setCellScanner(null);
1997 
1998     long nonceGroup = request.hasNonceGroup() ? request.getNonceGroup() : HConstants.NO_NONCE;
1999 
2000     // this will contain all the cells that we need to return. It's created later, if needed.
2001     List<CellScannable> cellsToReturn = null;
2002     MultiResponse.Builder responseBuilder = MultiResponse.newBuilder();
2003     RegionActionResult.Builder regionActionResultBuilder = RegionActionResult.newBuilder();
2004     Boolean processed = null;
2005 
2006     RpcCallContext context = RpcServer.getCurrentCall();
2007     this.rpcMultiRequestCount.increment();
2008     for (RegionAction regionAction : request.getRegionActionList()) {
2009       this.requestCount.add(regionAction.getActionCount());
2010       OperationQuota quota;
2011       Region region;
2012       regionActionResultBuilder.clear();
2013       try {
2014         region = getRegion(regionAction.getRegion());
2015         quota = getQuotaManager().checkQuota(region, regionAction.getActionList());
2016       } catch (IOException e) {
2017         rpcServer.getMetrics().exception(e);
2018         regionActionResultBuilder.setException(ResponseConverter.buildException(e));
2019         responseBuilder.addRegionActionResult(regionActionResultBuilder.build());
2020         continue;  // For this region it's a failure.
2021       }
2022 
2023       if (regionAction.hasAtomic() && regionAction.getAtomic()) {
2024         // How does this call happen?  It may need some work to play well w/ the surroundings.
2025         // Need to return an item per Action along w/ Action index.  TODO.
2026         try {
2027           if (request.hasCondition()) {
2028             Condition condition = request.getCondition();
2029             byte[] row = condition.getRow().toByteArray();
2030             byte[] family = condition.getFamily().toByteArray();
2031             byte[] qualifier = condition.getQualifier().toByteArray();
2032             CompareOp compareOp = CompareOp.valueOf(condition.getCompareType().name());
2033             ByteArrayComparable comparator =
2034                 ProtobufUtil.toComparator(condition.getComparator());
2035             processed = checkAndRowMutate(region, regionAction.getActionList(),
2036                   cellScanner, row, family, qualifier, compareOp, comparator);
2037           } else {
2038             ClientProtos.RegionLoadStats stats = mutateRows(region, regionAction.getActionList(),
2039                 cellScanner);
2040             // add the stats to the request
2041             if(stats != null) {
2042               responseBuilder.addRegionActionResult(RegionActionResult.newBuilder()
2043                   .addResultOrException(ResultOrException.newBuilder().setLoadStats(stats)));
2044             }
2045             processed = Boolean.TRUE;
2046           }
2047         } catch (IOException e) {
2048           rpcServer.getMetrics().exception(e);
2049           // As it's atomic, we may expect it's a global failure.
2050           regionActionResultBuilder.setException(ResponseConverter.buildException(e));
2051         }
2052       } else {
2053         // doNonAtomicRegionMutation manages the exception internally
2054         cellsToReturn = doNonAtomicRegionMutation(region, quota, regionAction, cellScanner,
2055             regionActionResultBuilder, cellsToReturn, nonceGroup);
2056       }
2057       responseBuilder.addRegionActionResult(regionActionResultBuilder.build());
2058       quota.close();
2059     }
2060     // Load the controller with the Cells to return.
2061     if (cellsToReturn != null && !cellsToReturn.isEmpty() && controller != null) {
2062       controller.setCellScanner(CellUtil.createCellScanner(cellsToReturn));
2063     }
2064     if (processed != null) responseBuilder.setProcessed(processed);
2065     return responseBuilder.build();
2066   }
2067 
2068   /**
2069    * Mutate data in a table.
2070    *
2071    * @param rpcc the RPC controller
2072    * @param request the mutate request
2073    * @throws ServiceException
2074    */
2075   @Override
2076   public MutateResponse mutate(final RpcController rpcc,
2077       final MutateRequest request) throws ServiceException {
2078     // rpc controller is how we bring in data via the back door;  it is unprotobuf'ed data.
2079     // It is also the conduit via which we pass back data.
2080     PayloadCarryingRpcController controller = (PayloadCarryingRpcController)rpcc;
2081     CellScanner cellScanner = controller != null? controller.cellScanner(): null;
2082     OperationQuota quota = null;
2083     // Clear scanner so we are not holding on to reference across call.
2084     if (controller != null) controller.setCellScanner(null);
2085     try {
2086       checkOpen();
2087       requestCount.increment();
2088       rpcMutateRequestCount.increment();
2089       Region region = getRegion(request.getRegion());
2090       MutateResponse.Builder builder = MutateResponse.newBuilder();
2091       MutationProto mutation = request.getMutation();
2092       if (!region.getRegionInfo().isMetaTable()) {
2093         regionServer.cacheFlusher.reclaimMemStoreMemory();
2094       }
2095       long nonceGroup = request.hasNonceGroup() ? request.getNonceGroup() : HConstants.NO_NONCE;
2096       Result r = null;
2097       Boolean processed = null;
2098       MutationType type = mutation.getMutateType();
2099       long mutationSize = 0;
2100       quota = getQuotaManager().checkQuota(region, OperationQuota.OperationType.MUTATE);
2101       switch (type) {
2102       case APPEND:
2103         // TODO: this doesn't actually check anything.
2104         r = append(region, quota, mutation, cellScanner, nonceGroup);
2105         break;
2106       case INCREMENT:
2107         // TODO: this doesn't actually check anything.
2108         r = increment(region, quota, mutation, cellScanner, nonceGroup);
2109         break;
2110       case PUT:
2111         Put put = ProtobufUtil.toPut(mutation, cellScanner);
2112         quota.addMutation(put);
2113         if (request.hasCondition()) {
2114           Condition condition = request.getCondition();
2115           byte[] row = condition.getRow().toByteArray();
2116           byte[] family = condition.getFamily().toByteArray();
2117           byte[] qualifier = condition.getQualifier().toByteArray();
2118           CompareOp compareOp = CompareOp.valueOf(condition.getCompareType().name());
2119           ByteArrayComparable comparator =
2120             ProtobufUtil.toComparator(condition.getComparator());
2121           if (region.getCoprocessorHost() != null) {
2122             processed = region.getCoprocessorHost().preCheckAndPut(
2123               row, family, qualifier, compareOp, comparator, put);
2124           }
2125           if (processed == null) {
2126             boolean result = region.checkAndMutate(row, family,
2127               qualifier, compareOp, comparator, put, true);
2128             if (region.getCoprocessorHost() != null) {
2129               result = region.getCoprocessorHost().postCheckAndPut(row, family,
2130                 qualifier, compareOp, comparator, put, result);
2131             }
2132             processed = result;
2133           }
2134         } else {
2135           region.put(put);
2136           processed = Boolean.TRUE;
2137         }
2138         break;
2139       case DELETE:
2140         Delete delete = ProtobufUtil.toDelete(mutation, cellScanner);
2141         quota.addMutation(delete);
2142         if (request.hasCondition()) {
2143           Condition condition = request.getCondition();
2144           byte[] row = condition.getRow().toByteArray();
2145           byte[] family = condition.getFamily().toByteArray();
2146           byte[] qualifier = condition.getQualifier().toByteArray();
2147           CompareOp compareOp = CompareOp.valueOf(condition.getCompareType().name());
2148           ByteArrayComparable comparator =
2149             ProtobufUtil.toComparator(condition.getComparator());
2150           if (region.getCoprocessorHost() != null) {
2151             processed = region.getCoprocessorHost().preCheckAndDelete(
2152               row, family, qualifier, compareOp, comparator, delete);
2153           }
2154           if (processed == null) {
2155             boolean result = region.checkAndMutate(row, family,
2156               qualifier, compareOp, comparator, delete, true);
2157             if (region.getCoprocessorHost() != null) {
2158               result = region.getCoprocessorHost().postCheckAndDelete(row, family,
2159                 qualifier, compareOp, comparator, delete, result);
2160             }
2161             processed = result;
2162           }
2163         } else {
2164           region.delete(delete);
2165           processed = Boolean.TRUE;
2166         }
2167         break;
2168       default:
2169           throw new DoNotRetryIOException(
2170             "Unsupported mutate type: " + type.name());
2171       }
2172       if (processed != null) builder.setProcessed(processed.booleanValue());
2173       addResult(builder, r, controller);
2174       return builder.build();
2175     } catch (IOException ie) {
2176       regionServer.checkFileSystem();
2177       throw new ServiceException(ie);
2178     } finally {
2179       if (quota != null) {
2180         quota.close();
2181       }
2182     }
2183   }
2184 
2185   /**
2186    * Scan data in a table.
2187    *
2188    * @param controller the RPC controller
2189    * @param request the scan request
2190    * @throws ServiceException
2191    */
2192   @Override
2193   public ScanResponse scan(final RpcController controller, final ScanRequest request)
2194   throws ServiceException {
2195     OperationQuota quota = null;
2196     Leases.Lease lease = null;
2197     String scannerName = null;
2198     try {
2199       if (!request.hasScannerId() && !request.hasScan()) {
2200         throw new DoNotRetryIOException(
2201           "Missing required input: scannerId or scan");
2202       }
2203       long scannerId = -1;
2204       if (request.hasScannerId()) {
2205         scannerId = request.getScannerId();
2206         scannerName = String.valueOf(scannerId);
2207       }
2208       try {
2209         checkOpen();
2210       } catch (IOException e) {
2211         // If checkOpen failed, server not running or filesystem gone,
2212         // cancel this lease; filesystem is gone or we're closing or something.
2213         if (scannerName != null) {
2214           LOG.debug("Server shutting down and client tried to access missing scanner "
2215             + scannerName);
2216           if (regionServer.leases != null) {
2217             try {
2218               regionServer.leases.cancelLease(scannerName);
2219             } catch (LeaseException le) {
2220               // No problem, ignore
2221             }
2222           }
2223         }
2224         throw e;
2225       }
2226       requestCount.increment();
2227       rpcScanRequestCount.increment();
2228 
2229       int ttl = 0;
2230       Region region = null;
2231       RegionScanner scanner = null;
2232       RegionScannerHolder rsh = null;
2233       boolean moreResults = true;
2234       boolean closeScanner = false;
2235       boolean isSmallScan = false;
2236       ScanResponse.Builder builder = ScanResponse.newBuilder();
2237       if (request.hasCloseScanner()) {
2238         closeScanner = request.getCloseScanner();
2239       }
2240       int rows = closeScanner ? 0 : 1;
2241       if (request.hasNumberOfRows()) {
2242         rows = request.getNumberOfRows();
2243       }
2244       if (request.hasScannerId()) {
2245         rsh = scanners.get(scannerName);
2246         if (rsh == null) {
2247           LOG.info("Client tried to access missing scanner " + scannerName);
2248           throw new UnknownScannerException(
2249             "Name: " + scannerName + ", already closed?");
2250         }
2251         scanner = rsh.s;
2252         HRegionInfo hri = scanner.getRegionInfo();
2253         region = regionServer.getRegion(hri.getRegionName());
2254         if (region != rsh.r) { // Yes, should be the same instance
2255           throw new NotServingRegionException("Region was re-opened after the scanner"
2256             + scannerName + " was created: " + hri.getRegionNameAsString());
2257         }
2258       } else {
2259         region = getRegion(request.getRegion());
2260         ClientProtos.Scan protoScan = request.getScan();
2261         boolean isLoadingCfsOnDemandSet = protoScan.hasLoadColumnFamiliesOnDemand();
2262         Scan scan = ProtobufUtil.toScan(protoScan);
2263         // if the request doesn't set this, get the default region setting.
2264         if (!isLoadingCfsOnDemandSet) {
2265           scan.setLoadColumnFamiliesOnDemand(region.isLoadingCfsOnDemandDefault());
2266         }
2267 
2268         isSmallScan = scan.isSmall();
2269         if (!scan.hasFamilies()) {
2270           // Adding all families to scanner
2271           for (byte[] family: region.getTableDesc().getFamiliesKeys()) {
2272             scan.addFamily(family);
2273           }
2274         }
2275 
2276         if (region.getCoprocessorHost() != null) {
2277           scanner = region.getCoprocessorHost().preScannerOpen(scan);
2278         }
2279         if (scanner == null) {
2280           scanner = region.getScanner(scan);
2281         }
2282         if (region.getCoprocessorHost() != null) {
2283           scanner = region.getCoprocessorHost().postScannerOpen(scan, scanner);
2284         }
2285         scannerId = addScanner(scanner, region);
2286         scannerName = String.valueOf(scannerId);
2287         ttl = this.scannerLeaseTimeoutPeriod;
2288       }
2289       if (request.hasRenew() && request.getRenew()) {
2290         rsh = scanners.get(scannerName);
2291         lease = regionServer.leases.removeLease(scannerName);
2292         if (lease != null && rsh != null) {
2293           regionServer.leases.addLease(lease);
2294           // Increment the nextCallSeq value which is the next expected from client.
2295           rsh.incNextCallSeq();
2296         }
2297         return builder.build();
2298       }
2299 
2300       quota = getQuotaManager().checkQuota(region, OperationQuota.OperationType.SCAN);
2301       long maxQuotaResultSize = Math.min(maxScannerResultSize, quota.getReadAvailable());
2302       if (rows > 0) {
2303         // if nextCallSeq does not match throw Exception straight away. This needs to be
2304         // performed even before checking of Lease.
2305         // See HBASE-5974
2306         if (request.hasNextCallSeq()) {
2307           if (rsh == null) {
2308             rsh = scanners.get(scannerName);
2309           }
2310           if (rsh != null) {
2311             if (request.getNextCallSeq() != rsh.getNextCallSeq()) {
2312               throw new OutOfOrderScannerNextException(
2313                 "Expected nextCallSeq: " + rsh.getNextCallSeq()
2314                 + " But the nextCallSeq got from client: " + request.getNextCallSeq() +
2315                 "; request=" + TextFormat.shortDebugString(request));
2316             }
2317             // Increment the nextCallSeq value which is the next expected from client.
2318             rsh.incNextCallSeq();
2319           }
2320         }
2321         try {
2322           // Remove lease while its being processed in server; protects against case
2323           // where processing of request takes > lease expiration time.
2324           lease = regionServer.leases.removeLease(scannerName);
2325           List<Result> results = new ArrayList<Result>();
2326           long totalCellSize = 0;
2327           long currentScanResultSize = 0;
2328 
2329           boolean done = false;
2330           // Call coprocessor. Get region info from scanner.
2331           if (region != null && region.getCoprocessorHost() != null) {
2332             Boolean bypass = region.getCoprocessorHost().preScannerNext(
2333               scanner, results, rows);
2334             if (!results.isEmpty()) {
2335               for (Result r : results) {
2336                 for (Cell cell : r.rawCells()) {
2337                   totalCellSize += CellUtil.estimatedSerializedSizeOf(cell);
2338                   currentScanResultSize += CellUtil.estimatedHeapSizeOfWithoutTags(cell);
2339                 }
2340               }
2341             }
2342             if (bypass != null && bypass.booleanValue()) {
2343               done = true;
2344             }
2345           }
2346 
2347           if (!done) {
2348             long maxResultSize = Math.min(scanner.getMaxResultSize(), maxQuotaResultSize);
2349             if (maxResultSize <= 0) {
2350               maxResultSize = maxQuotaResultSize;
2351             }
2352             // This is cells inside a row. Default size is 10 so if many versions or many cfs,
2353             // then we'll resize. Resizings show in profiler. Set it higher than 10. For now
2354             // arbitrary 32. TODO: keep record of general size of results being returned.
2355             List<Cell> values = new ArrayList<Cell>(32);
2356             region.startRegionOperation(Operation.SCAN);
2357             try {
2358               int i = 0;
2359               long before = EnvironmentEdgeManager.currentTime();
2360               synchronized(scanner) {
2361                 boolean stale = (region.getRegionInfo().getReplicaId() != 0);
2362                 boolean clientHandlesPartials =
2363                     request.hasClientHandlesPartials() && request.getClientHandlesPartials();
2364                 boolean clientHandlesHeartbeats =
2365                     request.hasClientHandlesHeartbeats() && request.getClientHandlesHeartbeats();
2366 
2367                 // On the server side we must ensure that the correct ordering of partial results is
2368                 // returned to the client to allow them to properly reconstruct the partial results.
2369                 // If the coprocessor host is adding to the result list, we cannot guarantee the
2370                 // correct ordering of partial results and so we prevent partial results from being
2371                 // formed.
2372                 boolean serverGuaranteesOrderOfPartials = currentScanResultSize == 0;
2373                 boolean allowPartialResults =
2374                     clientHandlesPartials && serverGuaranteesOrderOfPartials && !isSmallScan;
2375                 boolean moreRows = false;
2376 
2377                 // Heartbeat messages occur when the processing of the ScanRequest is exceeds a
2378                 // certain time threshold on the server. When the time threshold is exceeded, the
2379                 // server stops the scan and sends back whatever Results it has accumulated within
2380                 // that time period (may be empty). Since heartbeat messages have the potential to
2381                 // create partial Results (in the event that the timeout occurs in the middle of a
2382                 // row), we must only generate heartbeat messages when the client can handle both
2383                 // heartbeats AND partials
2384                 boolean allowHeartbeatMessages = clientHandlesHeartbeats && allowPartialResults;
2385 
2386                 // Default value of timeLimit is negative to indicate no timeLimit should be
2387                 // enforced.
2388                 long timeLimit = -1;
2389 
2390                 // Set the time limit to be half of the more restrictive timeout value (one of the
2391                 // timeout values must be positive). In the event that both values are positive, the
2392                 // more restrictive of the two is used to calculate the limit.
2393                 if (allowHeartbeatMessages && (scannerLeaseTimeoutPeriod > 0 || rpcTimeout > 0)) {
2394                   long timeLimitDelta;
2395                   if (scannerLeaseTimeoutPeriod > 0 && rpcTimeout > 0) {
2396                     timeLimitDelta = Math.min(scannerLeaseTimeoutPeriod, rpcTimeout);
2397                   } else {
2398                     timeLimitDelta =
2399                         scannerLeaseTimeoutPeriod > 0 ? scannerLeaseTimeoutPeriod : rpcTimeout;
2400                   }
2401                   // Use half of whichever timeout value was more restrictive... But don't allow
2402                   // the time limit to be less than the allowable minimum (could cause an
2403                   // immediatate timeout before scanning any data).
2404                   timeLimitDelta = Math.max(timeLimitDelta / 2, minimumScanTimeLimitDelta);
2405                   timeLimit = System.currentTimeMillis() + timeLimitDelta;
2406                 }
2407 
2408                 final LimitScope sizeScope =
2409                     allowPartialResults ? LimitScope.BETWEEN_CELLS : LimitScope.BETWEEN_ROWS;
2410                 final LimitScope timeScope =
2411                     allowHeartbeatMessages ? LimitScope.BETWEEN_CELLS : LimitScope.BETWEEN_ROWS;
2412 
2413                 // Configure with limits for this RPC. Set keep progress true since size progress
2414                 // towards size limit should be kept between calls to nextRaw
2415                 ScannerContext.Builder contextBuilder = ScannerContext.newBuilder(true);
2416                 contextBuilder.setSizeLimit(sizeScope, maxResultSize);
2417                 contextBuilder.setBatchLimit(scanner.getBatch());
2418                 contextBuilder.setTimeLimit(timeScope, timeLimit);
2419                 ScannerContext scannerContext = contextBuilder.build();
2420 
2421                 boolean limitReached = false;
2422                 while (i < rows) {
2423                   // Reset the batch progress to 0 before every call to RegionScanner#nextRaw. The
2424                   // batch limit is a limit on the number of cells per Result. Thus, if progress is
2425                   // being tracked (i.e. scannerContext.keepProgress() is true) then we need to
2426                   // reset the batch progress between nextRaw invocations since we don't want the
2427                   // batch progress from previous calls to affect future calls
2428                   scannerContext.setBatchProgress(0);
2429 
2430                   // Collect values to be returned here
2431                   moreRows = scanner.nextRaw(values, scannerContext);
2432 
2433                   if (!values.isEmpty()) {
2434                     for (Cell cell : values) {
2435                       totalCellSize += CellUtil.estimatedSerializedSizeOf(cell);
2436                     }
2437                     final boolean partial = scannerContext.partialResultFormed();
2438                     results.add(Result.create(values, null, stale, partial));
2439                     i++;
2440                   }
2441 
2442                   boolean sizeLimitReached = scannerContext.checkSizeLimit(LimitScope.BETWEEN_ROWS);
2443                   boolean timeLimitReached = scannerContext.checkTimeLimit(LimitScope.BETWEEN_ROWS);
2444                   boolean rowLimitReached = i >= rows;
2445                   limitReached = sizeLimitReached || timeLimitReached || rowLimitReached;
2446 
2447                   if (limitReached || !moreRows) {
2448                     if (LOG.isTraceEnabled()) {
2449                       LOG.trace("Done scanning. limitReached: " + limitReached + " moreRows: "
2450                           + moreRows + " scannerContext: " + scannerContext);
2451                     }
2452                     // We only want to mark a ScanResponse as a heartbeat message in the event that
2453                     // there are more values to be read server side. If there aren't more values,
2454                     // marking it as a heartbeat is wasteful because the client will need to issue
2455                     // another ScanRequest only to realize that they already have all the values
2456                     if (moreRows) {
2457                       // Heartbeat messages occur when the time limit has been reached.
2458                       builder.setHeartbeatMessage(timeLimitReached);
2459                     }
2460                     break;
2461                   }
2462                   values.clear();
2463                 }
2464 
2465                 if (limitReached || moreRows) {
2466                   // We stopped prematurely
2467                   builder.setMoreResultsInRegion(true);
2468                 } else {
2469                   // We didn't get a single batch
2470                   builder.setMoreResultsInRegion(false);
2471                 }
2472               }
2473               region.updateReadRequestsCount(i);
2474               long end = EnvironmentEdgeManager.currentTime();
2475               region.getMetrics().updateScanSize(totalCellSize);
2476               region.getMetrics().updateScanTime(end - before);
2477               if (regionServer.metricsRegionServer != null) {
2478                 regionServer.metricsRegionServer.updateScanSize(totalCellSize);
2479                 regionServer.metricsRegionServer.updateScanTime(end - before);
2480               }
2481             } finally {
2482               region.closeRegionOperation();
2483             }
2484 
2485             // coprocessor postNext hook
2486             if (region != null && region.getCoprocessorHost() != null) {
2487               region.getCoprocessorHost().postScannerNext(scanner, results, rows, true);
2488             }
2489           }
2490 
2491           quota.addScanResult(results);
2492 
2493           // If the scanner's filter - if any - is done with the scan
2494           // and wants to tell the client to stop the scan. This is done by passing
2495           // a null result, and setting moreResults to false.
2496           if (scanner.isFilterDone() && results.isEmpty()) {
2497             moreResults = false;
2498             results = null;
2499           } else {
2500             addResults(builder, results, controller, RegionReplicaUtil.isDefaultReplica(region.getRegionInfo()));
2501           }
2502         } catch (IOException e) {
2503           // if we have an exception on scanner next and we are using the callSeq
2504           // we should rollback because the client will retry with the same callSeq
2505           // and get an OutOfOrderScannerNextException if we don't do so.
2506           if (rsh != null && request.hasNextCallSeq()) {
2507             rsh.rollbackNextCallSeq();
2508           }
2509           throw e;
2510         } finally {
2511           // We're done. On way out re-add the above removed lease.
2512           // Adding resets expiration time on lease.
2513           if (scanners.containsKey(scannerName)) {
2514             if (lease != null) regionServer.leases.addLease(lease);
2515             ttl = this.scannerLeaseTimeoutPeriod;
2516           }
2517         }
2518       }
2519 
2520       if (!moreResults || closeScanner) {
2521         ttl = 0;
2522         moreResults = false;
2523         if (region != null && region.getCoprocessorHost() != null) {
2524           if (region.getCoprocessorHost().preScannerClose(scanner)) {
2525             return builder.build(); // bypass
2526           }
2527         }
2528         rsh = scanners.remove(scannerName);
2529         if (rsh != null) {
2530           scanner = rsh.s;
2531           scanner.close();
2532           regionServer.leases.cancelLease(scannerName);
2533           if (region != null && region.getCoprocessorHost() != null) {
2534             region.getCoprocessorHost().postScannerClose(scanner);
2535           }
2536         }
2537       }
2538 
2539       if (ttl > 0) {
2540         builder.setTtl(ttl);
2541       }
2542       builder.setScannerId(scannerId);
2543       builder.setMoreResults(moreResults);
2544       return builder.build();
2545     } catch (IOException ie) {
2546       if (scannerName != null && ie instanceof NotServingRegionException) {
2547         RegionScannerHolder rsh = scanners.remove(scannerName);
2548         if (rsh != null) {
2549           try {
2550             RegionScanner scanner = rsh.s;
2551             LOG.warn(scannerName + " encountered " + ie.getMessage() + ", closing ...");
2552             scanner.close();
2553             regionServer.leases.cancelLease(scannerName);
2554           } catch (IOException e) {
2555            LOG.warn("Getting exception closing " + scannerName, e);
2556           }
2557         }
2558       }
2559       throw new ServiceException(ie);
2560     } finally {
2561       if (quota != null) {
2562         quota.close();
2563       }
2564     }
2565   }
2566 
2567   @Override
2568   public CoprocessorServiceResponse execRegionServerService(RpcController controller,
2569       CoprocessorServiceRequest request) throws ServiceException {
2570     return regionServer.execRegionServerService(controller, request);
2571   }
2572 
2573   @Override
2574   public UpdateConfigurationResponse updateConfiguration(
2575       RpcController controller, UpdateConfigurationRequest request)
2576       throws ServiceException {
2577     try {
2578       this.regionServer.updateConfiguration();
2579     } catch (Exception e) {
2580       throw new ServiceException(e);
2581     }
2582     return UpdateConfigurationResponse.getDefaultInstance();
2583   }
2584 }