1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.ipc;
22
23 import java.io.IOException;
24 import java.lang.reflect.Array;
25 import java.lang.reflect.InvocationHandler;
26 import java.lang.reflect.InvocationTargetException;
27 import java.lang.reflect.Method;
28 import java.lang.reflect.Proxy;
29 import java.net.InetSocketAddress;
30 import java.util.HashMap;
31 import java.util.Map;
32 import java.util.concurrent.ConcurrentHashMap;
33
34 import javax.net.SocketFactory;
35
36 import org.apache.commons.logging.Log;
37 import org.apache.commons.logging.LogFactory;
38 import org.apache.hadoop.conf.Configuration;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.client.Operation;
41 import org.apache.hadoop.hbase.io.HbaseObjectWritable;
42 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
43 import org.apache.hadoop.hbase.regionserver.HRegionServer;
44 import org.apache.hadoop.hbase.security.User;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.util.Objects;
47 import org.apache.hadoop.io.Writable;
48 import org.apache.hadoop.ipc.RPC;
49 import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
50 import org.codehaus.jackson.map.ObjectMapper;
51
52
53 class WritableRpcEngine implements RpcEngine {
54
55
56 private static final Log LOG = LogFactory.getLog("org.apache.hadoop.ipc.RPCEngine");
57
58 private static class Invoker implements InvocationHandler {
59 private Class<? extends VersionedProtocol> protocol;
60 private InetSocketAddress address;
61 private User ticket;
62 private HBaseClient client;
63 final private int rpcTimeout;
64
65 public Invoker(HBaseClient client,
66 Class<? extends VersionedProtocol> protocol,
67 InetSocketAddress address, User ticket,
68 Configuration conf, int rpcTimeout) {
69 this.protocol = protocol;
70 this.address = address;
71 this.ticket = ticket;
72 this.client = client;
73 this.rpcTimeout = rpcTimeout;
74 }
75
76 public Object invoke(Object proxy, Method method, Object[] args)
77 throws Throwable {
78 final boolean logDebug = LOG.isDebugEnabled();
79 long startTime = 0;
80 if (logDebug) {
81 startTime = System.currentTimeMillis();
82 }
83
84 HbaseObjectWritable value = (HbaseObjectWritable)
85 client.call(new Invocation(method, protocol, args), address,
86 protocol, ticket, rpcTimeout);
87 if (logDebug) {
88
89 long callTime = System.currentTimeMillis() - startTime;
90 LOG.debug("Call: " + method.getName() + " " + callTime);
91 }
92 return value.get();
93 }
94 }
95
96 private Configuration conf;
97 private HBaseClient client;
98
99 @Override
100 public void setConf(Configuration config) {
101 this.conf = config;
102
103 if (this.client != null) {
104 this.client.stop();
105 }
106 this.client = new HBaseClient(HbaseObjectWritable.class, conf);
107 }
108
109 @Override
110 public Configuration getConf() {
111 return conf;
112 }
113
114
115
116 @Override
117 public <T extends VersionedProtocol> T getProxy(
118 Class<T> protocol, long clientVersion,
119 InetSocketAddress addr, Configuration conf, int rpcTimeout)
120 throws IOException {
121 if (this.client == null) {
122 throw new IOException("Client must be initialized by calling setConf(Configuration)");
123 }
124
125 T proxy =
126 (T) Proxy.newProxyInstance(
127 protocol.getClassLoader(), new Class[] { protocol },
128 new Invoker(client, protocol, addr, User.getCurrent(), conf,
129 HBaseRPC.getRpcTimeout(rpcTimeout)));
130
131
132
133
134
135
136 long serverVersion = ((VersionedProtocol)proxy)
137 .getProtocolVersion(protocol.getName(), clientVersion);
138 if (serverVersion != clientVersion) {
139 throw new HBaseRPC.VersionMismatch(protocol.getName(), clientVersion,
140 serverVersion);
141 }
142
143 return proxy;
144 }
145
146
147
148
149 @Override
150 public Object[] call(Method method, Object[][] params,
151 InetSocketAddress[] addrs,
152 Class<? extends VersionedProtocol> protocol,
153 User ticket, Configuration conf)
154 throws IOException, InterruptedException {
155 if (this.client == null) {
156 throw new IOException("Client must be initialized by calling setConf(Configuration)");
157 }
158
159 Invocation[] invocations = new Invocation[params.length];
160 for (int i = 0; i < params.length; i++) {
161 invocations[i] = new Invocation(method, protocol, params[i]);
162 }
163
164 Writable[] wrappedValues =
165 client.call(invocations, addrs, protocol, ticket);
166
167 if (method.getReturnType() == Void.TYPE) {
168 return null;
169 }
170
171 Object[] values =
172 (Object[])Array.newInstance(method.getReturnType(), wrappedValues.length);
173 for (int i = 0; i < values.length; i++) {
174 if (wrappedValues[i] != null) {
175 values[i] = ((HbaseObjectWritable)wrappedValues[i]).get();
176 }
177 }
178
179 return values;
180 }
181
182 @Override
183 public void close() {
184 if (this.client != null) {
185 this.client.stop();
186 }
187 }
188
189
190
191 public Server getServer(Class<? extends VersionedProtocol> protocol,
192 Object instance,
193 Class<?>[] ifaces,
194 String bindAddress, int port,
195 int numHandlers,
196 int metaHandlerCount, boolean verbose,
197 Configuration conf, int highPriorityLevel)
198 throws IOException {
199 return new Server(instance, ifaces, conf, bindAddress, port, numHandlers,
200 metaHandlerCount, verbose, highPriorityLevel);
201 }
202
203
204 public static class Server extends HBaseServer {
205 private Object instance;
206 private Class<?> implementation;
207 private Class<?>[] ifaces;
208 private boolean verbose;
209 private boolean authorize = false;
210
211
212 private static ObjectMapper mapper = new ObjectMapper();
213
214 private static final String WARN_RESPONSE_TIME =
215 "hbase.ipc.warn.response.time";
216 private static final String WARN_RESPONSE_SIZE =
217 "hbase.ipc.warn.response.size";
218
219
220 private static final int DEFAULT_WARN_RESPONSE_TIME = 10000;
221 private static final int DEFAULT_WARN_RESPONSE_SIZE = 100 * 1024 * 1024;
222
223
224 private static final String ABOVE_ONE_SEC_METRIC = ".aboveOneSec.";
225
226 private final int warnResponseTime;
227 private final int warnResponseSize;
228
229 private static String classNameBase(String className) {
230 String[] names = className.split("\\.", -1);
231 if (names == null || names.length == 0) {
232 return className;
233 }
234 return names[names.length-1];
235 }
236
237
238
239
240
241
242
243
244
245
246 public Server(Object instance, final Class<?>[] ifaces,
247 Configuration conf, String bindAddress, int port,
248 int numHandlers, int metaHandlerCount, boolean verbose,
249 int highPriorityLevel) throws IOException {
250 super(bindAddress, port, Invocation.class, numHandlers, metaHandlerCount,
251 conf, classNameBase(instance.getClass().getName()),
252 highPriorityLevel);
253 this.instance = instance;
254 this.implementation = instance.getClass();
255 this.verbose = verbose;
256
257 this.ifaces = ifaces;
258
259
260 String [] metricSuffixes = new String [] {ABOVE_ONE_SEC_METRIC};
261 this.rpcMetrics.createMetrics(this.ifaces, false, metricSuffixes);
262
263 this.authorize =
264 conf.getBoolean(
265 ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, false);
266
267 this.warnResponseTime = conf.getInt(WARN_RESPONSE_TIME,
268 DEFAULT_WARN_RESPONSE_TIME);
269 this.warnResponseSize = conf.getInt(WARN_RESPONSE_SIZE,
270 DEFAULT_WARN_RESPONSE_SIZE);
271 }
272
273 private Map<String, Method> methodMaps = new ConcurrentHashMap<String, Method>();
274
275 private Method getMethod(Class<? extends VersionedProtocol> protocol,
276 String methodName, Class[] parameterClasses) throws Exception {
277 StringBuilder sb = new StringBuilder();
278
279 sb.append(protocol.getName());
280 sb.append('-');
281 sb.append(methodName);
282 sb.append('-');
283 for (Class c : parameterClasses) {
284 sb.append(c.getName());
285 sb.append('-');
286 }
287 String signature = sb.toString();
288 Method method = methodMaps.get(signature);
289 if (method == null) {
290 method = protocol.getMethod(methodName, parameterClasses);
291 method.setAccessible(true);
292 methodMaps.put(signature, method);
293 }
294 return method;
295 }
296
297 @Override
298 public Writable call(Class<? extends VersionedProtocol> protocol,
299 Writable param, long receivedTime, MonitoredRPCHandler status)
300 throws IOException {
301 try {
302 Invocation call = (Invocation)param;
303 if(call.getMethodName() == null) {
304 throw new IOException("Could not find requested method, the usual " +
305 "cause is a version mismatch between client and server.");
306 }
307 if (verbose) log("Call: " + call);
308 status.setRPC(call.getMethodName(), call.getParameters(), receivedTime);
309 status.setRPCPacket(param);
310 status.resume("Servicing call");
311
312 Method method = getMethod(protocol, call.getMethodName(),
313 call.getParameterClasses());
314
315
316
317 if (!method.getDeclaringClass().equals(VersionedProtocol.class)) {
318 long clientVersion = call.getProtocolVersion();
319 ProtocolSignature serverInfo = ((VersionedProtocol) instance)
320 .getProtocolSignature(protocol.getCanonicalName(), call
321 .getProtocolVersion(), call.getClientMethodsHash());
322 long serverVersion = serverInfo.getVersion();
323 if (serverVersion != clientVersion) {
324 LOG.warn("Version mismatch: client version=" + clientVersion
325 + ", server version=" + serverVersion);
326 throw new RPC.VersionMismatch(protocol.getName(), clientVersion,
327 serverVersion);
328 }
329 }
330 Object impl = null;
331 if (protocol.isAssignableFrom(this.implementation)) {
332 impl = this.instance;
333 }
334 else {
335 throw new HBaseRPC.UnknownProtocolException(protocol);
336 }
337
338 long startTime = System.currentTimeMillis();
339 Object[] params = call.getParameters();
340 Object value = method.invoke(impl, params);
341 int processingTime = (int) (System.currentTimeMillis() - startTime);
342 int qTime = (int) (startTime-receivedTime);
343 if (TRACELOG.isDebugEnabled()) {
344 TRACELOG.debug("Call #" + CurCall.get().id +
345 "; Served: " + protocol.getSimpleName()+"#"+call.getMethodName() +
346 " queueTime=" + qTime +
347 " processingTime=" + processingTime +
348 " contents=" + Objects.describeQuantity(params));
349 }
350 rpcMetrics.rpcQueueTime.inc(qTime);
351 rpcMetrics.rpcProcessingTime.inc(processingTime);
352 rpcMetrics.inc(call.getMethodName(), processingTime);
353 if (verbose) log("Return: "+value);
354
355 HbaseObjectWritable retVal =
356 new HbaseObjectWritable(method.getReturnType(), value);
357 long responseSize = retVal.getWritableSize();
358
359
360 boolean tooSlow = (processingTime > warnResponseTime
361 && warnResponseTime > -1);
362 boolean tooLarge = (responseSize > warnResponseSize
363 && warnResponseSize > -1);
364 if (tooSlow || tooLarge) {
365
366
367 logResponse(call, (tooLarge ? "TooLarge" : "TooSlow"),
368 status.getClient(), startTime, processingTime, qTime,
369 responseSize);
370
371 if (tooSlow) {
372 rpcMetrics.rpcSlowResponseTime.inc(processingTime);
373 }
374 }
375 if (processingTime > 1000) {
376
377
378
379 rpcMetrics.inc(call.getMethodName() + ABOVE_ONE_SEC_METRIC,
380 processingTime);
381 }
382
383 return retVal;
384 } catch (InvocationTargetException e) {
385 Throwable target = e.getTargetException();
386 if (target instanceof IOException) {
387 throw (IOException)target;
388 }
389 IOException ioe = new IOException(target.toString());
390 ioe.setStackTrace(target.getStackTrace());
391 throw ioe;
392 } catch (Throwable e) {
393 if (!(e instanceof IOException)) {
394 LOG.error("Unexpected throwable object ", e);
395 }
396 IOException ioe = new IOException(e.toString());
397 ioe.setStackTrace(e.getStackTrace());
398 throw ioe;
399 }
400 }
401
402
403
404
405
406
407
408
409
410
411
412
413
414 private void logResponse(Invocation call, String tag, String clientAddress,
415 long startTime, int processingTime, int qTime, long responseSize)
416 throws IOException {
417 Object params[] = call.getParameters();
418
419 ObjectMapper mapper = new ObjectMapper();
420
421 Map<String, Object> responseInfo = new HashMap<String, Object>();
422 responseInfo.put("starttimems", startTime);
423 responseInfo.put("processingtimems", processingTime);
424 responseInfo.put("queuetimems", qTime);
425 responseInfo.put("responsesize", responseSize);
426 responseInfo.put("client", clientAddress);
427 responseInfo.put("class", instance.getClass().getSimpleName());
428 responseInfo.put("method", call.getMethodName());
429 if (params.length == 2 && instance instanceof HRegionServer &&
430 params[0] instanceof byte[] &&
431 params[1] instanceof Operation) {
432
433
434 byte [] tableName =
435 HRegionInfo.parseRegionName((byte[]) params[0])[0];
436 responseInfo.put("table", Bytes.toStringBinary(tableName));
437
438 responseInfo.putAll(((Operation) params[1]).toMap());
439
440 LOG.warn("(operation" + tag + "): " +
441 mapper.writeValueAsString(responseInfo));
442 } else if (params.length == 1 && instance instanceof HRegionServer &&
443 params[0] instanceof Operation) {
444
445 responseInfo.putAll(((Operation) params[0]).toMap());
446
447 LOG.warn("(operation" + tag + "): " +
448 mapper.writeValueAsString(responseInfo));
449 } else {
450
451
452 responseInfo.put("call", call.toString());
453 LOG.warn("(response" + tag + "): " +
454 mapper.writeValueAsString(responseInfo));
455 }
456 }
457 }
458
459 protected static void log(String value) {
460 String v = value;
461 if (v != null && v.length() > 55)
462 v = v.substring(0, 55)+"...";
463 LOG.info(v);
464 }
465 }