1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master;
21
22 import java.io.IOException;
23 import java.net.InetAddress;
24 import java.util.ArrayList;
25 import java.util.Collections;
26 import java.util.HashMap;
27 import java.util.HashSet;
28 import java.util.Iterator;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.Set;
32 import java.util.SortedMap;
33 import java.util.Map.Entry;
34 import java.util.concurrent.ConcurrentHashMap;
35 import java.util.concurrent.ConcurrentSkipListMap;
36
37 import org.apache.commons.logging.Log;
38 import org.apache.commons.logging.LogFactory;
39 import org.apache.hadoop.conf.Configuration;
40 import org.apache.hadoop.hbase.ClockOutOfSyncException;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.HServerAddress;
43 import org.apache.hadoop.hbase.HServerLoad;
44 import org.apache.hadoop.hbase.PleaseHoldException;
45 import org.apache.hadoop.hbase.Server;
46 import org.apache.hadoop.hbase.ServerName;
47 import org.apache.hadoop.hbase.YouAreDeadException;
48 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
49 import org.apache.hadoop.hbase.client.HConnection;
50 import org.apache.hadoop.hbase.client.HConnectionManager;
51 import org.apache.hadoop.hbase.client.RetriesExhaustedException;
52 import org.apache.hadoop.hbase.ipc.HRegionInterface;
53 import org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler;
54 import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
55 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
56 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
57 import org.apache.hadoop.hbase.util.Bytes;
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72 public class ServerManager {
73 public static final String WAIT_ON_REGIONSERVERS_MAXTOSTART =
74 "hbase.master.wait.on.regionservers.maxtostart";
75
76 public static final String WAIT_ON_REGIONSERVERS_MINTOSTART =
77 "hbase.master.wait.on.regionservers.mintostart";
78
79 public static final String WAIT_ON_REGIONSERVERS_TIMEOUT =
80 "hbase.master.wait.on.regionservers.timeout";
81
82 public static final String WAIT_ON_REGIONSERVERS_INTERVAL =
83 "hbase.master.wait.on.regionservers.interval";
84
85 private static final Log LOG = LogFactory.getLog(ServerManager.class);
86
87
88 private volatile boolean clusterShutdown = false;
89
90 private final SortedMap<byte[], Long> flushedSequenceIdByRegion =
91 new ConcurrentSkipListMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
92
93
94 private final Map<ServerName, HServerLoad> onlineServers =
95 new ConcurrentHashMap<ServerName, HServerLoad>();
96
97
98
99
100
101 private final Map<ServerName, HRegionInterface> serverConnections =
102 new HashMap<ServerName, HRegionInterface>();
103
104
105
106
107
108 private final ArrayList<ServerName> drainingServers =
109 new ArrayList<ServerName>();
110
111 private final Server master;
112 private final MasterServices services;
113 private final HConnection connection;
114
115 private final DeadServer deadservers;
116
117 private final long maxSkew;
118 private final long warningSkew;
119
120
121
122
123
124
125
126 private Set<ServerName> deadNotExpiredServers = new HashSet<ServerName>();
127
128
129
130
131
132
133
134 public ServerManager(final Server master, final MasterServices services)
135 throws ZooKeeperConnectionException {
136 this(master, services, true);
137 }
138
139 ServerManager(final Server master, final MasterServices services,
140 final boolean connect) throws ZooKeeperConnectionException {
141 this.master = master;
142 this.services = services;
143 Configuration c = master.getConfiguration();
144 maxSkew = c.getLong("hbase.master.maxclockskew", 30000);
145 warningSkew = c.getLong("hbase.master.warningclockskew", 10000);
146 this.deadservers = new DeadServer();
147 this.connection = connect ? HConnectionManager.getConnection(c) : null;
148 }
149
150
151
152
153
154
155
156
157
158
159 ServerName regionServerStartup(final InetAddress ia, final int port,
160 final long serverStartcode, long serverCurrentTime)
161 throws IOException {
162
163
164
165
166
167
168
169 ServerName sn = new ServerName(ia.getHostName(), port, serverStartcode);
170 checkClockSkew(sn, serverCurrentTime);
171 checkIsDead(sn, "STARTUP");
172 checkAlreadySameHostPort(sn);
173 recordNewServer(sn, HServerLoad.EMPTY_HSERVERLOAD);
174 return sn;
175 }
176
177
178
179
180
181
182 private void updateLastFlushedSequenceIds(ServerName sn, HServerLoad hsl) {
183 Map<byte[], HServerLoad.RegionLoad> regionsLoad = hsl.getRegionsLoad();
184 for (Entry<byte[], HServerLoad.RegionLoad> entry : regionsLoad.entrySet()) {
185 Long existingValue = flushedSequenceIdByRegion.get(entry.getKey());
186 long newValue = entry.getValue().getCompleteSequenceId();
187 if (existingValue != null) {
188 if (newValue >= 0 && newValue < existingValue) {
189 if (LOG.isDebugEnabled()) {
190 LOG.debug("RegionServer " + sn +
191 " indicates a last flushed sequence id (" + entry.getValue() +
192 ") that is less than the previous last flushed sequence id (" +
193 existingValue + ") for region " +
194 Bytes.toString(entry.getKey()) + " Ignoring.");
195 }
196 continue;
197 }
198 }
199 flushedSequenceIdByRegion.put(entry.getKey(), newValue);
200 }
201 }
202
203 void regionServerReport(ServerName sn, HServerLoad hsl)
204 throws YouAreDeadException, PleaseHoldException {
205 checkIsDead(sn, "REPORT");
206 if (!this.onlineServers.containsKey(sn)) {
207
208 checkAlreadySameHostPort(sn);
209
210
211
212
213
214 recordNewServer(sn, hsl);
215 } else {
216 this.onlineServers.put(sn, hsl);
217 }
218 updateLastFlushedSequenceIds(sn, hsl);
219 }
220
221
222
223
224
225
226 void checkAlreadySameHostPort(final ServerName serverName)
227 throws PleaseHoldException {
228 ServerName existingServer =
229 ServerName.findServerWithSameHostnamePort(getOnlineServersList(), serverName);
230 if (existingServer != null) {
231 String message = "Server serverName=" + serverName +
232 " rejected; we already have " + existingServer.toString() +
233 " registered with same hostname and port";
234 LOG.info(message);
235 if (existingServer.getStartcode() < serverName.getStartcode()) {
236 LOG.info("Triggering server recovery; existingServer " +
237 existingServer + " looks stale, new server:" + serverName);
238 expireServer(existingServer);
239 }
240 if (services.isServerShutdownHandlerEnabled()) {
241
242 throw new PleaseHoldException(message);
243 }
244 }
245 }
246
247
248
249
250
251
252
253
254
255 private void checkClockSkew(final ServerName serverName, final long serverCurrentTime)
256 throws ClockOutOfSyncException {
257 long skew = System.currentTimeMillis() - serverCurrentTime;
258 if (skew > maxSkew) {
259 String message = "Server " + serverName + " has been " +
260 "rejected; Reported time is too far out of sync with master. " +
261 "Time difference of " + skew + "ms > max allowed of " + maxSkew + "ms";
262 LOG.warn(message);
263 throw new ClockOutOfSyncException(message);
264 } else if (skew > warningSkew){
265 String message = "Reported time for server " + serverName + " is out of sync with master " +
266 "by " + skew + "ms. (Warning threshold is " + warningSkew + "ms; " +
267 "error threshold is " + maxSkew + "ms)";
268 LOG.warn(message);
269 }
270 }
271
272
273
274
275
276
277
278
279
280 private void checkIsDead(final ServerName serverName, final String what)
281 throws YouAreDeadException {
282 if (this.deadservers.isDeadServer(serverName)) {
283
284
285 String message = "Server " + what + " rejected; currently processing " +
286 serverName + " as dead server";
287 LOG.debug(message);
288 throw new YouAreDeadException(message);
289 }
290
291
292
293 if ((this.services == null || ((HMaster) this.services).isInitialized())
294 && this.deadservers.cleanPreviousInstance(serverName)) {
295
296
297 LOG.debug(what + ":" + " Server " + serverName + " came back up," +
298 " removed it from the dead servers list");
299 }
300 }
301
302
303
304
305
306
307 void recordNewServer(final ServerName serverName, final HServerLoad hsl) {
308 LOG.info("Registering server=" + serverName);
309 this.onlineServers.put(serverName, hsl);
310 this.serverConnections.remove(serverName);
311 }
312
313 public long getLastFlushedSequenceId(byte[] regionName) {
314 long seqId = Long.MIN_VALUE;
315 if (flushedSequenceIdByRegion.containsKey(regionName)) {
316 seqId = flushedSequenceIdByRegion.get(regionName);
317 }
318 return seqId;
319 }
320
321
322
323
324
325 public HServerLoad getLoad(final ServerName serverName) {
326 return this.onlineServers.get(serverName);
327 }
328
329
330
331
332
333
334 public HServerLoad getLoad(final HServerAddress address) {
335 ServerName sn = new ServerName(address.toString(), ServerName.NON_STARTCODE);
336 ServerName actual =
337 ServerName.findServerWithSameHostnamePort(this.getOnlineServersList(), sn);
338 return actual == null? null: getLoad(actual);
339 }
340
341
342
343
344
345
346
347 public double getAverageLoad() {
348 int totalLoad = 0;
349 int numServers = 0;
350 double averageLoad = 0.0;
351 for (HServerLoad hsl: this.onlineServers.values()) {
352 numServers++;
353 totalLoad += hsl.getNumberOfRegions();
354 }
355 averageLoad = (double)totalLoad / (double)numServers;
356 return averageLoad;
357 }
358
359
360 int countOfRegionServers() {
361
362 return this.onlineServers.size();
363 }
364
365
366
367
368 public Map<ServerName, HServerLoad> getOnlineServers() {
369
370 synchronized (this.onlineServers) {
371 return Collections.unmodifiableMap(this.onlineServers);
372 }
373 }
374
375 public Set<ServerName> getDeadServers() {
376 return this.deadservers.clone();
377 }
378
379
380
381
382
383 public boolean areDeadServersInProgress() {
384 return this.deadservers.areDeadServersInProgress();
385 }
386
387 void letRegionServersShutdown() {
388 long previousLogTime = 0;
389 while (!onlineServers.isEmpty()) {
390
391 if (System.currentTimeMillis() > (previousLogTime + 1000)) {
392 StringBuilder sb = new StringBuilder();
393 for (ServerName key : this.onlineServers.keySet()) {
394 if (sb.length() > 0) {
395 sb.append(", ");
396 }
397 sb.append(key);
398 }
399 LOG.info("Waiting on regionserver(s) to go down " + sb.toString());
400 previousLogTime = System.currentTimeMillis();
401 }
402
403 synchronized (onlineServers) {
404 try {
405 onlineServers.wait(100);
406 } catch (InterruptedException ignored) {
407
408 }
409 }
410 }
411 }
412
413
414
415
416
417 public synchronized void expireServer(final ServerName serverName) {
418 if (!services.isServerShutdownHandlerEnabled()) {
419 LOG.info("Master doesn't enable ServerShutdownHandler during initialization, "
420 + "delay expiring server " + serverName);
421 this.deadNotExpiredServers.add(serverName);
422 return;
423 }
424 if (!this.onlineServers.containsKey(serverName)) {
425 LOG.warn("Received expiration of " + serverName +
426 " but server is not currently online");
427 return;
428 }
429 if (this.deadservers.contains(serverName)) {
430
431 LOG.warn("Received expiration of " + serverName +
432 " but server shutdown is already in progress");
433 return;
434 }
435
436
437
438 this.deadservers.add(serverName);
439 this.onlineServers.remove(serverName);
440 synchronized (onlineServers) {
441 onlineServers.notifyAll();
442 }
443 this.serverConnections.remove(serverName);
444
445
446 if (this.clusterShutdown) {
447 LOG.info("Cluster shutdown set; " + serverName +
448 " expired; onlineServers=" + this.onlineServers.size());
449 if (this.onlineServers.isEmpty()) {
450 master.stop("Cluster shutdown set; onlineServer=0");
451 }
452 return;
453 }
454
455 boolean carryingRoot = services.getAssignmentManager().isCarryingRoot(serverName);
456 boolean carryingMeta = services.getAssignmentManager().isCarryingMeta(serverName);
457 if (carryingRoot || carryingMeta) {
458 this.services.getExecutorService().submit(new MetaServerShutdownHandler(this.master,
459 this.services, this.deadservers, serverName, carryingRoot, carryingMeta));
460 } else {
461 this.services.getExecutorService().submit(new ServerShutdownHandler(this.master,
462 this.services, this.deadservers, serverName, true));
463 }
464 LOG.debug("Added=" + serverName +
465 " to dead servers, submitted shutdown handler to be executed, root=" +
466 carryingRoot + ", meta=" + carryingMeta);
467 }
468
469
470
471
472
473
474 synchronized void expireDeadNotExpiredServers() throws IOException {
475 if (!services.isServerShutdownHandlerEnabled()) {
476 throw new IOException("Master hasn't enabled ServerShutdownHandler ");
477 }
478 Iterator<ServerName> serverIterator = deadNotExpiredServers.iterator();
479 while (serverIterator.hasNext()) {
480 expireServer(serverIterator.next());
481 serverIterator.remove();
482 }
483 }
484
485
486
487
488 public boolean removeServerFromDrainList(final ServerName sn) {
489
490
491
492 if (!this.isServerOnline(sn)) {
493 LOG.warn("Server " + sn + " is not currently online. " +
494 "Removing from draining list anyway, as requested.");
495 }
496
497 return this.drainingServers.remove(sn);
498 }
499
500
501
502
503 public boolean addServerToDrainList(final ServerName sn) {
504
505
506
507 if (!this.isServerOnline(sn)) {
508 LOG.warn("Server " + sn + " is not currently online. " +
509 "Ignoring request to add it to draining list.");
510 return false;
511 }
512
513
514 if (this.drainingServers.contains(sn)) {
515 LOG.warn("Server " + sn + " is already in the draining server list." +
516 "Ignoring request to add it again.");
517 return false;
518 }
519 return this.drainingServers.add(sn);
520 }
521
522
523
524
525
526
527
528
529
530
531
532
533
534 public RegionOpeningState sendRegionOpen(final ServerName server,
535 HRegionInfo region, int versionOfOfflineNode)
536 throws IOException {
537 HRegionInterface hri = getServerConnection(server);
538 if (hri == null) {
539 LOG.warn("Attempting to send OPEN RPC to server " + server.toString() +
540 " failed because no RPC connection found to this server");
541 return RegionOpeningState.FAILED_OPENING;
542 }
543 return (versionOfOfflineNode == -1) ? hri.openRegion(region) : hri
544 .openRegion(region, versionOfOfflineNode);
545 }
546
547
548
549
550
551
552
553
554
555 public void sendRegionOpen(ServerName server, List<HRegionInfo> regions)
556 throws IOException {
557 HRegionInterface hri = getServerConnection(server);
558 if (hri == null) {
559 LOG.warn("Attempting to send OPEN RPC to server " + server.toString() +
560 " failed because no RPC connection found to this server");
561 return;
562 }
563 hri.openRegions(regions);
564 }
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579 public boolean sendRegionClose(ServerName server, HRegionInfo region,
580 int versionOfClosingNode) throws IOException {
581 if (server == null) throw new NullPointerException("Passed server is null");
582 HRegionInterface hri = getServerConnection(server);
583 if (hri == null) {
584 throw new IOException("Attempting to send CLOSE RPC to server " +
585 server.toString() + " for region " +
586 region.getRegionNameAsString() +
587 " failed because no RPC connection found to this server");
588 }
589 return hri.closeRegion(region, versionOfClosingNode);
590 }
591
592
593
594
595
596
597
598
599 private HRegionInterface getServerConnection(final ServerName sn)
600 throws IOException {
601 HRegionInterface hri = this.serverConnections.get(sn);
602 if (hri == null) {
603 LOG.debug("New connection to " + sn.toString());
604 hri = this.connection.getHRegionConnection(sn.getHostname(), sn.getPort());
605 this.serverConnections.put(sn, hri);
606 }
607 return hri;
608 }
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623 public void waitForRegionServers(MonitoredTask status)
624 throws InterruptedException {
625 final long interval = this.master.getConfiguration().
626 getLong(WAIT_ON_REGIONSERVERS_INTERVAL, 1500);
627 final long timeout = this.master.getConfiguration().
628 getLong(WAIT_ON_REGIONSERVERS_TIMEOUT, 4500);
629 int minToStart = this.master.getConfiguration().
630 getInt(WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
631 if (minToStart < 1) {
632 LOG.warn(String.format(
633 "The value of '%s' (%d) can not be less than 1, ignoring.",
634 WAIT_ON_REGIONSERVERS_MINTOSTART, minToStart));
635 minToStart = 1;
636 }
637 int maxToStart = this.master.getConfiguration().
638 getInt(WAIT_ON_REGIONSERVERS_MAXTOSTART, Integer.MAX_VALUE);
639 if (maxToStart < minToStart) {
640 LOG.warn(String.format(
641 "The value of '%s' (%d) is set less than '%s' (%d), ignoring.",
642 WAIT_ON_REGIONSERVERS_MAXTOSTART, maxToStart,
643 WAIT_ON_REGIONSERVERS_MINTOSTART, minToStart));
644 maxToStart = Integer.MAX_VALUE;
645 }
646
647 long now = System.currentTimeMillis();
648 final long startTime = now;
649 long slept = 0;
650 long lastLogTime = 0;
651 long lastCountChange = startTime;
652 int count = countOfRegionServers();
653 int oldCount = 0;
654 while (
655 !this.master.isStopped() &&
656 count < maxToStart &&
657 (lastCountChange+interval > now || timeout > slept || count < minToStart)
658 ){
659
660
661 if (oldCount != count || lastLogTime+interval < now){
662 lastLogTime = now;
663 String msg =
664 "Waiting for region servers count to settle; currently"+
665 " checked in " + count + ", slept for " + slept + " ms," +
666 " expecting minimum of " + minToStart + ", maximum of "+ maxToStart+
667 ", timeout of "+timeout+" ms, interval of "+interval+" ms.";
668 LOG.info(msg);
669 status.setStatus(msg);
670 }
671
672
673 final long sleepTime = 50;
674 Thread.sleep(sleepTime);
675 now = System.currentTimeMillis();
676 slept = now - startTime;
677
678 oldCount = count;
679 count = countOfRegionServers();
680 if (count != oldCount) {
681 lastCountChange = now;
682 }
683 }
684
685 LOG.info("Finished waiting for region servers count to settle;" +
686 " checked in " + count + ", slept for " + slept + " ms," +
687 " expecting minimum of " + minToStart + ", maximum of "+ maxToStart+","+
688 " master is "+ (this.master.isStopped() ? "stopped.": "running.")
689 );
690 }
691
692
693
694
695 public List<ServerName> getOnlineServersList() {
696
697
698 return new ArrayList<ServerName>(this.onlineServers.keySet());
699 }
700
701
702
703
704 public List<ServerName> getDrainingServersList() {
705 return new ArrayList<ServerName>(this.drainingServers);
706 }
707
708
709
710
711 Set<ServerName> getDeadNotExpiredServers() {
712 return new HashSet<ServerName>(this.deadNotExpiredServers);
713 }
714
715 public boolean isServerOnline(ServerName serverName) {
716 return onlineServers.containsKey(serverName);
717 }
718
719 public void shutdownCluster() {
720 this.clusterShutdown = true;
721 this.master.stop("Cluster shutdown requested");
722 }
723
724 public boolean isClusterShutdown() {
725 return this.clusterShutdown;
726 }
727
728
729
730
731 public void stop() {
732 if (connection != null) {
733 try {
734 connection.close();
735 } catch (IOException e) {
736 LOG.error("Attempt to close connection to master failed", e);
737 }
738 }
739 }
740
741
742
743
744 void clearDeadServersWithSameHostNameAndPortOfOnlineServer() {
745 ServerName sn = null;
746 for (ServerName serverName : getOnlineServersList()) {
747 while ((sn = ServerName.
748 findServerWithSameHostnamePort(this.deadservers, serverName)) != null) {
749 this.deadservers.remove(sn);
750 }
751 }
752 }
753
754 }