1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master.handler;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.NavigableMap;
27 import java.util.Set;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.HRegionInfo;
33 import org.apache.hadoop.hbase.Server;
34 import org.apache.hadoop.hbase.ServerName;
35 import org.apache.hadoop.hbase.catalog.CatalogTracker;
36 import org.apache.hadoop.hbase.catalog.MetaEditor;
37 import org.apache.hadoop.hbase.catalog.MetaReader;
38 import org.apache.hadoop.hbase.client.Result;
39 import org.apache.hadoop.hbase.executor.EventHandler;
40 import org.apache.hadoop.hbase.master.AssignmentManager;
41 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
42 import org.apache.hadoop.hbase.master.DeadServer;
43 import org.apache.hadoop.hbase.master.MasterServices;
44 import org.apache.hadoop.hbase.master.ServerManager;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.util.Pair;
47 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
48 import org.apache.zookeeper.KeeperException;
49
50
51
52
53
54
55 public class ServerShutdownHandler extends EventHandler {
56 private static final Log LOG = LogFactory.getLog(ServerShutdownHandler.class);
57 protected final ServerName serverName;
58 protected final MasterServices services;
59 protected final DeadServer deadServers;
60 protected final boolean shouldSplitHlog;
61
62 public ServerShutdownHandler(final Server server, final MasterServices services,
63 final DeadServer deadServers, final ServerName serverName,
64 final boolean shouldSplitHlog) {
65 this(server, services, deadServers, serverName, EventType.M_SERVER_SHUTDOWN,
66 shouldSplitHlog);
67 }
68
69 ServerShutdownHandler(final Server server, final MasterServices services,
70 final DeadServer deadServers, final ServerName serverName, EventType type,
71 final boolean shouldSplitHlog) {
72 super(server, type);
73 this.serverName = serverName;
74 this.server = server;
75 this.services = services;
76 this.deadServers = deadServers;
77 if (!this.deadServers.contains(this.serverName)) {
78 LOG.warn(this.serverName + " is NOT in deadservers; it should be!");
79 }
80 this.shouldSplitHlog = shouldSplitHlog;
81 }
82
83 @Override
84 public String getInformativeName() {
85 if (serverName != null) {
86 return this.getClass().getSimpleName() + " for " + serverName;
87 } else {
88 return super.getInformativeName();
89 }
90 }
91
92
93
94
95 boolean isCarryingRoot() {
96 return false;
97 }
98
99
100
101
102 boolean isCarryingMeta() {
103 return false;
104 }
105
106 @Override
107 public String toString() {
108 String name = "UnknownServerName";
109 if(server != null && server.getServerName() != null) {
110 name = server.getServerName().toString();
111 }
112 return getClass().getSimpleName() + "-" + name + "-" + getSeqid();
113 }
114
115 @Override
116 public void process() throws IOException {
117 final ServerName serverName = this.serverName;
118 try {
119 try {
120 if (this.shouldSplitHlog) {
121 LOG.info("Splitting logs for " + serverName);
122 this.services.getMasterFileSystem().splitLog(serverName);
123 } else {
124 LOG.info("Skipping log splitting for " + serverName);
125 }
126 } catch (IOException ioe) {
127
128
129 this.services.getExecutorService().submit((ServerShutdownHandler)this);
130 this.deadServers.add(serverName);
131 throw new IOException("failed log splitting for " +
132 serverName + ", will retry", ioe);
133 }
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152 if (isCarryingRoot() || isCarryingMeta()) {
153 this.services.getExecutorService().submit(new ServerShutdownHandler(
154 this.server, this.services, this.deadServers, serverName, false));
155 this.deadServers.add(serverName);
156 return;
157 }
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175 NavigableMap<HRegionInfo, Result> hris = null;
176 while (!this.server.isStopped()) {
177 try {
178 this.server.getCatalogTracker().waitForMeta();
179 hris = MetaReader.getServerUserRegions(this.server.getCatalogTracker(),
180 this.serverName);
181 break;
182 } catch (InterruptedException e) {
183 Thread.currentThread().interrupt();
184 throw new IOException("Interrupted", e);
185 } catch (IOException ioe) {
186 LOG.info("Received exception accessing META during server shutdown of " +
187 serverName + ", retrying META read", ioe);
188 }
189 }
190
191
192
193 Pair<Set<HRegionInfo>, List<RegionState>> p = this.services.getAssignmentManager()
194 .processServerShutdown(this.serverName);
195 Set<HRegionInfo> ritsGoingToServer = p.getFirst();
196 List<RegionState> ritsOnServer = p.getSecond();
197
198 List<HRegionInfo> regionsToAssign = getRegionsToAssign(hris, ritsOnServer, ritsGoingToServer);
199 for (HRegionInfo hri : ritsGoingToServer) {
200 if (!this.services.getAssignmentManager().isRegionAssigned(hri)) {
201 if (!regionsToAssign.contains(hri)) {
202 regionsToAssign.add(hri);
203 }
204 }
205 }
206 for (HRegionInfo hri : regionsToAssign) {
207 this.services.getAssignmentManager().assign(hri, true);
208 }
209 LOG.info(regionsToAssign.size() + " regions which were planned to open on " + this.serverName
210 + " have been re-assigned.");
211 } finally {
212 this.deadServers.finish(serverName);
213 }
214 LOG.info("Finished processing of shutdown of " + serverName);
215 }
216
217
218
219
220
221
222
223
224
225 private List<HRegionInfo> getRegionsToAssign(final NavigableMap<HRegionInfo, Result> metaHRIs,
226 final List<RegionState> ritsOnServer, Set<HRegionInfo> ritsGoingToServer) throws IOException {
227 List<HRegionInfo> toAssign = new ArrayList<HRegionInfo>();
228
229
230 if (metaHRIs == null || metaHRIs.isEmpty()) return toAssign;
231
232
233
234
235
236
237
238
239 for (RegionState rs : ritsOnServer) {
240 if (!rs.isClosing() && !rs.isPendingClose() && !rs.isSplitting()) {
241 LOG.debug("Removed " + rs.getRegion().getRegionNameAsString()
242 + " from list of regions to assign because region state: " + rs.getState());
243 metaHRIs.remove(rs.getRegion());
244 }
245 }
246
247 for (Map.Entry<HRegionInfo, Result> e : metaHRIs.entrySet()) {
248 RegionState rit = services.getAssignmentManager().getRegionsInTransition().get(
249 e.getKey().getEncodedName());
250 AssignmentManager assignmentManager = this.services.getAssignmentManager();
251 if (processDeadRegion(e.getKey(), e.getValue(), assignmentManager,
252 this.server.getCatalogTracker())) {
253 ServerName addressFromAM = assignmentManager.getRegionServerOfRegion(e.getKey());
254 if (rit != null && !rit.isClosing() && !rit.isPendingClose() && !rit.isSplitting()
255 && !ritsGoingToServer.contains(e.getKey())) {
256
257
258 LOG.info("Skip assigning region " + rit.toString());
259 } else if (addressFromAM != null && !addressFromAM.equals(this.serverName)) {
260 LOG.debug("Skip assigning region " + e.getKey().getRegionNameAsString()
261 + " because it has been opened in " + addressFromAM.getServerName());
262 ritsGoingToServer.remove(e.getKey());
263 } else {
264 if (rit != null) {
265
266 try {
267 LOG.info("Reassigning region with rs =" + rit + " and deleting zk node if exists");
268 ZKAssign.deleteNodeFailSilent(services.getZooKeeper(), e.getKey());
269 } catch (KeeperException ke) {
270 this.server.abort("Unexpected ZK exception deleting unassigned node " + e.getKey(),
271 ke);
272 return null;
273 }
274 }
275 toAssign.add(e.getKey());
276 }
277 } else if (rit != null && (rit.isSplitting() || rit.isSplit())) {
278
279
280
281
282
283 HRegionInfo region = rit.getRegion();
284 AssignmentManager am = assignmentManager;
285 am.regionOffline(region);
286 ritsGoingToServer.remove(region);
287 }
288
289
290
291
292 toAssign = checkForDisablingOrDisabledTables(ritsGoingToServer, toAssign, rit, assignmentManager);
293 }
294 return toAssign;
295 }
296
297 private List<HRegionInfo> checkForDisablingOrDisabledTables(Set<HRegionInfo> regionsFromRIT,
298 List<HRegionInfo> toAssign, RegionState rit, AssignmentManager assignmentManager) {
299 if (rit == null) {
300 return toAssign;
301 }
302 if (!rit.isClosing() && !rit.isPendingClose()) {
303 return toAssign;
304 }
305 if (!assignmentManager.getZKTable().isDisablingOrDisabledTable(
306 rit.getRegion().getTableNameAsString())) {
307 return toAssign;
308 }
309 HRegionInfo hri = rit.getRegion();
310 AssignmentManager am = assignmentManager;
311 am.deleteClosingOrClosedNode(hri);
312 am.regionOffline(hri);
313
314 toAssign.remove(hri);
315 regionsFromRIT.remove(hri);
316 return toAssign;
317 }
318
319
320
321
322
323
324
325
326
327
328
329 public static boolean processDeadRegion(HRegionInfo hri, Result result,
330 AssignmentManager assignmentManager, CatalogTracker catalogTracker)
331 throws IOException {
332 boolean tablePresent = assignmentManager.getZKTable().isTablePresent(
333 hri.getTableNameAsString());
334 if (!tablePresent) {
335 LOG.info("The table " + hri.getTableNameAsString()
336 + " was deleted. Hence not proceeding.");
337 return false;
338 }
339
340 boolean disabled = assignmentManager.getZKTable().isDisabledTable(
341 hri.getTableNameAsString());
342 if (disabled){
343 LOG.info("The table " + hri.getTableNameAsString()
344 + " was disabled. Hence not proceeding.");
345 return false;
346 }
347 if (hri.isOffline() && hri.isSplit()) {
348 LOG.debug("Offlined and split region " + hri.getRegionNameAsString() +
349 "; checking daughter presence");
350 if (MetaReader.getRegion(catalogTracker, hri.getRegionName()) == null) {
351 return false;
352 }
353 fixupDaughters(result, assignmentManager, catalogTracker);
354 return false;
355 }
356 boolean disabling = assignmentManager.getZKTable().isDisablingTable(
357 hri.getTableNameAsString());
358 if (disabling) {
359 LOG.info("The table " + hri.getTableNameAsString()
360 + " is disabled. Hence not assigning region" + hri.getEncodedName());
361 return false;
362 }
363 return true;
364 }
365
366
367
368
369
370
371
372
373 public static int fixupDaughters(final Result result,
374 final AssignmentManager assignmentManager,
375 final CatalogTracker catalogTracker)
376 throws IOException {
377 int fixedA = fixupDaughter(result, HConstants.SPLITA_QUALIFIER,
378 assignmentManager, catalogTracker);
379 int fixedB = fixupDaughter(result, HConstants.SPLITB_QUALIFIER,
380 assignmentManager, catalogTracker);
381 return fixedA + fixedB;
382 }
383
384
385
386
387
388
389
390
391 static int fixupDaughter(final Result result, final byte [] qualifier,
392 final AssignmentManager assignmentManager,
393 final CatalogTracker catalogTracker)
394 throws IOException {
395 HRegionInfo daughter =
396 MetaReader.parseHRegionInfoFromCatalogResult(result, qualifier);
397 if (daughter == null) return 0;
398 if (isDaughterMissing(catalogTracker, daughter)) {
399 LOG.info("Fixup; missing daughter " + daughter.getRegionNameAsString());
400 MetaEditor.addDaughter(catalogTracker, daughter, null);
401
402
403
404
405
406
407 assignmentManager.assign(daughter, true);
408 return 1;
409 } else {
410 LOG.debug("Daughter " + daughter.getRegionNameAsString() + " present");
411 }
412 return 0;
413 }
414
415
416
417
418
419
420
421
422 private static boolean isDaughterMissing(final CatalogTracker catalogTracker,
423 final HRegionInfo daughter) throws IOException {
424 FindDaughterVisitor visitor = new FindDaughterVisitor(daughter);
425
426
427
428
429
430
431 byte [] startrow = daughter.getRegionName();
432 MetaReader.fullScan(catalogTracker, visitor, startrow);
433 return !visitor.foundDaughter();
434 }
435
436
437
438
439
440 static class FindDaughterVisitor implements MetaReader.Visitor {
441 private final HRegionInfo daughter;
442 private boolean found = false;
443
444 FindDaughterVisitor(final HRegionInfo daughter) {
445 this.daughter = daughter;
446 }
447
448
449
450
451 boolean foundDaughter() {
452 return this.found;
453 }
454
455 @Override
456 public boolean visit(Result r) throws IOException {
457 HRegionInfo hri =
458 MetaReader.parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER);
459 if (hri == null) {
460 LOG.warn("No serialized HRegionInfo in " + r);
461 return true;
462 }
463 byte [] value = r.getValue(HConstants.CATALOG_FAMILY,
464 HConstants.SERVER_QUALIFIER);
465
466 if (value == null) return false;
467
468
469 if (!Bytes.equals(daughter.getTableName(),
470 hri.getTableName())) {
471
472 return false;
473 }
474
475 if (!Bytes.equals(daughter.getStartKey(), hri.getStartKey())) {
476 return false;
477 }
478
479
480
481 this.found = true;
482 return false;
483 }
484 }
485 }