1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.List;
25 import java.util.Set;
26 import java.util.UUID;
27 import java.util.concurrent.locks.Lock;
28 import java.util.concurrent.locks.ReentrantLock;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.conf.Configuration;
33 import org.apache.hadoop.fs.FileStatus;
34 import org.apache.hadoop.fs.FileSystem;
35 import org.apache.hadoop.fs.Path;
36 import org.apache.hadoop.fs.PathFilter;
37 import org.apache.hadoop.hbase.HColumnDescriptor;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.HTableDescriptor;
41 import org.apache.hadoop.hbase.InvalidFamilyOperationException;
42 import org.apache.hadoop.hbase.RemoteExceptionHandler;
43 import org.apache.hadoop.hbase.Server;
44 import org.apache.hadoop.hbase.ServerName;
45 import org.apache.hadoop.hbase.backup.HFileArchiver;
46 import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
47 import org.apache.hadoop.hbase.regionserver.HRegion;
48 import org.apache.hadoop.hbase.regionserver.wal.HLog;
49 import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
50 import org.apache.hadoop.hbase.regionserver.wal.OrphanHLogAfterSplitException;
51 import org.apache.hadoop.hbase.util.Bytes;
52 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
53 import org.apache.hadoop.hbase.util.FSTableDescriptors;
54 import org.apache.hadoop.hbase.util.FSUtils;
55
56
57
58
59
60
61 public class MasterFileSystem {
62 private static final Log LOG = LogFactory.getLog(MasterFileSystem.class.getName());
63
64 Configuration conf;
65
66 Server master;
67
68 MasterMetrics metrics;
69
70 private String clusterId;
71
72 private final FileSystem fs;
73
74 private volatile boolean fsOk = true;
75
76 private final Path oldLogDir;
77
78 private final Path rootdir;
79
80 private final Path tempdir;
81
82 final Lock splitLogLock = new ReentrantLock();
83 final boolean distributedLogSplitting;
84 final SplitLogManager splitLogManager;
85 private final MasterServices services;
86
87 private final static PathFilter META_FILTER = new PathFilter() {
88 public boolean accept(Path p) {
89 return HLog.isMetaFile(p);
90 }
91 };
92
93 private final static PathFilter NON_META_FILTER = new PathFilter() {
94 public boolean accept(Path p) {
95 return !HLog.isMetaFile(p);
96 }
97 };
98
99 public MasterFileSystem(Server master, MasterServices services,
100 MasterMetrics metrics, boolean masterRecovery)
101 throws IOException {
102 this.conf = master.getConfiguration();
103 this.master = master;
104 this.services = services;
105 this.metrics = metrics;
106
107
108
109
110 this.rootdir = FSUtils.getRootDir(conf);
111 this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
112
113
114 this.fs = this.rootdir.getFileSystem(conf);
115 String fsUri = this.fs.getUri().toString();
116 conf.set("fs.default.name", fsUri);
117 conf.set("fs.defaultFS", fsUri);
118
119 fs.setConf(conf);
120 this.distributedLogSplitting =
121 conf.getBoolean("hbase.master.distributed.log.splitting", true);
122 if (this.distributedLogSplitting) {
123 this.splitLogManager = new SplitLogManager(master.getZooKeeper(),
124 master.getConfiguration(), master, this.services, master.getServerName().toString());
125 this.splitLogManager.finishInitialization(masterRecovery);
126 } else {
127 this.splitLogManager = null;
128 }
129
130
131 this.oldLogDir = createInitialFileSystemLayout();
132 }
133
134
135
136
137
138
139
140
141
142
143
144 private Path createInitialFileSystemLayout() throws IOException {
145
146 checkRootDir(this.rootdir, conf, this.fs);
147
148
149 checkTempDir(this.tempdir, conf, this.fs);
150
151 Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
152
153
154 if(!this.fs.exists(oldLogDir)) {
155 this.fs.mkdirs(oldLogDir);
156 }
157
158 return oldLogDir;
159 }
160
161 public FileSystem getFileSystem() {
162 return this.fs;
163 }
164
165
166
167
168
169 public Path getOldLogDir() {
170 return this.oldLogDir;
171 }
172
173
174
175
176
177
178 public boolean checkFileSystem() {
179 if (this.fsOk) {
180 try {
181 FSUtils.checkFileSystemAvailable(this.fs);
182 FSUtils.checkDfsSafeMode(this.conf);
183 } catch (IOException e) {
184 master.abort("Shutting down HBase cluster: file system not available", e);
185 this.fsOk = false;
186 }
187 }
188 return this.fsOk;
189 }
190
191
192
193
194 public Path getRootDir() {
195 return this.rootdir;
196 }
197
198
199
200
201 public Path getTempDir() {
202 return this.tempdir;
203 }
204
205
206
207
208 public String getClusterId() {
209 return clusterId;
210 }
211
212
213
214
215
216 void splitLogAfterStartup() {
217 boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
218 HLog.SPLIT_SKIP_ERRORS_DEFAULT);
219 Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
220 do {
221 if (master.isStopped()) {
222 LOG.warn("Master stopped while splitting logs");
223 break;
224 }
225 List<ServerName> serverNames = new ArrayList<ServerName>();
226 try {
227 if (!this.fs.exists(logsDirPath)) return;
228 FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
229
230
231 Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
232 .keySet();
233
234 if (logFolders == null || logFolders.length == 0) {
235 LOG.debug("No log files to split, proceeding...");
236 return;
237 }
238 for (FileStatus status : logFolders) {
239 String sn = status.getPath().getName();
240
241 if (sn.endsWith(HLog.SPLITTING_EXT)) {
242 sn = sn.substring(0, sn.length() - HLog.SPLITTING_EXT.length());
243 }
244 ServerName serverName = ServerName.parseServerName(sn);
245 if (!onlineServers.contains(serverName)) {
246 LOG.info("Log folder " + status.getPath() + " doesn't belong "
247 + "to a known region server, splitting");
248 serverNames.add(serverName);
249 } else {
250 LOG.info("Log folder " + status.getPath()
251 + " belongs to an existing region server");
252 }
253 }
254 if (services.shouldSplitMetaSeparately()) {
255 splitLog(serverNames, META_FILTER);
256 splitLog(serverNames, NON_META_FILTER);
257 } else {
258 splitAllLogs(serverNames);
259 }
260 retrySplitting = false;
261 } catch (IOException ioe) {
262 LOG.warn("Failed splitting of " + serverNames, ioe);
263 if (!checkFileSystem()) {
264 LOG.warn("Bad Filesystem, exiting");
265 Runtime.getRuntime().halt(1);
266 }
267 try {
268 if (retrySplitting) {
269 Thread.sleep(conf.getInt(
270 "hbase.hlog.split.failure.retry.interval", 30 * 1000));
271 }
272 } catch (InterruptedException e) {
273 LOG.warn("Interrupted, aborting since cannot return w/o splitting");
274 Thread.currentThread().interrupt();
275 retrySplitting = false;
276 Runtime.getRuntime().halt(1);
277 }
278 }
279 } while (retrySplitting);
280 }
281
282 public void splitLog(final ServerName serverName) throws IOException {
283 List<ServerName> serverNames = new ArrayList<ServerName>();
284 serverNames.add(serverName);
285 splitLog(serverNames);
286 }
287
288 public void splitAllLogs(final ServerName serverName) throws IOException {
289 List<ServerName> serverNames = new ArrayList<ServerName>();
290 serverNames.add(serverName);
291 splitAllLogs(serverNames);
292 }
293
294
295
296
297
298
299 public void splitMetaLog(final ServerName serverName) throws IOException {
300 long splitTime = 0, splitLogSize = 0;
301 List<ServerName> serverNames = new ArrayList<ServerName>();
302 serverNames.add(serverName);
303 List<Path> logDirs = getLogDirs(serverNames);
304 if (logDirs.isEmpty()) {
305 LOG.info("No meta logs to split");
306 return;
307 }
308 splitLogManager.handleDeadWorkers(serverNames);
309 splitTime = EnvironmentEdgeManager.currentTimeMillis();
310 splitLogSize = splitLogManager.splitLogDistributed(logDirs, META_FILTER);
311 splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
312 if (this.metrics != null) {
313 this.metrics.addSplit(splitTime, splitLogSize);
314 }
315 }
316
317 private List<Path> getLogDirs(final List<ServerName> serverNames) throws IOException {
318 List<Path> logDirs = new ArrayList<Path>();
319 for(ServerName serverName: serverNames){
320 Path logDir = new Path(this.rootdir,
321 HLog.getHLogDirectoryName(serverName.toString()));
322 Path splitDir = logDir.suffix(HLog.SPLITTING_EXT);
323
324 if (fs.exists(logDir)) {
325 if (!this.fs.rename(logDir, splitDir)) {
326 throw new IOException("Failed fs.rename for log split: " + logDir);
327 }
328 logDir = splitDir;
329 LOG.debug("Renamed region directory: " + splitDir);
330 } else if (!fs.exists(splitDir)) {
331 LOG.info("Log dir for server " + serverName + " does not exist");
332 continue;
333 }
334 logDirs.add(splitDir);
335 }
336 return logDirs;
337 }
338
339 public void splitLog(final List<ServerName> serverNames) throws IOException {
340 splitLog(serverNames, NON_META_FILTER);
341 }
342
343 public void splitAllLogs(final List<ServerName> serverNames) throws IOException {
344 splitLog(serverNames, null);
345 }
346
347
348
349
350
351
352
353
354 public void splitLog(final List<ServerName> serverNames, PathFilter filter) throws IOException {
355 long splitTime = 0, splitLogSize = 0;
356 List<Path> logDirs = getLogDirs(serverNames);
357
358 if (logDirs.isEmpty()) {
359 LOG.info("No logs to split");
360 return;
361 }
362
363 if (distributedLogSplitting) {
364 splitLogManager.handleDeadWorkers(serverNames);
365 splitTime = EnvironmentEdgeManager.currentTimeMillis();
366 splitLogSize = splitLogManager.splitLogDistributed(logDirs,filter);
367 splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
368 } else {
369 for(Path logDir: logDirs){
370
371
372 this.splitLogLock.lock();
373 try {
374 HLogSplitter splitter = HLogSplitter.createLogSplitter(
375 conf, rootdir, logDir, oldLogDir, this.fs);
376 try {
377
378 FSUtils.waitOnSafeMode(conf, conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 1000));
379 splitter.splitLog();
380 } catch (OrphanHLogAfterSplitException e) {
381 LOG.warn("Retrying splitting because of:", e);
382
383 splitter = HLogSplitter.createLogSplitter(conf, rootdir, logDir,
384 oldLogDir, this.fs);
385 splitter.splitLog();
386 }
387 splitTime = splitter.getTime();
388 splitLogSize = splitter.getSize();
389 } finally {
390 this.splitLogLock.unlock();
391 }
392 }
393 }
394
395 if (this.metrics != null) {
396 this.metrics.addSplit(splitTime, splitLogSize);
397 }
398 }
399
400
401
402
403
404
405
406
407
408
409 private Path checkRootDir(final Path rd, final Configuration c,
410 final FileSystem fs)
411 throws IOException {
412
413 FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
414 10 * 1000));
415
416 try {
417 if (!fs.exists(rd)) {
418 fs.mkdirs(rd);
419
420
421
422
423
424
425
426 FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
427 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
428 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
429 } else {
430 if (!fs.isDirectory(rd)) {
431 throw new IllegalArgumentException(rd.toString() + " is not a directory");
432 }
433
434 FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
435 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
436 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
437 }
438 } catch (IllegalArgumentException iae) {
439 LOG.fatal("Please fix invalid configuration for "
440 + HConstants.HBASE_DIR + " " + rd.toString(), iae);
441 throw iae;
442 }
443
444 if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
445 HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
446 FSUtils.setClusterId(fs, rd, UUID.randomUUID().toString(), c.getInt(
447 HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
448 }
449 clusterId = FSUtils.getClusterId(fs, rd);
450
451
452 if (!FSUtils.rootRegionExists(fs, rd)) {
453 bootstrap(rd, c);
454 }
455 createRootTableInfo(rd);
456 return rd;
457 }
458
459 private void createRootTableInfo(Path rd) throws IOException {
460
461 if (!FSTableDescriptors.isTableInfoExists(fs, rd,
462 Bytes.toString(HRegionInfo.ROOT_REGIONINFO.getTableName()))) {
463 FSTableDescriptors.createTableDescriptor(HTableDescriptor.ROOT_TABLEDESC, this.conf);
464 }
465 }
466
467
468
469
470
471 private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
472 throws IOException {
473
474 if (fs.exists(tmpdir)) {
475
476
477 for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
478 for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
479 HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
480 }
481 }
482 if (!fs.delete(tmpdir, true)) {
483 throw new IOException("Unable to clean the temp directory: " + tmpdir);
484 }
485 }
486
487
488 if (!fs.mkdirs(tmpdir)) {
489 throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
490 }
491 }
492
493 private static void bootstrap(final Path rd, final Configuration c)
494 throws IOException {
495 LOG.info("BOOTSTRAP: creating ROOT and first META regions");
496 try {
497
498
499
500
501 HRegionInfo rootHRI = new HRegionInfo(HRegionInfo.ROOT_REGIONINFO);
502 setInfoFamilyCachingForRoot(false);
503 HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
504 setInfoFamilyCachingForMeta(false);
505 HRegion root = HRegion.createHRegion(rootHRI, rd, c,
506 HTableDescriptor.ROOT_TABLEDESC);
507 HRegion meta = HRegion.createHRegion(metaHRI, rd, c,
508 HTableDescriptor.META_TABLEDESC);
509 setInfoFamilyCachingForRoot(true);
510 setInfoFamilyCachingForMeta(true);
511
512 HRegion.addRegionToMETA(root, meta);
513 root.close();
514 root.getLog().closeAndDelete();
515 meta.close();
516 meta.getLog().closeAndDelete();
517 } catch (IOException e) {
518 e = RemoteExceptionHandler.checkIOException(e);
519 LOG.error("bootstrap", e);
520 throw e;
521 }
522 }
523
524
525
526
527 public static void setInfoFamilyCachingForRoot(final boolean b) {
528 for (HColumnDescriptor hcd:
529 HTableDescriptor.ROOT_TABLEDESC.getColumnFamilies()) {
530 if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
531 hcd.setBlockCacheEnabled(b);
532 hcd.setInMemory(b);
533 }
534 }
535 }
536
537
538
539
540 public static void setInfoFamilyCachingForMeta(final boolean b) {
541 for (HColumnDescriptor hcd:
542 HTableDescriptor.META_TABLEDESC.getColumnFamilies()) {
543 if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
544 hcd.setBlockCacheEnabled(b);
545 hcd.setInMemory(b);
546 }
547 }
548 }
549
550
551 public void deleteRegion(HRegionInfo region) throws IOException {
552 HFileArchiver.archiveRegion(conf, fs, region);
553 }
554
555 public void deleteTable(byte[] tableName) throws IOException {
556 fs.delete(new Path(rootdir, Bytes.toString(tableName)), true);
557 }
558
559
560
561
562
563
564
565 public Path moveToTemp(final Path path) throws IOException {
566 Path tempPath = new Path(this.tempdir, path.getName());
567
568
569 if (!fs.exists(tempdir) && !fs.mkdirs(tempdir)) {
570 throw new IOException("HBase temp directory '" + tempdir + "' creation failure.");
571 }
572
573 if (!fs.rename(path, tempPath)) {
574 throw new IOException("Unable to move '" + path + "' to temp '" + tempPath + "'");
575 }
576
577 return tempPath;
578 }
579
580
581
582
583
584
585
586 public Path moveTableToTemp(byte[] tableName) throws IOException {
587 return moveToTemp(HTableDescriptor.getTableDir(this.rootdir, tableName));
588 }
589
590 public void updateRegionInfo(HRegionInfo region) {
591
592
593
594 }
595
596 public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
597 throws IOException {
598
599 Path tableDir = new Path(rootdir, region.getTableNameAsString());
600 HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
601
602
603 Path familyDir = new Path(tableDir,
604 new Path(region.getEncodedName(), Bytes.toString(familyName)));
605 if (fs.delete(familyDir, true) == false) {
606 throw new IOException("Could not delete family "
607 + Bytes.toString(familyName) + " from FileSystem for region "
608 + region.getRegionNameAsString() + "(" + region.getEncodedName()
609 + ")");
610 }
611 }
612
613 public void stop() {
614 if (splitLogManager != null) {
615 this.splitLogManager.stop();
616 }
617 }
618
619
620
621
622
623
624 public void createTableDescriptor(HTableDescriptor htableDescriptor)
625 throws IOException {
626 FSTableDescriptors.createTableDescriptor(htableDescriptor, conf);
627 }
628
629
630
631
632
633
634
635
636 public HTableDescriptor deleteColumn(byte[] tableName, byte[] familyName)
637 throws IOException {
638 LOG.info("DeleteColumn. Table = " + Bytes.toString(tableName)
639 + " family = " + Bytes.toString(familyName));
640 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
641 htd.removeFamily(familyName);
642 this.services.getTableDescriptors().add(htd);
643 return htd;
644 }
645
646
647
648
649
650
651
652
653 public HTableDescriptor modifyColumn(byte[] tableName, HColumnDescriptor hcd)
654 throws IOException {
655 LOG.info("AddModifyColumn. Table = " + Bytes.toString(tableName)
656 + " HCD = " + hcd.toString());
657
658 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
659 byte [] familyName = hcd.getName();
660 if(!htd.hasFamily(familyName)) {
661 throw new InvalidFamilyOperationException("Family '" +
662 Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
663 }
664 htd.addFamily(hcd);
665 this.services.getTableDescriptors().add(htd);
666 return htd;
667 }
668
669
670
671
672
673
674
675
676 public HTableDescriptor addColumn(byte[] tableName, HColumnDescriptor hcd)
677 throws IOException {
678 LOG.info("AddColumn. Table = " + Bytes.toString(tableName) + " HCD = " +
679 hcd.toString());
680 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
681 if (htd == null) {
682 throw new InvalidFamilyOperationException("Family '" +
683 hcd.getNameAsString() + "' cannot be modified as HTD is null");
684 }
685 htd.addFamily(hcd);
686 this.services.getTableDescriptors().add(htd);
687 return htd;
688 }
689 }