1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.backup.util;
20
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.HashMap;
25 import java.util.TreeMap;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FileStatus;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.FileUtil;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.HConstants;
35 import org.apache.hadoop.hbase.HTableDescriptor;
36 import org.apache.hadoop.hbase.TableName;
37 import org.apache.hadoop.hbase.backup.BackupRestoreServerFactory;
38 import org.apache.hadoop.hbase.backup.HBackupFileSystem;
39 import org.apache.hadoop.hbase.backup.IncrementalRestoreService;
40 import org.apache.hadoop.hbase.classification.InterfaceAudience;
41 import org.apache.hadoop.hbase.classification.InterfaceStability;
42 import org.apache.hadoop.hbase.client.Admin;
43 import org.apache.hadoop.hbase.client.Connection;
44 import org.apache.hadoop.hbase.client.ConnectionFactory;
45 import org.apache.hadoop.hbase.client.HBaseAdmin;
46 import org.apache.hadoop.hbase.io.HFileLink;
47 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
48 import org.apache.hadoop.hbase.io.hfile.HFile;
49 import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
50 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
51 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
52 import org.apache.hadoop.hbase.regionserver.HStore;
53 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
54 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
55 import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
56 import org.apache.hadoop.hbase.util.Bytes;
57
58
59
60
61 @InterfaceAudience.Private
62 @InterfaceStability.Evolving
63 public class RestoreServerUtil {
64
65 public static final Log LOG = LogFactory.getLog(RestoreServerUtil.class);
66
67 private final String[] ignoreDirs = { "recovered.edits" };
68
69 protected Configuration conf = null;
70
71 protected Path backupRootPath;
72
73 protected String backupId;
74
75 protected FileSystem fs;
76 private final String RESTORE_TMP_PATH = "/tmp";
77 private final Path restoreTmpPath;
78
79
80 private final HashMap<TableName, Path> snapshotMap = new HashMap<>();
81
82 public RestoreServerUtil(Configuration conf, final Path backupRootPath, final String backupId)
83 throws IOException {
84 this.conf = conf;
85 this.backupRootPath = backupRootPath;
86 this.backupId = backupId;
87 this.fs = backupRootPath.getFileSystem(conf);
88 this.restoreTmpPath = new Path(conf.get("hbase.fs.tmp.dir") != null?
89 conf.get("hbase.fs.tmp.dir"): RESTORE_TMP_PATH,
90 "restore");
91 }
92
93
94
95
96
97
98
99
100 Path getTableArchivePath(TableName tableName)
101 throws IOException {
102 Path baseDir = new Path(HBackupFileSystem.getTableBackupPath(tableName, backupRootPath,
103 backupId), HConstants.HFILE_ARCHIVE_DIRECTORY);
104 Path dataDir = new Path(baseDir, HConstants.BASE_NAMESPACE_DIR);
105 Path archivePath = new Path(dataDir, tableName.getNamespaceAsString());
106 Path tableArchivePath =
107 new Path(archivePath, tableName.getQualifierAsString());
108 if (!fs.exists(tableArchivePath) || !fs.getFileStatus(tableArchivePath).isDirectory()) {
109 LOG.debug("Folder tableArchivePath: " + tableArchivePath.toString() + " does not exists");
110 tableArchivePath = null;
111 }
112 return tableArchivePath;
113 }
114
115
116
117
118
119
120
121
122 ArrayList<Path> getRegionList(TableName tableName)
123 throws FileNotFoundException, IOException {
124 Path tableArchivePath = this.getTableArchivePath(tableName);
125 ArrayList<Path> regionDirList = new ArrayList<Path>();
126 FileStatus[] children = fs.listStatus(tableArchivePath);
127 for (FileStatus childStatus : children) {
128
129 Path child = childStatus.getPath();
130 regionDirList.add(child);
131 }
132 return regionDirList;
133 }
134
135
136
137
138
139
140
141
142
143
144 public void incrementalRestoreTable(Path[] logDirs,
145 TableName[] tableNames, TableName[] newTableNames) throws IOException {
146
147 if (tableNames.length != newTableNames.length) {
148 throw new IOException("Number of source tables and target tables does not match!");
149 }
150
151
152
153 try (Connection conn = ConnectionFactory.createConnection(conf);
154 Admin admin = conn.getAdmin()) {
155 for (TableName tableName : newTableNames) {
156 if (!admin.tableExists(tableName)) {
157 admin.close();
158 throw new IOException("HBase table " + tableName
159 + " does not exist. Create the table first, e.g. by restoring a full backup.");
160 }
161 }
162 IncrementalRestoreService restoreService =
163 BackupRestoreServerFactory.getIncrementalRestoreService(conf);
164
165 restoreService.run(logDirs, tableNames, newTableNames);
166 }
167 }
168
169 public void fullRestoreTable(Path tableBackupPath, TableName tableName, TableName newTableName,
170 boolean converted, boolean truncateIfExists) throws IOException {
171 restoreTableAndCreate(tableName, newTableName, tableBackupPath, converted, truncateIfExists);
172 }
173
174
175
176
177
178
179
180
181
182 static Path getTableSnapshotPath(Path backupRootPath, TableName tableName,
183 String backupId) {
184 return new Path(HBackupFileSystem.getTableBackupPath(tableName, backupRootPath, backupId),
185 HConstants.SNAPSHOT_DIR_NAME);
186 }
187
188
189
190
191
192
193
194
195
196
197
198 Path getTableInfoPath(TableName tableName)
199 throws FileNotFoundException, IOException {
200 Path tableSnapShotPath = getTableSnapshotPath(backupRootPath, tableName, backupId);
201 Path tableInfoPath = null;
202
203
204 FileStatus[] snapshots = fs.listStatus(tableSnapShotPath);
205 for (FileStatus snapshot : snapshots) {
206 tableInfoPath = snapshot.getPath();
207
208 if (tableInfoPath.getName().endsWith("data.manifest")) {
209 break;
210 }
211 }
212 return tableInfoPath;
213 }
214
215
216
217
218
219 HTableDescriptor getTableDesc(TableName tableName)
220 throws FileNotFoundException, IOException {
221 Path tableInfoPath = this.getTableInfoPath(tableName);
222 SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath);
223 SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc);
224 HTableDescriptor tableDescriptor = manifest.getTableDescriptor();
225 if (!tableDescriptor.getTableName().equals(tableName)) {
226 LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: "
227 + tableInfoPath.toString());
228 LOG.error("tableDescriptor.getNameAsString() = " + tableDescriptor.getNameAsString());
229 throw new FileNotFoundException("couldn't find Table Desc for table: " + tableName +
230 " under tableInfoPath: " + tableInfoPath.toString());
231 }
232 return tableDescriptor;
233 }
234
235
236
237
238
239
240
241
242
243 Path checkLocalAndBackup(Path tableArchivePath) throws IOException {
244
245 boolean isCopyNeeded = false;
246
247 FileSystem srcFs = tableArchivePath.getFileSystem(conf);
248 FileSystem desFs = FileSystem.get(conf);
249 if (tableArchivePath.getName().startsWith("/")) {
250 isCopyNeeded = true;
251 } else {
252
253
254 if (srcFs.getUri().equals(desFs.getUri())) {
255 LOG.debug("cluster hold the backup image: " + srcFs.getUri() + "; local cluster node: "
256 + desFs.getUri());
257 isCopyNeeded = true;
258 }
259 }
260 if (isCopyNeeded) {
261 LOG.debug("File " + tableArchivePath + " on local cluster, back it up before restore");
262 if (desFs.exists(restoreTmpPath)) {
263 try {
264 desFs.delete(restoreTmpPath, true);
265 } catch (IOException e) {
266 LOG.debug("Failed to delete path: " + restoreTmpPath
267 + ", need to check whether restore target DFS cluster is healthy");
268 }
269 }
270 FileUtil.copy(srcFs, tableArchivePath, desFs, restoreTmpPath, false, conf);
271 LOG.debug("Copied to temporary path on local cluster: " + restoreTmpPath);
272 tableArchivePath = restoreTmpPath;
273 }
274 return tableArchivePath;
275 }
276
277 private void restoreTableAndCreate(TableName tableName, TableName newTableName,
278 Path tableBackupPath, boolean converted, boolean truncateIfExists) throws IOException {
279 if (newTableName == null || newTableName.equals("")) {
280 newTableName = tableName;
281 }
282
283 FileSystem fileSys = tableBackupPath.getFileSystem(this.conf);
284
285
286 HTableDescriptor tableDescriptor = null;
287
288 Path tableSnapshotPath = getTableSnapshotPath(backupRootPath, tableName, backupId);
289
290 if (fileSys.exists(tableSnapshotPath)) {
291
292
293 if (snapshotMap.get(tableName) != null) {
294 SnapshotDescription desc =
295 SnapshotDescriptionUtils.readSnapshotInfo(fileSys, tableSnapshotPath);
296 SnapshotManifest manifest = SnapshotManifest.open(conf, fileSys, tableSnapshotPath, desc);
297 tableDescriptor = manifest.getTableDescriptor();
298 } else {
299 tableDescriptor = getTableDesc(tableName);
300 snapshotMap.put(tableName, getTableInfoPath(tableName));
301 }
302 if (tableDescriptor == null) {
303 LOG.debug("Found no table descriptor in the snapshot dir, previous schema would be lost");
304 }
305 } else if (converted) {
306
307 LOG.error("convert will be supported in a future jira");
308 }
309
310 Path tableArchivePath = getTableArchivePath(tableName);
311 if (tableArchivePath == null) {
312 if (tableDescriptor != null) {
313
314 if(LOG.isDebugEnabled()) {
315 LOG.debug("find table descriptor but no archive dir for table " + tableName
316 + ", will only create table");
317 }
318 tableDescriptor.setName(newTableName);
319 checkAndCreateTable(tableBackupPath, tableName, newTableName, null,
320 tableDescriptor, truncateIfExists);
321 return;
322 } else {
323 throw new IllegalStateException("Cannot restore hbase table because directory '"
324 + " tableArchivePath is null.");
325 }
326 }
327
328 if (tableDescriptor == null) {
329 tableDescriptor = new HTableDescriptor(newTableName);
330 } else {
331 tableDescriptor.setName(newTableName);
332 }
333
334 if (!converted) {
335
336
337 try {
338 ArrayList<Path> regionPathList = getRegionList(tableName);
339
340
341
342 checkAndCreateTable(tableBackupPath, tableName, newTableName, regionPathList,
343 tableDescriptor, truncateIfExists);
344 if (tableArchivePath != null) {
345
346
347 Path tempTableArchivePath = checkLocalAndBackup(tableArchivePath);
348 if (tempTableArchivePath.equals(tableArchivePath)) {
349 if(LOG.isDebugEnabled()) {
350 LOG.debug("TableArchivePath for bulkload using existPath: " + tableArchivePath);
351 }
352 } else {
353 regionPathList = getRegionList(tempTableArchivePath);
354 if(LOG.isDebugEnabled()) {
355 LOG.debug("TableArchivePath for bulkload using tempPath: " + tempTableArchivePath);
356 }
357 }
358
359 LoadIncrementalHFiles loader = createLoader(tempTableArchivePath, false);
360 for (Path regionPath : regionPathList) {
361 String regionName = regionPath.toString();
362 if(LOG.isDebugEnabled()) {
363 LOG.debug("Restoring HFiles from directory " + regionName);
364 }
365 String[] args = { regionName, newTableName.getNameAsString()};
366 loader.run(args);
367 }
368 }
369
370 } catch (Exception e) {
371 throw new IllegalStateException("Cannot restore hbase table", e);
372 }
373 } else {
374 LOG.debug("convert will be supported in a future jira");
375 }
376 }
377
378
379
380
381
382
383
384
385 ArrayList<Path> getRegionList(Path tableArchivePath) throws FileNotFoundException,
386 IOException {
387 ArrayList<Path> regionDirList = new ArrayList<Path>();
388 FileStatus[] children = fs.listStatus(tableArchivePath);
389 for (FileStatus childStatus : children) {
390
391 Path child = childStatus.getPath();
392 regionDirList.add(child);
393 }
394 return regionDirList;
395 }
396
397
398
399
400
401
402
403 int getNumberOfFilesInDir(Path regionPath) throws IOException {
404 int result = 0;
405
406 if (!fs.exists(regionPath) || !fs.getFileStatus(regionPath).isDirectory()) {
407 throw new IllegalStateException("Cannot restore hbase table because directory '"
408 + regionPath.toString() + "' is not a directory.");
409 }
410
411 FileStatus[] tableDirContent = fs.listStatus(regionPath);
412 for (FileStatus subDirStatus : tableDirContent) {
413 FileStatus[] colFamilies = fs.listStatus(subDirStatus.getPath());
414 for (FileStatus colFamilyStatus : colFamilies) {
415 FileStatus[] colFamilyContent = fs.listStatus(colFamilyStatus.getPath());
416 result += colFamilyContent.length;
417 }
418 }
419 return result;
420 }
421
422
423
424
425
426
427
428
429 int getMaxNumberOfFilesInSubDir(Path tableArchivePath) throws IOException {
430 int result = 1;
431 ArrayList<Path> regionPathList = getRegionList(tableArchivePath);
432
433
434 if (regionPathList == null || regionPathList.size() == 0) {
435 throw new IllegalStateException("Cannot restore hbase table because directory '"
436 + tableArchivePath + "' is not a directory.");
437 }
438
439 for (Path regionPath : regionPathList) {
440 result = Math.max(result, getNumberOfFilesInDir(regionPath));
441 }
442 return result;
443 }
444
445
446
447
448
449
450
451 private LoadIncrementalHFiles createLoader(Path tableArchivePath, boolean multipleTables)
452 throws IOException {
453
454
455
456
457 Integer milliSecInMin = 60000;
458 Integer previousMillis = this.conf.getInt("hbase.rpc.timeout", 0);
459 Integer numberOfFilesInDir =
460 multipleTables ? getMaxNumberOfFilesInSubDir(tableArchivePath) :
461 getNumberOfFilesInDir(tableArchivePath);
462 Integer calculatedMillis = numberOfFilesInDir * milliSecInMin;
463 Integer resultMillis = Math.max(calculatedMillis, previousMillis);
464 if (resultMillis > previousMillis) {
465 LOG.info("Setting configuration for restore with LoadIncrementalHFile: "
466 + "hbase.rpc.timeout to " + calculatedMillis / milliSecInMin
467 + " minutes, to handle the number of files in backup " + tableArchivePath);
468 this.conf.setInt("hbase.rpc.timeout", resultMillis);
469 }
470
471
472
473 this.conf.setInt(LoadIncrementalHFiles.MAX_FILES_PER_REGION_PER_FAMILY, Integer.MAX_VALUE);
474 LoadIncrementalHFiles loader = null;
475 try {
476 loader = new LoadIncrementalHFiles(this.conf);
477 } catch (Exception e1) {
478 throw new IOException(e1);
479 }
480 return loader;
481 }
482
483
484
485
486
487
488 byte[][] generateBoundaryKeys(ArrayList<Path> regionDirList)
489 throws FileNotFoundException, IOException {
490 TreeMap<byte[], Integer> map = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
491
492 byte[][] keys = null;
493
494 for (Path regionDir : regionDirList) {
495 LOG.debug("Parsing region dir: " + regionDir);
496 Path hfofDir = regionDir;
497
498 if (!fs.exists(hfofDir)) {
499 LOG.warn("HFileOutputFormat dir " + hfofDir + " not found");
500 }
501
502 FileStatus[] familyDirStatuses = fs.listStatus(hfofDir);
503 if (familyDirStatuses == null) {
504 throw new IOException("No families found in " + hfofDir);
505 }
506
507 for (FileStatus stat : familyDirStatuses) {
508 if (!stat.isDirectory()) {
509 LOG.warn("Skipping non-directory " + stat.getPath());
510 continue;
511 }
512 boolean isIgnore = false;
513 String pathName = stat.getPath().getName();
514 for (String ignore : ignoreDirs) {
515 if (pathName.contains(ignore)) {
516 LOG.warn("Skipping non-family directory" + pathName);
517 isIgnore = true;
518 break;
519 }
520 }
521 if (isIgnore) {
522 continue;
523 }
524 Path familyDir = stat.getPath();
525 LOG.debug("Parsing family dir [" + familyDir.toString() + " in region [" + regionDir + "]");
526
527 if (familyDir.getName().startsWith("_") || familyDir.getName().startsWith(".")) {
528 continue;
529 }
530
531
532 Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir));
533 for (Path hfile : hfiles) {
534 if (hfile.getName().startsWith("_") || hfile.getName().startsWith(".")
535 || StoreFileInfo.isReference(hfile.getName())
536 || HFileLink.isHFileLink(hfile.getName())) {
537 continue;
538 }
539 HFile.Reader reader = HFile.createReader(fs, hfile, new CacheConfig(conf), conf);
540 final byte[] first, last;
541 try {
542 reader.loadFileInfo();
543 first = reader.getFirstRowKey();
544 last = reader.getLastRowKey();
545 LOG.debug("Trying to figure out region boundaries hfile=" + hfile + " first="
546 + Bytes.toStringBinary(first) + " last=" + Bytes.toStringBinary(last));
547
548
549 Integer value = map.containsKey(first) ? (Integer) map.get(first) : 0;
550 map.put(first, value + 1);
551 value = map.containsKey(last) ? (Integer) map.get(last) : 0;
552 map.put(last, value - 1);
553 } finally {
554 reader.close();
555 }
556 }
557 }
558 }
559 keys = LoadIncrementalHFiles.inferBoundaries(map);
560 return keys;
561 }
562
563
564
565
566
567
568
569
570
571
572
573 private void checkAndCreateTable(Path tableBackupPath, TableName tableName,
574 TableName targetTableName, ArrayList<Path> regionDirList,
575 HTableDescriptor htd, boolean truncateIfExists)
576 throws IOException {
577 HBaseAdmin hbadmin = null;
578 Connection conn = null;
579 try {
580 conn = ConnectionFactory.createConnection(conf);
581 hbadmin = (HBaseAdmin) conn.getAdmin();
582 boolean createNew = false;
583 if (hbadmin.tableExists(targetTableName)) {
584 if(truncateIfExists) {
585 LOG.info("Truncating exising target table '" + targetTableName +
586 "', preserving region splits");
587 hbadmin.disableTable(targetTableName);
588 hbadmin.truncateTable(targetTableName, true);
589 } else{
590 LOG.info("Using exising target table '" + targetTableName + "'");
591 }
592 } else {
593 createNew = true;
594 }
595 if(createNew){
596 LOG.info("Creating target table '" + targetTableName + "'");
597
598 if (regionDirList == null || regionDirList.size() == 0) {
599 hbadmin.createTable(htd);
600 return;
601 }
602 byte[][] keys = generateBoundaryKeys(regionDirList);
603
604 hbadmin.createTable(htd, keys);
605 }
606 } catch (Exception e) {
607 throw new IOException(e);
608 } finally {
609 if (hbadmin != null) {
610 hbadmin.close();
611 }
612 if(conn != null){
613 conn.close();
614 }
615 }
616 }
617
618 }