1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.backup.impl;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Arrays;
24 import java.util.HashMap;
25 import java.util.Iterator;
26 import java.util.List;
27 import java.util.Map.Entry;
28 import java.util.TreeSet;
29
30 import org.apache.commons.lang.StringUtils;
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.TableName;
36 import org.apache.hadoop.hbase.backup.BackupType;
37 import org.apache.hadoop.hbase.backup.HBackupFileSystem;
38 import org.apache.hadoop.hbase.backup.RestoreClient;
39 import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage;
40 import org.apache.hadoop.hbase.backup.util.BackupClientUtil;
41 import org.apache.hadoop.hbase.backup.util.RestoreServerUtil;
42 import org.apache.hadoop.hbase.classification.InterfaceAudience;
43 import org.apache.hadoop.hbase.classification.InterfaceStability;
44 import org.apache.hadoop.hbase.client.Admin;
45 import org.apache.hadoop.hbase.client.Connection;
46 import org.apache.hadoop.hbase.client.ConnectionFactory;
47
48
49
50
51 @InterfaceAudience.Public
52 @InterfaceStability.Evolving
53 public final class RestoreClientImpl implements RestoreClient {
54
55 private static final Log LOG = LogFactory.getLog(RestoreClientImpl.class);
56 private Configuration conf;
57
58 public RestoreClientImpl() {
59 }
60
61 @Override
62 public void setConf(Configuration conf) {
63 this.conf = conf;
64 }
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79 @Override
80 public void restore(String backupRootDir,
81 String backupId, boolean check, TableName[] sTableArray,
82 TableName[] tTableArray, boolean isOverwrite) throws IOException {
83
84 HashMap<TableName, BackupManifest> backupManifestMap = new HashMap<>();
85
86 Path rootPath = new Path(backupRootDir);
87 HBackupFileSystem.checkImageManifestExist(backupManifestMap, sTableArray, conf, rootPath,
88 backupId);
89 try {
90
91 if (check) {
92 if (validate(backupManifestMap)) {
93 LOG.info("Checking backup images: ok");
94 } else {
95 String errMsg = "Some dependencies are missing for restore";
96 LOG.error(errMsg);
97 throw new IOException(errMsg);
98 }
99 }
100
101 if (tTableArray == null) {
102 tTableArray = sTableArray;
103 }
104
105 checkTargetTables(tTableArray, isOverwrite);
106
107 restoreStage(backupManifestMap, sTableArray, tTableArray, isOverwrite);
108 LOG.info("Restore for " + Arrays.asList(sTableArray) + " are successful!");
109 } catch (IOException e) {
110 LOG.error("ERROR: restore failed with error: " + e.getMessage());
111 throw e;
112 }
113
114 }
115
116 private boolean validate(HashMap<TableName, BackupManifest> backupManifestMap)
117 throws IOException {
118 boolean isValid = true;
119
120 for (Entry<TableName, BackupManifest> manifestEntry : backupManifestMap.entrySet()) {
121 TableName table = manifestEntry.getKey();
122 TreeSet<BackupImage> imageSet = new TreeSet<BackupImage>();
123
124 ArrayList<BackupImage> depList = manifestEntry.getValue().getDependentListByTable(table);
125 if (depList != null && !depList.isEmpty()) {
126 imageSet.addAll(depList);
127 }
128
129 LOG.info("Dependent image(s) from old to new:");
130 for (BackupImage image : imageSet) {
131 String imageDir =
132 HBackupFileSystem.getTableBackupDir(image.getRootDir(), image.getBackupId(), table);
133 if (!BackupClientUtil.checkPathExist(imageDir, conf)) {
134 LOG.error("ERROR: backup image does not exist: " + imageDir);
135 isValid = false;
136 break;
137 }
138
139 LOG.info("Backup image: " + image.getBackupId() + " for '" + table + "' is available");
140 }
141 }
142
143 return isValid;
144 }
145
146
147
148
149
150
151
152 private void checkTargetTables(TableName[] tTableArray, boolean isOverwrite)
153 throws IOException {
154 ArrayList<TableName> existTableList = new ArrayList<>();
155 ArrayList<TableName> disabledTableList = new ArrayList<>();
156
157
158 try(Connection conn = ConnectionFactory.createConnection(conf);
159 Admin admin = conn.getAdmin()) {
160 for (TableName tableName : tTableArray) {
161 if (admin.tableExists(tableName)) {
162 existTableList.add(tableName);
163 if (admin.isTableDisabled(tableName)) {
164 disabledTableList.add(tableName);
165 }
166 } else {
167 LOG.info("HBase table " + tableName
168 + " does not exist. It will be created during restore process");
169 }
170 }
171 }
172
173 if (existTableList.size() > 0) {
174 if (!isOverwrite) {
175 LOG.error("Existing table found in the restore target, please add \"-overwrite\" "
176 + "option in the command if you mean to restore to these existing tables");
177 LOG.info("Existing table list in restore target: " + existTableList);
178 throw new IOException("Existing table found in target while no \"-overwrite\" "
179 + "option found");
180 } else {
181 if (disabledTableList.size() > 0) {
182 LOG.error("Found offline table in the restore target, "
183 + "please enable them before restore with \"-overwrite\" option");
184 LOG.info("Offline table list in restore target: " + disabledTableList);
185 throw new IOException(
186 "Found offline table in the target when restore with \"-overwrite\" option");
187 }
188 }
189 }
190 }
191
192
193
194
195
196
197
198
199
200 private void restoreStage(HashMap<TableName, BackupManifest> backupManifestMap,
201 TableName[] sTableArray, TableName[] tTableArray, boolean isOverwrite) throws IOException {
202 TreeSet<BackupImage> restoreImageSet = new TreeSet<BackupImage>();
203 boolean truncateIfExists = isOverwrite;
204 try {
205 for (int i = 0; i < sTableArray.length; i++) {
206 TableName table = sTableArray[i];
207 BackupManifest manifest = backupManifestMap.get(table);
208
209
210 List<BackupImage> list = new ArrayList<BackupImage>();
211 list.add(manifest.getBackupImage());
212 List<BackupImage> depList = manifest.getDependentListByTable(table);
213 list.addAll(depList);
214 TreeSet<BackupImage> restoreList = new TreeSet<BackupImage>(list);
215 LOG.debug("need to clear merged Image. to be implemented in future jira");
216 restoreImages(restoreList.iterator(), table, tTableArray[i], truncateIfExists);
217 restoreImageSet.addAll(restoreList);
218
219 if (restoreImageSet != null && !restoreImageSet.isEmpty()) {
220 LOG.info("Restore includes the following image(s):");
221 for (BackupImage image : restoreImageSet) {
222 LOG.info("Backup: "
223 + image.getBackupId()
224 + " "
225 + HBackupFileSystem.getTableBackupDir(image.getRootDir(), image.getBackupId(),
226 table));
227 }
228 }
229 }
230 } catch (Exception e) {
231 LOG.error("Failed", e);
232 throw new IOException(e);
233 }
234 LOG.debug("restoreStage finished");
235
236 }
237
238
239
240
241
242
243
244
245 private void restoreImages(Iterator<BackupImage> it, TableName sTable,
246 TableName tTable, boolean truncateIfExists)
247 throws IOException {
248
249
250 BackupImage image = it.next();
251
252 String rootDir = image.getRootDir();
253 String backupId = image.getBackupId();
254 Path backupRoot = new Path(rootDir);
255
256
257 RestoreServerUtil restoreTool = new RestoreServerUtil(conf, backupRoot, backupId);
258 BackupManifest manifest = HBackupFileSystem.getManifest(sTable, conf, backupRoot, backupId);
259
260 Path tableBackupPath = HBackupFileSystem.getTableBackupPath(sTable, backupRoot, backupId);
261
262
263 boolean converted = false;
264
265 if (manifest.getType() == BackupType.FULL || converted) {
266 LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from "
267 + (converted ? "converted" : "full") + " backup image " + tableBackupPath.toString());
268 restoreTool.fullRestoreTable(tableBackupPath, sTable, tTable,
269 converted, truncateIfExists);
270
271 } else {
272 throw new IOException("Unexpected backup type " + image.getType());
273 }
274
275
276 if (it.hasNext()) {
277 List<String> logDirList = new ArrayList<String>();
278 while (it.hasNext()) {
279 BackupImage im = it.next();
280 String logBackupDir = HBackupFileSystem.getLogBackupDir(im.getRootDir(), im.getBackupId());
281 logDirList.add(logBackupDir);
282 }
283 String logDirs = StringUtils.join(logDirList, ",");
284 LOG.info("Restoring '" + sTable + "' to '" + tTable
285 + "' from log dirs: " + logDirs);
286 String[] sarr = new String[logDirList.size()];
287 logDirList.toArray(sarr);
288 Path[] paths = org.apache.hadoop.util.StringUtils.stringToPath(sarr);
289 restoreTool.incrementalRestoreTable(paths, new TableName[] { sTable },
290 new TableName[] { tTable });
291 }
292 LOG.info(sTable + " has been successfully restored to " + tTable);
293 }
294
295 }