1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.backup.util;
20
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.net.URLDecoder;
24 import java.util.ArrayList;
25 import java.util.Collections;
26 import java.util.HashMap;
27 import java.util.Iterator;
28 import java.util.List;
29 import java.util.TreeMap;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FileStatus;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.LocatedFileStatus;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.fs.PathFilter;
39 import org.apache.hadoop.fs.RemoteIterator;
40 import org.apache.hadoop.hbase.HConstants;
41 import org.apache.hadoop.hbase.ServerName;
42 import org.apache.hadoop.hbase.TableName;
43 import org.apache.hadoop.hbase.backup.BackupInfo;
44 import org.apache.hadoop.hbase.backup.impl.BackupRestoreConstants;
45 import org.apache.hadoop.hbase.classification.InterfaceAudience;
46 import org.apache.hadoop.hbase.classification.InterfaceStability;
47
48
49
50
51 @InterfaceAudience.Private
52 @InterfaceStability.Evolving
53 public final class BackupClientUtil {
54 protected static final Log LOG = LogFactory.getLog(BackupClientUtil.class);
55 public static final String LOGNAME_SEPARATOR = ".";
56
57 private BackupClientUtil(){
58 throw new AssertionError("Instantiating utility class...");
59 }
60
61
62
63
64
65
66
67
68 public static boolean checkPathExist(String backupStr, Configuration conf)
69 throws IOException {
70 boolean isExist = false;
71 Path backupPath = new Path(backupStr);
72 FileSystem fileSys = backupPath.getFileSystem(conf);
73 String targetFsScheme = fileSys.getUri().getScheme();
74 if (LOG.isTraceEnabled()) {
75 LOG.trace("Schema of given url: " + backupStr + " is: " + targetFsScheme);
76 }
77 if (fileSys.exists(backupPath)) {
78 isExist = true;
79 }
80 return isExist;
81 }
82
83
84 public static void checkTargetDir(String backupRootPath, Configuration conf) throws IOException {
85 boolean targetExists = false;
86 try {
87 targetExists = checkPathExist(backupRootPath, conf);
88 } catch (IOException e) {
89 String expMsg = e.getMessage();
90 String newMsg = null;
91 if (expMsg.contains("No FileSystem for scheme")) {
92 newMsg =
93 "Unsupported filesystem scheme found in the backup target url. Error Message: "
94 + newMsg;
95 LOG.error(newMsg);
96 throw new IOException(newMsg);
97 } else {
98 throw e;
99 }
100 }
101
102 if (targetExists) {
103 LOG.info("Using existing backup root dir: " + backupRootPath);
104 } else {
105 LOG.info("Backup root dir " + backupRootPath + " does not exist. Will be created.");
106 }
107 }
108
109
110
111
112
113
114 public static <T> Long getMinValue(HashMap<T, Long> map) {
115 Long minTimestamp = null;
116 if (map != null) {
117 ArrayList<Long> timestampList = new ArrayList<Long>(map.values());
118 Collections.sort(timestampList);
119
120 minTimestamp = timestampList.get(0);
121 }
122 return minTimestamp;
123 }
124
125
126
127
128
129
130
131 public static String parseHostFromOldLog(Path p) throws IOException {
132 try{
133 String n = p.getName();
134 int idx = n.lastIndexOf(LOGNAME_SEPARATOR);
135 String s = URLDecoder.decode(n.substring(0, idx), "UTF8");
136 return ServerName.parseHostname(s) + ":" + ServerName.parsePort(s);
137 } catch(Exception e){
138 LOG.error("Failed to parse "+ p, e);
139 return null;
140 }
141 }
142
143
144
145
146
147
148
149 public static Long getCreationTime(Path p) throws IOException {
150 int idx = p.getName().lastIndexOf(LOGNAME_SEPARATOR);
151 if (idx < 0) {
152 throw new IOException("Cannot parse timestamp from path " + p);
153 }
154 String ts = p.getName().substring(idx + 1);
155 return Long.parseLong(ts);
156 }
157
158 public static List<String> getFiles(FileSystem fs, Path rootDir, List<String> files,
159 PathFilter filter) throws FileNotFoundException, IOException {
160 RemoteIterator<LocatedFileStatus> it = fs.listFiles(rootDir, true);
161
162 while (it.hasNext()) {
163 LocatedFileStatus lfs = it.next();
164 if (lfs.isDirectory()) {
165 continue;
166 }
167
168 if (filter.accept(lfs.getPath())) {
169 files.add(lfs.getPath().toString());
170 }
171 }
172 return files;
173 }
174
175 public static void cleanupBackupData(BackupInfo context, Configuration conf)
176 throws IOException
177 {
178 cleanupHLogDir(context, conf);
179 cleanupTargetDir(context, conf);
180 }
181
182
183
184
185
186 private static void cleanupHLogDir(BackupInfo backupContext, Configuration conf)
187 throws IOException {
188
189 String logDir = backupContext.getHLogTargetDir();
190 if (logDir == null) {
191 LOG.warn("No log directory specified for " + backupContext.getBackupId());
192 return;
193 }
194
195 Path rootPath = new Path(logDir).getParent();
196 FileSystem fs = FileSystem.get(rootPath.toUri(), conf);
197 FileStatus[] files = listStatus(fs, rootPath, null);
198 if (files == null) {
199 return;
200 }
201 for (FileStatus file : files) {
202 LOG.debug("Delete log files: " + file.getPath().getName());
203 fs.delete(file.getPath(), true);
204 }
205 }
206
207
208
209
210 private static void cleanupTargetDir(BackupInfo backupContext, Configuration conf) {
211 try {
212
213 LOG.debug("Trying to cleanup up target dir : " + backupContext.getBackupId());
214 String targetDir = backupContext.getTargetRootDir();
215 if (targetDir == null) {
216 LOG.warn("No target directory specified for " + backupContext.getBackupId());
217 return;
218 }
219
220 FileSystem outputFs =
221 FileSystem.get(new Path(backupContext.getTargetRootDir()).toUri(), conf);
222
223 for (TableName table : backupContext.getTables()) {
224 Path targetDirPath =
225 new Path(getTableBackupDir(backupContext.getTargetRootDir(),
226 backupContext.getBackupId(), table));
227 if (outputFs.delete(targetDirPath, true)) {
228 LOG.info("Cleaning up backup data at " + targetDirPath.toString() + " done.");
229 } else {
230 LOG.info("No data has been found in " + targetDirPath.toString() + ".");
231 }
232
233 Path tableDir = targetDirPath.getParent();
234 FileStatus[] backups = listStatus(outputFs, tableDir, null);
235 if (backups == null || backups.length == 0) {
236 outputFs.delete(tableDir, true);
237 LOG.debug(tableDir.toString() + " is empty, remove it.");
238 }
239 }
240 outputFs.delete(new Path(targetDir, backupContext.getBackupId()), true);
241 } catch (IOException e1) {
242 LOG.error("Cleaning up backup data of " + backupContext.getBackupId() + " at "
243 + backupContext.getTargetRootDir() + " failed due to " + e1.getMessage() + ".");
244 }
245 }
246
247
248
249
250
251
252
253
254
255
256 public static String getTableBackupDir(String backupRootDir, String backupId,
257 TableName tableName) {
258 return backupRootDir + Path.SEPARATOR+ backupId + Path.SEPARATOR +
259 tableName.getNamespaceAsString() + Path.SEPARATOR
260 + tableName.getQualifierAsString() + Path.SEPARATOR ;
261 }
262
263 public static TableName[] parseTableNames(String tables) {
264 if (tables == null) {
265 return null;
266 }
267 String[] tableArray = tables.split(BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
268
269 TableName[] ret = new TableName[tableArray.length];
270 for (int i = 0; i < tableArray.length; i++) {
271 ret[i] = TableName.valueOf(tableArray[i]);
272 }
273 return ret;
274 }
275
276
277
278
279
280
281 public static ArrayList<BackupInfo> sortHistoryListDesc(
282 ArrayList<BackupInfo> historyList) {
283 ArrayList<BackupInfo> list = new ArrayList<BackupInfo>();
284 TreeMap<String, BackupInfo> map = new TreeMap<String, BackupInfo>();
285 for (BackupInfo h : historyList) {
286 map.put(Long.toString(h.getStartTs()), h);
287 }
288 Iterator<String> i = map.descendingKeySet().iterator();
289 while (i.hasNext()) {
290 list.add(map.get(i.next()));
291 }
292 return list;
293 }
294
295
296
297
298
299
300
301
302 public static String getUniqueWALFileNamePart(String walFileName) throws IOException {
303 return getUniqueWALFileNamePart(new Path(walFileName));
304 }
305
306
307
308
309
310
311
312 public static String getUniqueWALFileNamePart(Path p) throws IOException {
313 return p.getName();
314 }
315
316
317
318
319
320
321
322
323
324
325
326
327 public static FileStatus [] listStatus(final FileSystem fs,
328 final Path dir, final PathFilter filter) throws IOException {
329 FileStatus [] status = null;
330 try {
331 status = filter == null ? fs.listStatus(dir) : fs.listStatus(dir, filter);
332 } catch (FileNotFoundException fnfe) {
333
334 if (LOG.isTraceEnabled()) {
335 LOG.trace(dir + " doesn't exist");
336 }
337 }
338 if (status == null || status.length < 1) return null;
339 return status;
340 }
341
342
343
344
345
346
347
348
349
350
351
352 public static String getPath(Path p) {
353 return p.toUri().getPath();
354 }
355
356
357
358
359
360
361
362
363 public static String getLogBackupDir(String backupRootDir, String backupId) {
364 return backupRootDir + Path.SEPARATOR + backupId+ Path.SEPARATOR
365 + HConstants.HREGION_LOGDIR_NAME;
366 }
367
368 }