View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.backup.impl;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.Arrays;
24  import java.util.HashMap;
25  import java.util.Iterator;
26  import java.util.List;
27  import java.util.Map.Entry;
28  import java.util.TreeSet;
29  
30  import org.apache.commons.lang.StringUtils;
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.TableName;
36  import org.apache.hadoop.hbase.backup.BackupType;
37  import org.apache.hadoop.hbase.backup.HBackupFileSystem;
38  import org.apache.hadoop.hbase.backup.RestoreClient;
39  import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage;
40  import org.apache.hadoop.hbase.backup.util.BackupClientUtil;
41  import org.apache.hadoop.hbase.backup.util.RestoreServerUtil;
42  import org.apache.hadoop.hbase.classification.InterfaceAudience;
43  import org.apache.hadoop.hbase.classification.InterfaceStability;
44  import org.apache.hadoop.hbase.client.Admin;
45  import org.apache.hadoop.hbase.client.Connection;
46  import org.apache.hadoop.hbase.client.ConnectionFactory;
47  
48  /**
49   * The main class which interprets the given arguments and trigger restore operation.
50   */
51  @InterfaceAudience.Public
52  @InterfaceStability.Evolving
53  public final class RestoreClientImpl implements RestoreClient {
54  
55    private static final Log LOG = LogFactory.getLog(RestoreClientImpl.class);
56    private Configuration conf;
57  
58    public RestoreClientImpl() {
59    }
60  
61    @Override
62    public void setConf(Configuration conf) {
63      this.conf = conf;
64    }
65  
66    
67    
68    /**
69     * Restore operation. Stage 1: validate backupManifest, and check target tables
70     * @param backupRootDir The root dir for backup image
71     * @param backupId The backup id for image to be restored
72     * @param check True if only do dependency check
73     * @param sTableArray The array of tables to be restored
74     * @param tTableArray The array of mapping tables to restore to
75     * @param isOverwrite True then do restore overwrite if target table exists, otherwise fail the
76     *          request if target table exists
77     * @throws IOException if any failure during restore
78     */
79    @Override
80    public void restore(String backupRootDir,
81        String backupId, boolean check, TableName[] sTableArray,
82        TableName[] tTableArray, boolean isOverwrite) throws IOException {
83  
84      HashMap<TableName, BackupManifest> backupManifestMap = new HashMap<>();
85      // check and load backup image manifest for the tables
86      Path rootPath = new Path(backupRootDir);
87      HBackupFileSystem.checkImageManifestExist(backupManifestMap, sTableArray, conf, rootPath,
88        backupId);
89      try {
90        // Check and validate the backup image and its dependencies
91        if (check) {
92          if (validate(backupManifestMap)) {
93            LOG.info("Checking backup images: ok");
94          } else {
95            String errMsg = "Some dependencies are missing for restore";
96            LOG.error(errMsg);
97            throw new IOException(errMsg);
98          }
99        }
100 
101       if (tTableArray == null) {
102         tTableArray = sTableArray;
103       }
104       // check the target tables
105       checkTargetTables(tTableArray, isOverwrite);
106       // start restore process      
107       restoreStage(backupManifestMap, sTableArray, tTableArray, isOverwrite);
108       LOG.info("Restore for " + Arrays.asList(sTableArray) + " are successful!");
109     } catch (IOException e) {
110       LOG.error("ERROR: restore failed with error: " + e.getMessage());
111       throw e;
112     }
113 
114   }
115 
116   private  boolean validate(HashMap<TableName, BackupManifest> backupManifestMap)
117       throws IOException {
118     boolean isValid = true;
119 
120     for (Entry<TableName, BackupManifest> manifestEntry : backupManifestMap.entrySet()) {
121       TableName table = manifestEntry.getKey();
122       TreeSet<BackupImage> imageSet = new TreeSet<BackupImage>();
123 
124       ArrayList<BackupImage> depList = manifestEntry.getValue().getDependentListByTable(table);
125       if (depList != null && !depList.isEmpty()) {
126         imageSet.addAll(depList);
127       }
128 
129       LOG.info("Dependent image(s) from old to new:");
130       for (BackupImage image : imageSet) {
131         String imageDir =
132             HBackupFileSystem.getTableBackupDir(image.getRootDir(), image.getBackupId(), table);
133         if (!BackupClientUtil.checkPathExist(imageDir, conf)) {
134           LOG.error("ERROR: backup image does not exist: " + imageDir);
135           isValid = false;
136           break;
137         }
138         // TODO More validation?
139         LOG.info("Backup image: " + image.getBackupId() + " for '" + table + "' is available");
140       }
141     }
142 
143     return isValid;
144   }
145 
146   /**
147    * Validate target Tables
148    * @param tTableArray: target tables
149    * @param isOverwrite overwrite existing table
150    * @throws IOException exception
151    */
152   private  void checkTargetTables(TableName[] tTableArray, boolean isOverwrite)
153       throws IOException {
154     ArrayList<TableName> existTableList = new ArrayList<>();
155     ArrayList<TableName> disabledTableList = new ArrayList<>();
156 
157     // check if the tables already exist
158     try(Connection conn = ConnectionFactory.createConnection(conf);
159         Admin admin = conn.getAdmin()) {
160       for (TableName tableName : tTableArray) {
161         if (admin.tableExists(tableName)) {
162           existTableList.add(tableName);
163           if (admin.isTableDisabled(tableName)) {
164             disabledTableList.add(tableName);
165           }
166         } else {
167           LOG.info("HBase table " + tableName
168               + " does not exist. It will be created during restore process");
169         }
170       }
171     }
172 
173     if (existTableList.size() > 0) {
174       if (!isOverwrite) {
175         LOG.error("Existing table found in the restore target, please add \"-overwrite\" "
176             + "option in the command if you mean to restore to these existing tables");
177         LOG.info("Existing table list in restore target: " + existTableList);
178         throw new IOException("Existing table found in target while no \"-overwrite\" "
179             + "option found");
180       } else {
181         if (disabledTableList.size() > 0) {
182           LOG.error("Found offline table in the restore target, "
183               + "please enable them before restore with \"-overwrite\" option");
184           LOG.info("Offline table list in restore target: " + disabledTableList);
185           throw new IOException(
186               "Found offline table in the target when restore with \"-overwrite\" option");
187         }
188       }
189     }
190   }
191 
192   /**
193    * Restore operation. Stage 2: resolved Backup Image dependency
194    * @param backupManifestMap : tableName,  Manifest
195    * @param sTableArray The array of tables to be restored
196    * @param tTableArray The array of mapping tables to restore to
197    * @return set of BackupImages restored
198    * @throws IOException exception
199    */
200   private void restoreStage(HashMap<TableName, BackupManifest> backupManifestMap,
201       TableName[] sTableArray, TableName[] tTableArray, boolean isOverwrite) throws IOException {
202     TreeSet<BackupImage> restoreImageSet = new TreeSet<BackupImage>();
203     boolean truncateIfExists = isOverwrite;
204     try {
205       for (int i = 0; i < sTableArray.length; i++) {
206         TableName table = sTableArray[i];
207         BackupManifest manifest = backupManifestMap.get(table);
208         // Get the image list of this backup for restore in time order from old
209         // to new.
210         List<BackupImage> list = new ArrayList<BackupImage>();
211         list.add(manifest.getBackupImage());
212         List<BackupImage> depList = manifest.getDependentListByTable(table);
213         list.addAll(depList);
214         TreeSet<BackupImage> restoreList = new TreeSet<BackupImage>(list);
215         LOG.debug("need to clear merged Image. to be implemented in future jira");
216         restoreImages(restoreList.iterator(), table, tTableArray[i], truncateIfExists);
217         restoreImageSet.addAll(restoreList);
218 
219         if (restoreImageSet != null && !restoreImageSet.isEmpty()) {
220           LOG.info("Restore includes the following image(s):");
221           for (BackupImage image : restoreImageSet) {
222             LOG.info("Backup: "
223                 + image.getBackupId()
224                 + " "
225                 + HBackupFileSystem.getTableBackupDir(image.getRootDir(), image.getBackupId(),
226                   table));
227           }
228         }
229       }
230     } catch (Exception e) {
231       LOG.error("Failed", e);
232       throw new IOException(e);
233     }
234     LOG.debug("restoreStage finished");
235 
236   }
237 
238   /**
239    * Restore operation handle each backupImage in iterator
240    * @param it: backupImage iterator - ascending
241    * @param sTable: table to be restored
242    * @param tTable: table to be restored to
243    * @throws IOException exception
244    */
245   private void restoreImages(Iterator<BackupImage> it, TableName sTable, 
246       TableName tTable, boolean truncateIfExists)
247       throws IOException {
248 
249     // First image MUST be image of a FULL backup
250     BackupImage image = it.next();
251 
252     String rootDir = image.getRootDir();
253     String backupId = image.getBackupId();
254     Path backupRoot = new Path(rootDir);
255     
256     // We need hFS only for full restore (see the code)
257     RestoreServerUtil restoreTool = new RestoreServerUtil(conf, backupRoot, backupId);
258     BackupManifest manifest = HBackupFileSystem.getManifest(sTable, conf, backupRoot, backupId);
259 
260     Path tableBackupPath = HBackupFileSystem.getTableBackupPath(sTable, backupRoot, backupId);
261 
262     // TODO: convert feature will be provided in a future JIRA
263     boolean converted = false;
264 
265     if (manifest.getType() == BackupType.FULL || converted) {
266       LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from "
267           + (converted ? "converted" : "full") + " backup image " + tableBackupPath.toString());
268       restoreTool.fullRestoreTable(tableBackupPath, sTable, tTable, 
269         converted, truncateIfExists);
270       
271     } else { // incremental Backup
272       throw new IOException("Unexpected backup type " + image.getType());
273     }
274 
275     // The rest one are incremental
276     if (it.hasNext()) {
277       List<String> logDirList = new ArrayList<String>();
278       while (it.hasNext()) {
279         BackupImage im = it.next();
280         String logBackupDir = HBackupFileSystem.getLogBackupDir(im.getRootDir(), im.getBackupId());
281         logDirList.add(logBackupDir);
282       }
283       String logDirs = StringUtils.join(logDirList, ",");
284       LOG.info("Restoring '" + sTable + "' to '" + tTable
285           + "' from log dirs: " + logDirs);
286       String[] sarr = new String[logDirList.size()];
287       logDirList.toArray(sarr);
288       Path[] paths = org.apache.hadoop.util.StringUtils.stringToPath(sarr);
289       restoreTool.incrementalRestoreTable(paths, new TableName[] { sTable },
290         new TableName[] { tTable });
291     }
292     LOG.info(sTable + " has been successfully restored to " + tTable);
293   }
294   
295 }