View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.backup.impl;
20  
21  import java.io.Closeable;
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.HashMap;
25  import java.util.Iterator;
26  import java.util.List;
27  import java.util.Set;
28  import java.util.concurrent.ExecutorService;
29  import java.util.concurrent.LinkedBlockingQueue;
30  import java.util.concurrent.ThreadPoolExecutor;
31  import java.util.concurrent.TimeUnit;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.HTableDescriptor;
39  import org.apache.hadoop.hbase.TableName;
40  import org.apache.hadoop.hbase.backup.BackupInfo;
41  import org.apache.hadoop.hbase.backup.BackupType;
42  import org.apache.hadoop.hbase.backup.HBackupFileSystem;
43  import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
44  import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage;
45  import org.apache.hadoop.hbase.backup.master.BackupController;
46  import org.apache.hadoop.hbase.backup.master.BackupLogCleaner;
47  import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
48  import org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager;
49  import org.apache.hadoop.hbase.classification.InterfaceAudience;
50  import org.apache.hadoop.hbase.classification.InterfaceStability;
51  import org.apache.hadoop.hbase.client.Admin;
52  import org.apache.hadoop.hbase.client.Connection;
53  import org.apache.hadoop.hbase.client.ConnectionFactory;
54  
55  import com.google.common.util.concurrent.ThreadFactoryBuilder;
56  
57  /**
58   * Handles backup requests on server-side, creates backup context records in hbase:backup
59   * to keep track backup. The timestamps kept in hbase:backup table will be used for future
60   * incremental backup. Creates BackupContext and DispatchRequest.
61   */
62  @InterfaceAudience.Private
63  @InterfaceStability.Evolving
64  public class BackupManager implements Closeable {
65    private static final Log LOG = LogFactory.getLog(BackupManager.class);
66  
67    private Configuration conf = null;
68    private BackupInfo backupInfo = null;
69  
70    private ExecutorService pool = null;
71  
72    private boolean backupComplete = false;
73  
74    private BackupSystemTable systemTable;
75  
76    private final Connection conn;
77  
78    /**
79     * Backup manager constructor.
80     * @param conf configuration
81     * @throws IOException exception
82     */
83    public BackupManager(Configuration conf) throws IOException {
84      if (!conf.getBoolean(HConstants.BACKUP_ENABLE_KEY, HConstants.BACKUP_ENABLE_DEFAULT)) {
85        throw new BackupException("HBase backup is not enabled. Check your " +
86            HConstants.BACKUP_ENABLE_KEY + " setting.");
87      }
88      this.conf = conf;
89      this.conn = ConnectionFactory.createConnection(conf);
90      this.systemTable = new BackupSystemTable(conn);
91       
92    }
93  
94    /**
95     * Return backup context
96     */
97    protected BackupInfo getBackupContext()
98    {
99      return backupInfo;
100   }
101   /**
102    * This method modifies the master's configuration in order to inject backup-related features
103    * @param conf configuration
104    */
105   public static void decorateMasterConfiguration(Configuration conf) {
106     if (!isBackupEnabled(conf)) {
107       return;
108     }
109     // Add WAL archive cleaner plug-in
110     String plugins = conf.get(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
111     String cleanerClass = BackupLogCleaner.class.getCanonicalName();
112     if (!plugins.contains(cleanerClass)) {
113       conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, plugins + "," + cleanerClass);
114     }    
115     
116     String classes = conf.get("hbase.procedure.master.classes");
117     String masterProcedureClass = LogRollMasterProcedureManager.class.getName();
118     if(classes == null){    
119       conf.set("hbase.procedure.master.classes", masterProcedureClass);
120     } else if(!classes.contains(masterProcedureClass)){
121       conf.set("hbase.procedure.master.classes", classes +","+masterProcedureClass);
122     }    
123  
124     // Set Master Observer - Backup Controller
125     classes = conf.get("hbase.coprocessor.master.classes");
126     String observerClass = BackupController.class.getName();
127     if(classes == null){    
128       conf.set("hbase.coprocessor.master.classes", observerClass);
129     } else if(!classes.contains(observerClass)){
130       conf.set("hbase.coprocessor.master.classes", classes +","+observerClass);
131     }    
132 
133     if (LOG.isDebugEnabled()) {
134       LOG.debug("Added log cleaner: " + cleanerClass);
135       LOG.debug("Added master procedure manager: "+masterProcedureClass);
136       LOG.debug("Added master observer: "+observerClass);      
137     }
138     
139   }
140 
141   /**
142    * This method modifies the RS configuration in order to inject backup-related features
143    * @param conf configuration
144    */
145   public static void decorateRSConfiguration(Configuration conf) {
146     if (!isBackupEnabled(conf)) {
147       return;
148     }
149     
150     String classes = conf.get("hbase.procedure.regionserver.classes");
151     String regionProcedureClass = LogRollRegionServerProcedureManager.class.getName();
152     if(classes == null){    
153       conf.set("hbase.procedure.regionserver.classes", regionProcedureClass);
154     } else if(!classes.contains(regionProcedureClass)){
155       conf.set("hbase.procedure.regionserver.classes", classes +","+regionProcedureClass);
156     }    
157     if (LOG.isDebugEnabled()) {
158       LOG.debug("Added region procedure manager: "+regionProcedureClass);
159     }
160     
161   }
162   
163   
164   private static boolean isBackupEnabled(Configuration conf) {
165     return conf.getBoolean(HConstants.BACKUP_ENABLE_KEY, HConstants.BACKUP_ENABLE_DEFAULT);
166   }
167 
168   /**
169    * Get configuration
170    * @return configuration
171    */
172   Configuration getConf() {
173     return conf;
174   }
175 
176   /**
177    * Stop all the work of backup.
178    */
179   @Override
180   public void close() {
181     // currently, we shutdown now for all ongoing back handlers, we may need to do something like
182     // record the failed list somewhere later
183     if (this.pool != null) {
184       this.pool.shutdownNow();
185     }
186     if (systemTable != null) {
187       try {
188         systemTable.close();
189       } catch (Exception e) {
190         LOG.error(e);
191       }
192     }
193     if (conn != null) {
194       try {
195         conn.close();
196       } catch (IOException e) {
197         LOG.error(e);
198       }
199     }
200   }
201 
202   /**
203    * Create a BackupContext based on input backup request.
204    * @param backupId backup id
205    * @param type    type
206    * @param tablelist table list
207    * @param targetRootDir root dir
208    * @param snapshot snapshot name
209    * @return BackupContext context
210    * @throws BackupException exception
211    */
212   public BackupInfo createBackupInfo(String backupId, BackupType type,
213       List<TableName> tableList, String targetRootDir, int workers, long bandwidth)
214           throws BackupException {
215     if (targetRootDir == null) {
216       throw new BackupException("Wrong backup request parameter: target backup root directory");
217     }
218 
219     if (type == BackupType.FULL && (tableList == null || tableList.isEmpty())) {
220       // If table list is null for full backup, which means backup all tables. Then fill the table
221       // list with all user tables from meta. It no table available, throw the request exception.
222 
223       HTableDescriptor[] htds = null;
224       try (Admin hbadmin = conn.getAdmin()) {
225         htds = hbadmin.listTables();
226       } catch (Exception e) {
227         throw new BackupException(e);
228       }
229 
230       if (htds == null) {
231         throw new BackupException("No table exists for full backup of all tables.");
232       } else {
233         tableList = new ArrayList<>();
234         for (HTableDescriptor hTableDescriptor : htds) {
235           tableList.add(hTableDescriptor.getTableName());
236         }
237 
238         LOG.info("Full backup all the tables available in the cluster: " + tableList);
239       }
240     }
241 
242     // there are one or more tables in the table list
243     backupInfo = new BackupInfo(backupId, type, 
244       tableList.toArray(new TableName[tableList.size()]),
245       targetRootDir);
246     backupInfo.setBandwidth(bandwidth);
247     backupInfo.setWorkers(workers);
248     return backupInfo;
249   }
250 
251   /**
252    * Check if any ongoing backup. Currently, we only reply on checking status in hbase:backup. We
253    * need to consider to handle the case of orphan records in the future. Otherwise, all the coming
254    * request will fail.
255    * @return the ongoing backup id if on going backup exists, otherwise null
256    * @throws IOException exception
257    */
258   private String getOngoingBackupId() throws IOException {
259 
260     ArrayList<BackupInfo> sessions = systemTable.getBackupContexts(BackupState.RUNNING);
261     if (sessions.size() == 0) {
262       return null;
263     }
264     return sessions.get(0).getBackupId();
265   }
266 
267   /**
268    * Start the backup manager service.
269    * @throws IOException exception
270    */
271   public void initialize() throws IOException {
272     String ongoingBackupId = this.getOngoingBackupId();
273     if (ongoingBackupId != null) {
274       LOG.info("There is a ongoing backup " + ongoingBackupId
275           + ". Can not launch new backup until no ongoing backup remains.");
276       throw new BackupException("There is ongoing backup.");
277     }
278 
279     // Initialize thread pools
280     int nrThreads = this.conf.getInt("hbase.backup.threads.max", 1);
281     ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
282     builder.setNameFormat("BackupHandler-%1$d");
283     this.pool =
284         new ThreadPoolExecutor(nrThreads, nrThreads, 60, TimeUnit.SECONDS,
285             new LinkedBlockingQueue<Runnable>(), builder.build());
286     ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true);
287   }
288 
289   public void setBackupInfo(BackupInfo backupInfo) {
290     this.backupInfo = backupInfo;
291   }
292 
293   /**
294    * Get direct ancestors of the current backup.
295    * @param backupCtx The backup context for the current backup
296    * @return The ancestors for the current backup
297    * @throws IOException exception
298    * @throws BackupException exception
299    */
300   public ArrayList<BackupImage> getAncestors(BackupInfo backupCtx) throws IOException,
301       BackupException {
302     LOG.debug("Getting the direct ancestors of the current backup "+ 
303       backupCtx.getBackupId());
304 
305     ArrayList<BackupImage> ancestors = new ArrayList<BackupImage>();
306 
307     // full backup does not have ancestor
308     if (backupCtx.getType() == BackupType.FULL) {
309       LOG.debug("Current backup is a full backup, no direct ancestor for it.");
310       return ancestors;
311     }
312 
313     // get all backup history list in descending order
314 
315     ArrayList<BackupInfo> allHistoryList = getBackupHistory(true);
316     for (BackupInfo backup : allHistoryList) {
317       BackupImage image =
318           new BackupImage(backup.getBackupId(), backup.getType(),
319             backup.getTargetRootDir(),
320               backup.getTableNames(), backup.getStartTs(), backup
321                   .getEndTs());
322       // add the full backup image as an ancestor until the last incremental backup
323       if (backup.getType().equals(BackupType.FULL)) {
324         // check the backup image coverage, if previous image could be covered by the newer ones,
325         // then no need to add
326         if (!BackupManifest.canCoverImage(ancestors, image)) {
327           ancestors.add(image);
328         }
329       } else {
330         // found last incremental backup, if previously added full backup ancestor images can cover
331         // it, then this incremental ancestor is not the dependent of the current incremental
332         // backup, that is to say, this is the backup scope boundary of current table set.
333         // Otherwise, this incremental backup ancestor is the dependent ancestor of the ongoing
334         // incremental backup
335         if (BackupManifest.canCoverImage(ancestors, image)) {
336           LOG.debug("Met the backup boundary of the current table set. "
337               + "The root full backup images for the current backup scope:");
338           for (BackupImage image1 : ancestors) {
339             LOG.debug("  BackupId: " + image1.getBackupId() + ", Backup directory: "
340                 + image1.getRootDir());
341           }
342         } else {
343           Path logBackupPath =
344               HBackupFileSystem.getLogBackupPath(backup.getTargetRootDir(),
345                 backup.getBackupId());
346           LOG.debug("Current backup has an incremental backup ancestor, "
347               + "touching its image manifest in " + logBackupPath.toString()
348               + " to construct the dependency.");
349           BackupManifest lastIncrImgManifest = new BackupManifest(conf, logBackupPath);
350           BackupImage lastIncrImage = lastIncrImgManifest.getBackupImage();
351           ancestors.add(lastIncrImage);
352 
353           LOG.debug("Last dependent incremental backup image information:");
354           LOG.debug("  Token: " + lastIncrImage.getBackupId());
355           LOG.debug("  Backup directory: " + lastIncrImage.getRootDir());
356         }
357       }
358     }
359     LOG.debug("Got " + ancestors.size() + " ancestors for the current backup.");
360     return ancestors;
361   }
362 
363   /**
364    * Get the direct ancestors of this backup for one table involved.
365    * @param backupContext backup context
366    * @param table table
367    * @return backupImages on the dependency list
368    * @throws BackupException exception
369    * @throws IOException exception
370    */
371   public ArrayList<BackupImage> getAncestors(BackupInfo backupContext, TableName table)
372       throws BackupException, IOException {
373     ArrayList<BackupImage> ancestors = getAncestors(backupContext);
374     ArrayList<BackupImage> tableAncestors = new ArrayList<BackupImage>();
375     for (BackupImage image : ancestors) {
376       if (image.hasTable(table)) {
377         tableAncestors.add(image);
378         if (image.getType() == BackupType.FULL) {
379           break;
380         }
381       }
382     }
383     return tableAncestors;
384   }
385 
386   /*
387    * hbase:backup operations
388    */
389 
390   /**
391    * Updates status (state) of a backup session in a persistent store
392    * @param context context
393    * @throws IOException exception
394    */
395   public void updateBackupInfo(BackupInfo context) throws IOException {
396     systemTable.updateBackupInfo(context);
397   }
398 
399   /**
400    * Read the last backup start code (timestamp) of last successful backup. Will return null
401    * if there is no startcode stored in hbase:backup or the value is of length 0. These two
402    * cases indicate there is no successful backup completed so far.
403    * @return the timestamp of a last successful backup
404    * @throws IOException exception
405    */
406   public String readBackupStartCode() throws IOException {
407     return systemTable.readBackupStartCode(backupInfo.getTargetRootDir());
408   }
409 
410   /**
411    * Write the start code (timestamp) to hbase:backup. If passed in null, then write 0 byte.
412    * @param startCode start code
413    * @throws IOException exception
414    */
415   public void writeBackupStartCode(Long startCode) throws IOException {
416     systemTable.writeBackupStartCode(startCode, backupInfo.getTargetRootDir());
417   }
418 
419   /**
420    * Get the RS log information after the last log roll from hbase:backup.
421    * @return RS log info
422    * @throws IOException exception
423    */
424   public HashMap<String, Long> readRegionServerLastLogRollResult() throws IOException {
425     return systemTable.readRegionServerLastLogRollResult(backupInfo.getTargetRootDir());
426   }
427 
428   /**
429    * Get all completed backup information (in desc order by time)
430    * @return history info of BackupCompleteData
431    * @throws IOException exception
432    */
433   public ArrayList<BackupInfo> getBackupHistory() throws IOException {
434     return systemTable.getBackupHistory();
435   }
436 
437   public ArrayList<BackupInfo> getBackupHistory(boolean completed) throws IOException {
438     return systemTable.getBackupHistory(completed);
439   }
440   /**
441    * Write the current timestamps for each regionserver to hbase:backup after a successful full or
442    * incremental backup. Each table may have a different set of log timestamps. The saved timestamp
443    * is of the last log file that was backed up already.
444    * @param tables tables
445    * @throws IOException exception
446    */
447   public void writeRegionServerLogTimestamp(Set<TableName> tables,
448       HashMap<String, Long> newTimestamps) throws IOException {
449     systemTable.writeRegionServerLogTimestamp(tables, newTimestamps, 
450       backupInfo.getTargetRootDir());
451   }
452 
453   /**
454    * Read the timestamp for each region server log after the last successful backup. Each table has
455    * its own set of the timestamps.
456    * @return the timestamp for each region server. key: tableName value:
457    *         RegionServer,PreviousTimeStamp
458    * @throws IOException exception
459    */
460   public HashMap<TableName, HashMap<String, Long>> readLogTimestampMap() throws IOException {
461     return systemTable.readLogTimestampMap(backupInfo.getTargetRootDir());
462   }
463 
464   /**
465    * Return the current tables covered by incremental backup.
466    * @return set of tableNames
467    * @throws IOException exception
468    */
469   public Set<TableName> getIncrementalBackupTableSet() throws IOException {
470     return systemTable.getIncrementalBackupTableSet(backupInfo.getTargetRootDir());
471   }
472 
473   /**
474    * Adds set of tables to overall incremental backup table set
475    * @param tables tables
476    * @throws IOException exception
477    */
478   public void addIncrementalBackupTableSet(Set<TableName> tables) throws IOException {
479     systemTable.addIncrementalBackupTableSet(tables, backupInfo.getTargetRootDir());
480   }
481 
482   /**
483    * Saves list of WAL files after incremental backup operation. These files will be stored until
484    * TTL expiration and are used by Backup Log Cleaner plugin to determine which WAL files can be
485    * safely purged.
486    */
487   public void recordWALFiles(List<String> files) throws IOException {
488     systemTable.addWALFiles(files, 
489       backupInfo.getBackupId(), backupInfo.getTargetRootDir());
490   }
491 
492   /**
493    * Get WAL files iterator
494    * @return WAL files iterator from hbase:backup
495    * @throws IOException
496    */
497   public Iterator<BackupSystemTable.WALItem> getWALFilesFromBackupSystem() throws IOException {
498     return  systemTable.getWALFilesIterator(backupInfo.getTargetRootDir());
499   }
500 
501   public Connection getConnection() {
502     return conn;
503   }
504 }