View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.backup.impl;
19  
20  import java.io.IOException;
21  import java.util.ArrayList;
22  import java.util.Arrays;
23  import java.util.List;
24  import java.util.Set;
25  
26  import org.apache.commons.lang.StringUtils;
27  import org.apache.hadoop.hbase.Cell;
28  import org.apache.hadoop.hbase.CellUtil;
29  import org.apache.hadoop.hbase.TableName;
30  import org.apache.hadoop.hbase.backup.BackupInfo;
31  import org.apache.hadoop.hbase.backup.util.BackupClientUtil;
32  import org.apache.hadoop.hbase.classification.InterfaceAudience;
33  import org.apache.hadoop.hbase.classification.InterfaceStability;
34  import org.apache.hadoop.hbase.client.Delete;
35  import org.apache.hadoop.hbase.client.Get;
36  import org.apache.hadoop.hbase.client.Put;
37  import org.apache.hadoop.hbase.client.Result;
38  import org.apache.hadoop.hbase.client.Scan;
39  import org.apache.hadoop.hbase.util.Bytes;
40  
41  
42  /**
43   * A collection for methods used by BackupSystemTable.
44   */
45  
46  @InterfaceAudience.Private
47  @InterfaceStability.Evolving
48  public final class BackupSystemTableHelper {
49  
50    /**
51     * hbase:backup schema:
52     * 1. Backup sessions rowkey= "session:" + backupId; value = serialized
53     * BackupContext
54     * 2. Backup start code rowkey = "startcode:" + backupRoot; value = startcode
55     * 3. Incremental backup set rowkey="incrbackupset:" + backupRoot; value=[list of tables]
56     * 4. Table-RS-timestamp map rowkey="trslm:"+ backupRoot+table_name; value = map[RS-> 
57     * last WAL timestamp]
58     * 5. RS - WAL ts map rowkey="rslogts:"+backupRoot +server; value = last WAL timestamp
59     * 6. WALs recorded rowkey="wals:"+WAL unique file name; value = backupId and full WAL file name
60     */
61  
62    private final static String BACKUP_INFO_PREFIX = "session:";
63    private final static String START_CODE_ROW = "startcode:";
64    private final static String INCR_BACKUP_SET = "incrbackupset:";
65    private final static String TABLE_RS_LOG_MAP_PREFIX = "trslm:";
66    private final static String RS_LOG_TS_PREFIX = "rslogts:";
67    private final static String WALS_PREFIX = "wals:";
68    private final static String SET_KEY_PREFIX = "backupset:";
69  
70    private final static byte[] EMPTY_VALUE = new byte[] {};
71    
72    // Safe delimiter in a string
73    private final static String NULL = "\u0000";
74  
75    private BackupSystemTableHelper() {
76      throw new AssertionError("Instantiating utility class...");
77    }
78  
79    /**
80     * Creates Put operation for a given backup context object
81     * @param context backup context
82     * @return put operation
83     * @throws IOException exception
84     */
85    static Put createPutForBackupContext(BackupInfo context) throws IOException {
86      Put put = new Put(rowkey(BACKUP_INFO_PREFIX, context.getBackupId()));
87      put.addColumn(BackupSystemTable.SESSIONS_FAMILY, "context".getBytes(), context.toByteArray());
88      return put;
89    }
90  
91    /**
92     * Creates Get operation for a given backup id
93     * @param backupId - backup's ID
94     * @return get operation
95     * @throws IOException exception
96     */
97    static Get createGetForBackupContext(String backupId) throws IOException {
98      Get get = new Get(rowkey(BACKUP_INFO_PREFIX, backupId));
99      get.addFamily(BackupSystemTable.SESSIONS_FAMILY);
100     get.setMaxVersions(1);
101     return get;
102   }
103 
104   /**
105    * Creates Delete operation for a given backup id
106    * @param backupId - backup's ID
107    * @return delete operation
108    * @throws IOException exception
109    */
110   public static Delete createDeleteForBackupInfo(String backupId) {
111     Delete del = new Delete(rowkey(BACKUP_INFO_PREFIX, backupId));
112     del.addFamily(BackupSystemTable.SESSIONS_FAMILY);
113     return del;
114   }
115 
116   /**
117    * Converts Result to BackupContext
118    * @param res - HBase result
119    * @return backup context instance
120    * @throws IOException exception
121    */
122   static BackupInfo resultToBackupInfo(Result res) throws IOException {
123     res.advance();
124     Cell cell = res.current();
125     return cellToBackupInfo(cell);
126   }
127 
128   /**
129    * Creates Get operation to retrieve start code from hbase:backup
130    * @return get operation
131    * @throws IOException exception
132    */
133   static Get createGetForStartCode(String rootPath) throws IOException {    
134     Get get = new Get(rowkey(START_CODE_ROW, rootPath));
135     get.addFamily(BackupSystemTable.META_FAMILY);
136     get.setMaxVersions(1);
137     return get;
138   }
139 
140   /**
141    * Creates Put operation to store start code to hbase:backup
142    * @return put operation
143    * @throws IOException exception
144    */
145   static Put createPutForStartCode(String startCode, String rootPath) {
146     Put put = new Put(rowkey(START_CODE_ROW, rootPath));
147     put.addColumn(BackupSystemTable.META_FAMILY, "startcode".getBytes(), startCode.getBytes());
148     return put;
149   }
150 
151   /**
152    * Creates Get to retrieve incremental backup table set from hbase:backup
153    * @return get operation
154    * @throws IOException exception
155    */
156   static Get createGetForIncrBackupTableSet(String backupRoot) throws IOException {
157     Get get = new Get(rowkey(INCR_BACKUP_SET, backupRoot));
158     get.addFamily(BackupSystemTable.META_FAMILY);
159     get.setMaxVersions(1);
160     return get;
161   }
162 
163   /**
164    * Creates Put to store incremental backup table set
165    * @param tables tables
166    * @return put operation
167    */
168   static Put createPutForIncrBackupTableSet(Set<TableName> tables, String backupRoot) {
169     Put put = new Put(rowkey(INCR_BACKUP_SET, backupRoot));
170     for (TableName table : tables) {
171       put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes(table.getNameAsString()),
172         EMPTY_VALUE);
173     }
174     return put;
175   }
176 
177   /**
178    * Creates Scan operation to load backup history
179    * @return scan operation
180    */
181   static Scan createScanForBackupHistory() {
182     Scan scan = new Scan();
183     byte[] startRow = BACKUP_INFO_PREFIX.getBytes();
184     byte[] stopRow = Arrays.copyOf(startRow, startRow.length);
185     stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1);
186     scan.setStartRow(startRow);
187     scan.setStopRow(stopRow);
188     scan.addFamily(BackupSystemTable.SESSIONS_FAMILY);
189     scan.setMaxVersions(1);
190     return scan;
191   }
192 
193   /**
194    * Converts cell to backup context instance.
195    * @param current - cell
196    * @return backup context instance
197    * @throws IOException exception
198    */
199   static BackupInfo cellToBackupInfo(Cell current) throws IOException {
200     byte[] data = CellUtil.cloneValue(current);
201     return BackupInfo.fromByteArray(data);
202   }
203 
204   /**
205    * Creates Put to write RS last roll log timestamp map
206    * @param table - table
207    * @param smap - map, containing RS:ts
208    * @return put operation
209    */
210   static Put createPutForWriteRegionServerLogTimestamp(TableName table, byte[] smap, 
211       String backupRoot) {    
212     Put put = new Put(rowkey(TABLE_RS_LOG_MAP_PREFIX, backupRoot, NULL, table.getNameAsString()));
213     put.addColumn(BackupSystemTable.META_FAMILY, "log-roll-map".getBytes(), smap);
214     return put;
215   }
216 
217   /**
218    * Creates Scan to load table-> { RS -> ts} map of maps
219    * @return scan operation
220    */
221   static Scan createScanForReadLogTimestampMap(String backupRoot) {
222     Scan scan = new Scan();
223     byte[] startRow = rowkey(TABLE_RS_LOG_MAP_PREFIX, backupRoot);
224     byte[] stopRow = Arrays.copyOf(startRow, startRow.length);
225     stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1);
226     scan.setStartRow(startRow);
227     scan.setStopRow(stopRow);
228     scan.addFamily(BackupSystemTable.META_FAMILY);
229 
230     return scan;
231   }
232 
233   /**
234    * Get table name from rowkey
235    * @param cloneRow rowkey
236    * @return table name
237    */
238   static String getTableNameForReadLogTimestampMap(byte[] cloneRow) {
239     String s = new String(cloneRow);
240     int index = s.lastIndexOf(NULL); 
241     return s.substring(index +1);
242   }
243 
244   /**
245    * Creates Put to store RS last log result
246    * @param server - server name
247    * @param timestamp - log roll result (timestamp)
248    * @return put operation
249    */
250   static Put createPutForRegionServerLastLogRollResult(String server, 
251       Long timestamp, String backupRoot ) {
252     Put put = new Put(rowkey(RS_LOG_TS_PREFIX, backupRoot, NULL, server));
253     put.addColumn(BackupSystemTable.META_FAMILY, "rs-log-ts".getBytes(), 
254       timestamp.toString().getBytes());
255     return put;
256   }
257 
258   /**
259    * Creates Scan operation to load last RS log roll results
260    * @return scan operation
261    */
262   static Scan createScanForReadRegionServerLastLogRollResult(String backupRoot) {
263     Scan scan = new Scan();
264     byte[] startRow = rowkey(RS_LOG_TS_PREFIX, backupRoot);
265     byte[] stopRow = Arrays.copyOf(startRow, startRow.length);
266     stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1);
267     scan.setStartRow(startRow);
268     scan.setStopRow(stopRow);
269     scan.addFamily(BackupSystemTable.META_FAMILY);
270     scan.setMaxVersions(1);
271 
272     return scan;
273   }
274 
275   /**
276    * Get server's name from rowkey
277    * @param row - rowkey
278    * @return server's name
279    */
280   static String getServerNameForReadRegionServerLastLogRollResult(byte[] row) {
281     String s = new String(row);
282     int index = s.lastIndexOf(NULL);
283     return s.substring(index +1);
284   }
285 
286   /**
287    * Creates put list for list of WAL files
288    * @param files list of WAL file paths
289    * @param backupId backup id
290    * @return put list
291    * @throws IOException exception
292    */
293   public static List<Put> createPutsForAddWALFiles(List<String> files, 
294     String backupId, String backupRoot)
295       throws IOException {
296 
297     List<Put> puts = new ArrayList<Put>();
298     for (String file : files) {
299       Put put = new Put(rowkey(WALS_PREFIX, BackupClientUtil.getUniqueWALFileNamePart(file)));
300       put.addColumn(BackupSystemTable.META_FAMILY, "backupId".getBytes(), backupId.getBytes());
301       put.addColumn(BackupSystemTable.META_FAMILY, "file".getBytes(), file.getBytes());
302       put.addColumn(BackupSystemTable.META_FAMILY, "root".getBytes(), backupRoot.getBytes());
303       puts.add(put);
304     }
305     return puts;
306   }
307 
308   /**
309    * Creates Scan operation to load WALs
310    * TODO: support for backupRoot
311    * @param backupRoot - path to backup destination 
312    * @return scan operation
313    */
314   public static Scan createScanForGetWALs(String backupRoot) {
315     Scan scan = new Scan();
316     byte[] startRow = WALS_PREFIX.getBytes();
317     byte[] stopRow = Arrays.copyOf(startRow, startRow.length);
318     stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1);
319     scan.setStartRow(startRow);
320     scan.setStopRow(stopRow);
321     scan.addFamily(BackupSystemTable.META_FAMILY);
322     return scan;
323   }
324   /**
325    * Creates Get operation for a given wal file name
326    * TODO: support for backup destination
327    * @param file file
328    * @return get operation
329    * @throws IOException exception
330    */
331   public static Get createGetForCheckWALFile(String file) throws IOException {
332     Get get = new Get(rowkey(WALS_PREFIX, BackupClientUtil.getUniqueWALFileNamePart(file)));
333     // add backup root column
334     get.addFamily(BackupSystemTable.META_FAMILY);
335     return get;
336   }
337 
338   
339  /**
340   * Creates Scan operation to load backup set list
341   * @return scan operation
342   */
343  static Scan createScanForBackupSetList() {
344    Scan scan = new Scan();
345    byte[] startRow = SET_KEY_PREFIX.getBytes();
346    byte[] stopRow = Arrays.copyOf(startRow, startRow.length);
347    stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1);
348    scan.setStartRow(startRow);
349    scan.setStopRow(stopRow);
350    scan.addFamily(BackupSystemTable.META_FAMILY);
351    return scan;
352  }
353 
354  /**
355   * Creates Get operation to load backup set content
356   * @return get operation
357   */
358  static Get createGetForBackupSet(String name) {    
359    Get get  = new Get(rowkey(SET_KEY_PREFIX, name));
360    get.addFamily(BackupSystemTable.META_FAMILY);
361    return get;
362  }
363  
364  /**
365   * Creates Delete operation to delete backup set content
366   * @param name - backup set's name
367   * @return delete operation
368   */
369  static Delete createDeleteForBackupSet(String name) {    
370    Delete del  = new Delete(rowkey(SET_KEY_PREFIX, name));
371    del.addFamily(BackupSystemTable.META_FAMILY);
372    return del;
373  }
374  
375  
376  /**
377   * Creates Put operation to update backup set content
378   * @param name - backup set's name
379   * @param tables - list of tables
380   * @return put operation
381   */
382  static Put createPutForBackupSet(String name, String[] tables) {    
383    Put put  = new Put(rowkey(SET_KEY_PREFIX, name));
384    byte[] value = convertToByteArray(tables);
385    put.addColumn(BackupSystemTable.META_FAMILY, "tables".getBytes(), value);
386    return put;
387  }
388  
389  private static byte[] convertToByteArray(String[] tables) {
390    return StringUtils.join(tables, ",").getBytes();
391  }
392 
393  
394  /**
395   * Converts cell to backup set list.
396   * @param current - cell
397   * @return backup set 
398   * @throws IOException
399   */
400  static  String[] cellValueToBackupSet(Cell current) throws IOException {
401    byte[] data = CellUtil.cloneValue(current);
402    if( data != null && data.length > 0){
403      return new String(data).split(",");
404    } else{
405      return new String[0];
406    }
407  }
408 
409  /**
410   * Converts cell key to backup set name.
411   * @param current - cell
412   * @return backup set name
413   * @throws IOException
414   */
415  static  String cellKeyToBackupSetName(Cell current) throws IOException {
416    byte[] data = CellUtil.cloneRow(current);    
417    return new String(data).substring(SET_KEY_PREFIX.length());    
418  }
419  
420  static byte[] rowkey(String s, String ... other){
421    StringBuilder sb = new StringBuilder(s);
422    for(String ss: other){
423      sb.append(ss);
424    }
425    return sb.toString().getBytes();   
426  }
427  
428 }