1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.wal;
20
21 import java.io.Closeable;
22 import java.io.DataInput;
23 import java.io.DataOutput;
24 import java.io.IOException;
25 import java.util.List;
26 import java.util.regex.Pattern;
27
28 import com.google.common.annotations.VisibleForTesting;
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.hbase.classification.InterfaceAudience;
32 import org.apache.hadoop.hbase.classification.InterfaceStability;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FSDataInputStream;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.ServerName;
39 import org.apache.hadoop.hbase.util.FSUtils;
40
41
42 import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
43 import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter;
44 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
45
46
47
48
49
50
51
52
53
54
55
56
57 @InterfaceAudience.Private
58 @InterfaceStability.Evolving
59 public class DefaultWALProvider implements WALProvider {
60 private static final Log LOG = LogFactory.getLog(DefaultWALProvider.class);
61
62
63 public interface Reader extends WAL.Reader {
64
65
66
67
68
69
70 void init(FileSystem fs, Path path, Configuration c, FSDataInputStream s) throws IOException;
71 }
72
73
74 public interface Writer extends WALProvider.Writer {
75 void init(FileSystem fs, Path path, Configuration c, boolean overwritable) throws IOException;
76 }
77
78 protected FSHLog log = null;
79
80
81
82
83
84
85
86
87 @Override
88 public void init(final WALFactory factory, final Configuration conf,
89 final List<WALActionsListener> listeners, String providerId) throws IOException {
90 if (null != log) {
91 throw new IllegalStateException("WALProvider.init should only be called once.");
92 }
93 if (null == providerId) {
94 providerId = DEFAULT_PROVIDER_ID;
95 }
96 final String logPrefix = factory.factoryId + WAL_FILE_NAME_DELIMITER + providerId;
97 log = new FSHLog(FileSystem.get(conf), FSUtils.getRootDir(conf),
98 getWALDirectoryName(factory.factoryId), HConstants.HREGION_OLDLOGDIR_NAME, conf, listeners,
99 true, logPrefix, META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null);
100 }
101
102 @Override
103 public WAL getWAL(final byte[] identifier) throws IOException {
104 return log;
105 }
106
107 @Override
108 public void close() throws IOException {
109 log.close();
110 }
111
112 @Override
113 public void shutdown() throws IOException {
114 log.shutdown();
115 }
116
117
118 public static final String WAL_FILE_NAME_DELIMITER = ".";
119
120 @VisibleForTesting
121 public static final String META_WAL_PROVIDER_ID = ".meta";
122 static final String DEFAULT_PROVIDER_ID = "default";
123
124
125
126 public static final String SPLITTING_EXT = "-splitting";
127
128
129
130
131
132
133
134 public static long getNumLogFiles(WALFactory walFactory) {
135 long result = 0;
136 if (walFactory.provider instanceof DefaultWALProvider) {
137 result += ((FSHLog)((DefaultWALProvider)walFactory.provider).log).getNumLogFiles();
138 }
139 WALProvider meta = walFactory.metaProvider.get();
140 if (meta instanceof DefaultWALProvider) {
141 result += ((FSHLog)((DefaultWALProvider)meta).log).getNumLogFiles();
142 }
143 return result;
144 }
145
146
147
148
149
150
151
152 public static long getLogFileSize(WALFactory walFactory) {
153 long result = 0;
154 if (walFactory.provider instanceof DefaultWALProvider) {
155 result += ((FSHLog)((DefaultWALProvider)walFactory.provider).log).getLogFileSize();
156 }
157 WALProvider meta = walFactory.metaProvider.get();
158 if (meta instanceof DefaultWALProvider) {
159 result += ((FSHLog)((DefaultWALProvider)meta).log).getLogFileSize();
160 }
161 return result;
162 }
163
164
165
166
167 @VisibleForTesting
168 public static int getNumRolledLogFiles(WAL wal) {
169 return ((FSHLog)wal).getNumRolledLogFiles();
170 }
171
172
173
174
175 @VisibleForTesting
176 public static Path getCurrentFileName(final WAL wal) {
177 return ((FSHLog)wal).getCurrentFileName();
178 }
179
180
181
182
183 @VisibleForTesting
184 static void requestLogRoll(final WAL wal) {
185 ((FSHLog)wal).requestLogRoll();
186 }
187
188
189
190
191
192
193
194
195 @VisibleForTesting
196 public static long extractFileNumFromWAL(final WAL wal) {
197 final Path walName = ((FSHLog)wal).getCurrentFileName();
198 return extractFileNumFromWAL(walName);
199 }
200
201 @VisibleForTesting
202 public static long extractFileNumFromWAL(final Path walName) {
203 if (walName == null) {
204 throw new IllegalArgumentException("The WAL path couldn't be null");
205 }
206 final String[] walPathStrs = walName.toString().split("\\" + WAL_FILE_NAME_DELIMITER);
207 return Long.parseLong(walPathStrs[walPathStrs.length - (isMetaFile(walName) ? 2:1)]);
208 }
209
210
211
212
213
214 private static final Pattern pattern = Pattern.compile(".*\\.\\d*("+META_WAL_PROVIDER_ID+")*");
215
216
217
218
219
220
221
222
223
224
225
226 public static boolean validateWALFilename(String filename) {
227 return pattern.matcher(filename).matches();
228 }
229
230
231
232
233
234
235
236
237
238
239
240 public static String getWALDirectoryName(final String serverName) {
241 StringBuilder dirName = new StringBuilder(HConstants.HREGION_LOGDIR_NAME);
242 dirName.append("/");
243 dirName.append(serverName);
244 return dirName.toString();
245 }
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263 public static ServerName getServerNameFromWALDirectoryName(Configuration conf, String path)
264 throws IOException {
265 if (path == null
266 || path.length() <= HConstants.HREGION_LOGDIR_NAME.length()) {
267 return null;
268 }
269
270 if (conf == null) {
271 throw new IllegalArgumentException("parameter conf must be set");
272 }
273
274 final String rootDir = conf.get(HConstants.HBASE_DIR);
275 if (rootDir == null || rootDir.isEmpty()) {
276 throw new IllegalArgumentException(HConstants.HBASE_DIR
277 + " key not found in conf.");
278 }
279
280 final StringBuilder startPathSB = new StringBuilder(rootDir);
281 if (!rootDir.endsWith("/"))
282 startPathSB.append('/');
283 startPathSB.append(HConstants.HREGION_LOGDIR_NAME);
284 if (!HConstants.HREGION_LOGDIR_NAME.endsWith("/"))
285 startPathSB.append('/');
286 final String startPath = startPathSB.toString();
287
288 String fullPath;
289 try {
290 fullPath = FileSystem.get(conf).makeQualified(new Path(path)).toString();
291 } catch (IllegalArgumentException e) {
292 LOG.info("Call to makeQualified failed on " + path + " " + e.getMessage());
293 return null;
294 }
295
296 if (!fullPath.startsWith(startPath)) {
297 return null;
298 }
299
300 final String serverNameAndFile = fullPath.substring(startPath.length());
301
302 if (serverNameAndFile.indexOf('/') < "a,0,0".length()) {
303
304 return null;
305 }
306
307 Path p = new Path(path);
308 return getServerNameFromWALDirectoryName(p);
309 }
310
311
312
313
314
315
316
317
318
319
320
321 public static ServerName getServerNameFromWALDirectoryName(Path logFile) {
322 String logDirName = logFile.getParent().getName();
323
324 if (logDirName.equals(HConstants.HREGION_LOGDIR_NAME)) {
325 logDirName = logFile.getName();
326 }
327 ServerName serverName = null;
328 if (logDirName.endsWith(SPLITTING_EXT)) {
329 logDirName = logDirName.substring(0, logDirName.length() - SPLITTING_EXT.length());
330 }
331 try {
332 serverName = ServerName.parseServerName(logDirName);
333 } catch (IllegalArgumentException ex) {
334 serverName = null;
335 LOG.warn("Cannot parse a server name from path=" + logFile + "; " + ex.getMessage());
336 }
337 if (serverName != null && serverName.getStartcode() < 0) {
338 LOG.warn("Invalid log file path=" + logFile);
339 serverName = null;
340 }
341 return serverName;
342 }
343
344 public static boolean isMetaFile(Path p) {
345 return isMetaFile(p.getName());
346 }
347
348 public static boolean isMetaFile(String p) {
349 if (p != null && p.endsWith(META_WAL_PROVIDER_ID)) {
350 return true;
351 }
352 return false;
353 }
354
355
356
357
358 public static Writer createWriter(final Configuration conf, final FileSystem fs, final Path path,
359 final boolean overwritable)
360 throws IOException {
361
362 Class<? extends Writer> logWriterClass = conf.getClass("hbase.regionserver.hlog.writer.impl",
363 ProtobufLogWriter.class, Writer.class);
364 Writer writer = null;
365 try {
366 writer = logWriterClass.newInstance();
367 writer.init(fs, path, conf, overwritable);
368 return writer;
369 } catch (Exception e) {
370 LOG.debug("Error instantiating log writer.", e);
371 if (writer != null) {
372 try{
373 writer.close();
374 } catch(IOException ee){
375 LOG.error("cannot close log writer", ee);
376 }
377 }
378 throw new IOException("cannot get log writer", e);
379 }
380 }
381
382 }