001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.hadoop.fs; 020 021 import java.io.*; 022 import java.nio.channels.ClosedChannelException; 023 import java.util.Arrays; 024 025 import org.apache.commons.logging.Log; 026 import org.apache.commons.logging.LogFactory; 027 import org.apache.hadoop.classification.InterfaceAudience; 028 import org.apache.hadoop.classification.InterfaceStability; 029 import org.apache.hadoop.conf.Configuration; 030 import org.apache.hadoop.fs.permission.FsPermission; 031 import org.apache.hadoop.util.Progressable; 032 import org.apache.hadoop.util.PureJavaCrc32; 033 034 /**************************************************************** 035 * Abstract Checksumed FileSystem. 036 * It provide a basice implementation of a Checksumed FileSystem, 037 * which creates a checksum file for each raw file. 038 * It generates & verifies checksums at the client side. 039 * 040 *****************************************************************/ 041 @InterfaceAudience.Public 042 @InterfaceStability.Stable 043 public abstract class ChecksumFileSystem extends FilterFileSystem { 044 private static final byte[] CHECKSUM_VERSION = new byte[] {'c', 'r', 'c', 0}; 045 private int bytesPerChecksum = 512; 046 private boolean verifyChecksum = true; 047 private boolean writeChecksum = true; 048 049 public static double getApproxChkSumLength(long size) { 050 return ChecksumFSOutputSummer.CHKSUM_AS_FRACTION * size; 051 } 052 053 public ChecksumFileSystem(FileSystem fs) { 054 super(fs); 055 } 056 057 public void setConf(Configuration conf) { 058 super.setConf(conf); 059 if (conf != null) { 060 bytesPerChecksum = conf.getInt(LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_KEY, 061 LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_DEFAULT); 062 } 063 } 064 065 /** 066 * Set whether to verify checksum. 067 */ 068 public void setVerifyChecksum(boolean verifyChecksum) { 069 this.verifyChecksum = verifyChecksum; 070 } 071 072 @Override 073 public void setWriteChecksum(boolean writeChecksum) { 074 this.writeChecksum = writeChecksum; 075 } 076 077 /** get the raw file system */ 078 public FileSystem getRawFileSystem() { 079 return fs; 080 } 081 082 /** Return the name of the checksum file associated with a file.*/ 083 public Path getChecksumFile(Path file) { 084 return new Path(file.getParent(), "." + file.getName() + ".crc"); 085 } 086 087 /** Return true iff file is a checksum file name.*/ 088 public static boolean isChecksumFile(Path file) { 089 String name = file.getName(); 090 return name.startsWith(".") && name.endsWith(".crc"); 091 } 092 093 /** Return the length of the checksum file given the size of the 094 * actual file. 095 **/ 096 public long getChecksumFileLength(Path file, long fileSize) { 097 return getChecksumLength(fileSize, getBytesPerSum()); 098 } 099 100 /** Return the bytes Per Checksum */ 101 public int getBytesPerSum() { 102 return bytesPerChecksum; 103 } 104 105 private int getSumBufferSize(int bytesPerSum, int bufferSize) { 106 int defaultBufferSize = getConf().getInt( 107 LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY, 108 LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT); 109 int proportionalBufferSize = bufferSize / bytesPerSum; 110 return Math.max(bytesPerSum, 111 Math.max(proportionalBufferSize, defaultBufferSize)); 112 } 113 114 /******************************************************* 115 * For open()'s FSInputStream 116 * It verifies that data matches checksums. 117 *******************************************************/ 118 private static class ChecksumFSInputChecker extends FSInputChecker { 119 public static final Log LOG 120 = LogFactory.getLog(FSInputChecker.class); 121 122 private ChecksumFileSystem fs; 123 private FSDataInputStream datas; 124 private FSDataInputStream sums; 125 126 private static final int HEADER_LENGTH = 8; 127 128 private int bytesPerSum = 1; 129 130 public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file) 131 throws IOException { 132 this(fs, file, fs.getConf().getInt( 133 LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY, 134 LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT)); 135 } 136 137 public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize) 138 throws IOException { 139 super( file, fs.getFileStatus(file).getReplication() ); 140 this.datas = fs.getRawFileSystem().open(file, bufferSize); 141 this.fs = fs; 142 Path sumFile = fs.getChecksumFile(file); 143 try { 144 int sumBufferSize = fs.getSumBufferSize(fs.getBytesPerSum(), bufferSize); 145 sums = fs.getRawFileSystem().open(sumFile, sumBufferSize); 146 147 byte[] version = new byte[CHECKSUM_VERSION.length]; 148 sums.readFully(version); 149 if (!Arrays.equals(version, CHECKSUM_VERSION)) 150 throw new IOException("Not a checksum file: "+sumFile); 151 this.bytesPerSum = sums.readInt(); 152 set(fs.verifyChecksum, new PureJavaCrc32(), bytesPerSum, 4); 153 } catch (FileNotFoundException e) { // quietly ignore 154 set(fs.verifyChecksum, null, 1, 0); 155 } catch (IOException e) { // loudly ignore 156 LOG.warn("Problem opening checksum file: "+ file + 157 ". Ignoring exception: " , e); 158 set(fs.verifyChecksum, null, 1, 0); 159 } 160 } 161 162 private long getChecksumFilePos( long dataPos ) { 163 return HEADER_LENGTH + 4*(dataPos/bytesPerSum); 164 } 165 166 protected long getChunkPosition( long dataPos ) { 167 return dataPos/bytesPerSum*bytesPerSum; 168 } 169 170 public int available() throws IOException { 171 return datas.available() + super.available(); 172 } 173 174 public int read(long position, byte[] b, int off, int len) 175 throws IOException { 176 // parameter check 177 if ((off | len | (off + len) | (b.length - (off + len))) < 0) { 178 throw new IndexOutOfBoundsException(); 179 } else if (len == 0) { 180 return 0; 181 } 182 if( position<0 ) { 183 throw new IllegalArgumentException( 184 "Parameter position can not to be negative"); 185 } 186 187 ChecksumFSInputChecker checker = new ChecksumFSInputChecker(fs, file); 188 checker.seek(position); 189 int nread = checker.read(b, off, len); 190 checker.close(); 191 return nread; 192 } 193 194 public void close() throws IOException { 195 datas.close(); 196 if( sums != null ) { 197 sums.close(); 198 } 199 set(fs.verifyChecksum, null, 1, 0); 200 } 201 202 203 @Override 204 public boolean seekToNewSource(long targetPos) throws IOException { 205 long sumsPos = getChecksumFilePos(targetPos); 206 fs.reportChecksumFailure(file, datas, targetPos, sums, sumsPos); 207 boolean newDataSource = datas.seekToNewSource(targetPos); 208 return sums.seekToNewSource(sumsPos) || newDataSource; 209 } 210 211 @Override 212 protected int readChunk(long pos, byte[] buf, int offset, int len, 213 byte[] checksum) throws IOException { 214 215 boolean eof = false; 216 if (needChecksum()) { 217 assert checksum != null; // we have a checksum buffer 218 assert checksum.length % CHECKSUM_SIZE == 0; // it is sane length 219 assert len >= bytesPerSum; // we must read at least one chunk 220 221 final int checksumsToRead = Math.min( 222 len/bytesPerSum, // number of checksums based on len to read 223 checksum.length / CHECKSUM_SIZE); // size of checksum buffer 224 long checksumPos = getChecksumFilePos(pos); 225 if(checksumPos != sums.getPos()) { 226 sums.seek(checksumPos); 227 } 228 229 int sumLenRead = sums.read(checksum, 0, CHECKSUM_SIZE * checksumsToRead); 230 if (sumLenRead >= 0 && sumLenRead % CHECKSUM_SIZE != 0) { 231 throw new ChecksumException( 232 "Checksum file not a length multiple of checksum size " + 233 "in " + file + " at " + pos + " checksumpos: " + checksumPos + 234 " sumLenread: " + sumLenRead, 235 pos); 236 } 237 if (sumLenRead <= 0) { // we're at the end of the file 238 eof = true; 239 } else { 240 // Adjust amount of data to read based on how many checksum chunks we read 241 len = Math.min(len, bytesPerSum * (sumLenRead / CHECKSUM_SIZE)); 242 } 243 } 244 if(pos != datas.getPos()) { 245 datas.seek(pos); 246 } 247 int nread = readFully(datas, buf, offset, len); 248 if (eof && nread > 0) { 249 throw new ChecksumException("Checksum error: "+file+" at "+pos, pos); 250 } 251 return nread; 252 } 253 } 254 255 private static class FSDataBoundedInputStream extends FSDataInputStream { 256 private FileSystem fs; 257 private Path file; 258 private long fileLen = -1L; 259 260 FSDataBoundedInputStream(FileSystem fs, Path file, InputStream in) 261 throws IOException { 262 super(in); 263 this.fs = fs; 264 this.file = file; 265 } 266 267 @Override 268 public boolean markSupported() { 269 return false; 270 } 271 272 /* Return the file length */ 273 private long getFileLength() throws IOException { 274 if( fileLen==-1L ) { 275 fileLen = fs.getContentSummary(file).getLength(); 276 } 277 return fileLen; 278 } 279 280 /** 281 * Skips over and discards <code>n</code> bytes of data from the 282 * input stream. 283 * 284 *The <code>skip</code> method skips over some smaller number of bytes 285 * when reaching end of file before <code>n</code> bytes have been skipped. 286 * The actual number of bytes skipped is returned. If <code>n</code> is 287 * negative, no bytes are skipped. 288 * 289 * @param n the number of bytes to be skipped. 290 * @return the actual number of bytes skipped. 291 * @exception IOException if an I/O error occurs. 292 * ChecksumException if the chunk to skip to is corrupted 293 */ 294 public synchronized long skip(long n) throws IOException { 295 long curPos = getPos(); 296 long fileLength = getFileLength(); 297 if( n+curPos > fileLength ) { 298 n = fileLength - curPos; 299 } 300 return super.skip(n); 301 } 302 303 /** 304 * Seek to the given position in the stream. 305 * The next read() will be from that position. 306 * 307 * <p>This method does not allow seek past the end of the file. 308 * This produces IOException. 309 * 310 * @param pos the postion to seek to. 311 * @exception IOException if an I/O error occurs or seeks after EOF 312 * ChecksumException if the chunk to seek to is corrupted 313 */ 314 315 public synchronized void seek(long pos) throws IOException { 316 if(pos>getFileLength()) { 317 throw new IOException("Cannot seek after EOF"); 318 } 319 super.seek(pos); 320 } 321 322 } 323 324 /** 325 * Opens an FSDataInputStream at the indicated Path. 326 * @param f the file name to open 327 * @param bufferSize the size of the buffer to be used. 328 */ 329 @Override 330 public FSDataInputStream open(Path f, int bufferSize) throws IOException { 331 FileSystem fs; 332 InputStream in; 333 if (verifyChecksum) { 334 fs = this; 335 in = new ChecksumFSInputChecker(this, f, bufferSize); 336 } else { 337 fs = getRawFileSystem(); 338 in = fs.open(f, bufferSize); 339 } 340 return new FSDataBoundedInputStream(fs, f, in); 341 } 342 343 /** {@inheritDoc} */ 344 public FSDataOutputStream append(Path f, int bufferSize, 345 Progressable progress) throws IOException { 346 throw new IOException("Not supported"); 347 } 348 349 /** 350 * Calculated the length of the checksum file in bytes. 351 * @param size the length of the data file in bytes 352 * @param bytesPerSum the number of bytes in a checksum block 353 * @return the number of bytes in the checksum file 354 */ 355 public static long getChecksumLength(long size, int bytesPerSum) { 356 //the checksum length is equal to size passed divided by bytesPerSum + 357 //bytes written in the beginning of the checksum file. 358 return ((size + bytesPerSum - 1) / bytesPerSum) * 4 + 359 CHECKSUM_VERSION.length + 4; 360 } 361 362 /** This class provides an output stream for a checksummed file. 363 * It generates checksums for data. */ 364 private static class ChecksumFSOutputSummer extends FSOutputSummer { 365 private FSDataOutputStream datas; 366 private FSDataOutputStream sums; 367 private static final float CHKSUM_AS_FRACTION = 0.01f; 368 private boolean isClosed = false; 369 370 public ChecksumFSOutputSummer(ChecksumFileSystem fs, 371 Path file, 372 boolean overwrite, 373 short replication, 374 long blockSize, 375 Configuration conf) 376 throws IOException { 377 this(fs, file, overwrite, 378 conf.getInt(LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY, 379 LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT), 380 replication, blockSize, null); 381 } 382 383 public ChecksumFSOutputSummer(ChecksumFileSystem fs, 384 Path file, 385 boolean overwrite, 386 int bufferSize, 387 short replication, 388 long blockSize, 389 Progressable progress) 390 throws IOException { 391 super(new PureJavaCrc32(), fs.getBytesPerSum(), 4); 392 int bytesPerSum = fs.getBytesPerSum(); 393 this.datas = fs.getRawFileSystem().create(file, overwrite, bufferSize, 394 replication, blockSize, progress); 395 int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize); 396 this.sums = fs.getRawFileSystem().create(fs.getChecksumFile(file), true, 397 sumBufferSize, replication, 398 blockSize); 399 sums.write(CHECKSUM_VERSION, 0, CHECKSUM_VERSION.length); 400 sums.writeInt(bytesPerSum); 401 } 402 403 public void close() throws IOException { 404 try { 405 flushBuffer(); 406 sums.close(); 407 datas.close(); 408 } finally { 409 isClosed = true; 410 } 411 } 412 413 @Override 414 protected void writeChunk(byte[] b, int offset, int len, byte[] checksum) 415 throws IOException { 416 datas.write(b, offset, len); 417 sums.write(checksum); 418 } 419 420 @Override 421 protected void checkClosed() throws IOException { 422 if (isClosed) { 423 throw new ClosedChannelException(); 424 } 425 } 426 } 427 428 /** {@inheritDoc} */ 429 @Override 430 public FSDataOutputStream create(Path f, FsPermission permission, 431 boolean overwrite, int bufferSize, short replication, long blockSize, 432 Progressable progress) throws IOException { 433 return create(f, permission, overwrite, true, bufferSize, 434 replication, blockSize, progress); 435 } 436 437 private FSDataOutputStream create(Path f, FsPermission permission, 438 boolean overwrite, boolean createParent, int bufferSize, 439 short replication, long blockSize, 440 Progressable progress) throws IOException { 441 Path parent = f.getParent(); 442 if (parent != null) { 443 if (!createParent && !exists(parent)) { 444 throw new FileNotFoundException("Parent directory doesn't exist: " 445 + parent); 446 } else if (!mkdirs(parent)) { 447 throw new IOException("Mkdirs failed to create " + parent); 448 } 449 } 450 final FSDataOutputStream out; 451 if (writeChecksum) { 452 out = new FSDataOutputStream( 453 new ChecksumFSOutputSummer(this, f, overwrite, bufferSize, replication, 454 blockSize, progress), null); 455 } else { 456 out = fs.create(f, permission, overwrite, bufferSize, replication, 457 blockSize, progress); 458 // remove the checksum file since we aren't writing one 459 Path checkFile = getChecksumFile(f); 460 if (fs.exists(checkFile)) { 461 fs.delete(checkFile, true); 462 } 463 } 464 if (permission != null) { 465 setPermission(f, permission); 466 } 467 return out; 468 } 469 470 /** {@inheritDoc} */ 471 @Override 472 public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, 473 boolean overwrite, int bufferSize, short replication, long blockSize, 474 Progressable progress) throws IOException { 475 return create(f, permission, overwrite, false, bufferSize, replication, 476 blockSize, progress); 477 } 478 479 /** 480 * Set replication for an existing file. 481 * Implement the abstract <tt>setReplication</tt> of <tt>FileSystem</tt> 482 * @param src file name 483 * @param replication new replication 484 * @throws IOException 485 * @return true if successful; 486 * false if file does not exist or is a directory 487 */ 488 public boolean setReplication(Path src, short replication) throws IOException { 489 boolean value = fs.setReplication(src, replication); 490 if (!value) 491 return false; 492 493 Path checkFile = getChecksumFile(src); 494 if (exists(checkFile)) 495 fs.setReplication(checkFile, replication); 496 497 return true; 498 } 499 500 /** 501 * Rename files/dirs 502 */ 503 public boolean rename(Path src, Path dst) throws IOException { 504 if (fs.isDirectory(src)) { 505 return fs.rename(src, dst); 506 } else { 507 if (fs.isDirectory(dst)) { 508 dst = new Path(dst, src.getName()); 509 } 510 511 boolean value = fs.rename(src, dst); 512 if (!value) 513 return false; 514 515 Path srcCheckFile = getChecksumFile(src); 516 Path dstCheckFile = getChecksumFile(dst); 517 if (fs.exists(srcCheckFile)) { //try to rename checksum 518 value = fs.rename(srcCheckFile, dstCheckFile); 519 } else if (fs.exists(dstCheckFile)) { 520 // no src checksum, so remove dst checksum 521 value = fs.delete(dstCheckFile, true); 522 } 523 524 return value; 525 } 526 } 527 528 /** 529 * Implement the delete(Path, boolean) in checksum 530 * file system. 531 */ 532 public boolean delete(Path f, boolean recursive) throws IOException{ 533 FileStatus fstatus = null; 534 try { 535 fstatus = fs.getFileStatus(f); 536 } catch(FileNotFoundException e) { 537 return false; 538 } 539 if (fstatus.isDirectory()) { 540 //this works since the crcs are in the same 541 //directories and the files. so we just delete 542 //everything in the underlying filesystem 543 return fs.delete(f, recursive); 544 } else { 545 Path checkFile = getChecksumFile(f); 546 if (fs.exists(checkFile)) { 547 fs.delete(checkFile, true); 548 } 549 return fs.delete(f, true); 550 } 551 } 552 553 final private static PathFilter DEFAULT_FILTER = new PathFilter() { 554 public boolean accept(Path file) { 555 return !isChecksumFile(file); 556 } 557 }; 558 559 /** 560 * List the statuses of the files/directories in the given path if the path is 561 * a directory. 562 * 563 * @param f 564 * given path 565 * @return the statuses of the files/directories in the given patch 566 * @throws IOException 567 */ 568 @Override 569 public FileStatus[] listStatus(Path f) throws IOException { 570 return fs.listStatus(f, DEFAULT_FILTER); 571 } 572 573 /** 574 * List the statuses of the files/directories in the given path if the path is 575 * a directory. 576 * 577 * @param f 578 * given path 579 * @return the statuses of the files/directories in the given patch 580 * @throws IOException 581 */ 582 @Override 583 public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f) 584 throws IOException { 585 return fs.listLocatedStatus(f, DEFAULT_FILTER); 586 } 587 588 @Override 589 public boolean mkdirs(Path f) throws IOException { 590 return fs.mkdirs(f); 591 } 592 593 @Override 594 public void copyFromLocalFile(boolean delSrc, Path src, Path dst) 595 throws IOException { 596 Configuration conf = getConf(); 597 FileUtil.copy(getLocal(conf), src, this, dst, delSrc, conf); 598 } 599 600 /** 601 * The src file is under FS, and the dst is on the local disk. 602 * Copy it from FS control to the local dst name. 603 */ 604 @Override 605 public void copyToLocalFile(boolean delSrc, Path src, Path dst) 606 throws IOException { 607 Configuration conf = getConf(); 608 FileUtil.copy(this, src, getLocal(conf), dst, delSrc, conf); 609 } 610 611 /** 612 * The src file is under FS, and the dst is on the local disk. 613 * Copy it from FS control to the local dst name. 614 * If src and dst are directories, the copyCrc parameter 615 * determines whether to copy CRC files. 616 */ 617 public void copyToLocalFile(Path src, Path dst, boolean copyCrc) 618 throws IOException { 619 if (!fs.isDirectory(src)) { // source is a file 620 fs.copyToLocalFile(src, dst); 621 FileSystem localFs = getLocal(getConf()).getRawFileSystem(); 622 if (localFs.isDirectory(dst)) { 623 dst = new Path(dst, src.getName()); 624 } 625 dst = getChecksumFile(dst); 626 if (localFs.exists(dst)) { //remove old local checksum file 627 localFs.delete(dst, true); 628 } 629 Path checksumFile = getChecksumFile(src); 630 if (copyCrc && fs.exists(checksumFile)) { //copy checksum file 631 fs.copyToLocalFile(checksumFile, dst); 632 } 633 } else { 634 FileStatus[] srcs = listStatus(src); 635 for (FileStatus srcFile : srcs) { 636 copyToLocalFile(srcFile.getPath(), 637 new Path(dst, srcFile.getPath().getName()), copyCrc); 638 } 639 } 640 } 641 642 @Override 643 public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile) 644 throws IOException { 645 return tmpLocalFile; 646 } 647 648 @Override 649 public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile) 650 throws IOException { 651 moveFromLocalFile(tmpLocalFile, fsOutputFile); 652 } 653 654 /** 655 * Report a checksum error to the file system. 656 * @param f the file name containing the error 657 * @param in the stream open on the file 658 * @param inPos the position of the beginning of the bad data in the file 659 * @param sums the stream open on the checksum file 660 * @param sumsPos the position of the beginning of the bad data in the checksum file 661 * @return if retry is neccessary 662 */ 663 public boolean reportChecksumFailure(Path f, FSDataInputStream in, 664 long inPos, FSDataInputStream sums, long sumsPos) { 665 return false; 666 } 667 }