001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.fs;
020    
021    import java.io.*;
022    import java.nio.channels.ClosedChannelException;
023    import java.util.Arrays;
024    
025    import org.apache.commons.logging.Log;
026    import org.apache.commons.logging.LogFactory;
027    import org.apache.hadoop.classification.InterfaceAudience;
028    import org.apache.hadoop.classification.InterfaceStability;
029    import org.apache.hadoop.conf.Configuration;
030    import org.apache.hadoop.fs.permission.FsPermission;
031    import org.apache.hadoop.util.Progressable;
032    import org.apache.hadoop.util.PureJavaCrc32;
033    
034    /****************************************************************
035     * Abstract Checksumed FileSystem.
036     * It provide a basice implementation of a Checksumed FileSystem,
037     * which creates a checksum file for each raw file.
038     * It generates & verifies checksums at the client side.
039     *
040     *****************************************************************/
041    @InterfaceAudience.Public
042    @InterfaceStability.Stable
043    public abstract class ChecksumFileSystem extends FilterFileSystem {
044      private static final byte[] CHECKSUM_VERSION = new byte[] {'c', 'r', 'c', 0};
045      private int bytesPerChecksum = 512;
046      private boolean verifyChecksum = true;
047      private boolean writeChecksum = true;
048    
049      public static double getApproxChkSumLength(long size) {
050        return ChecksumFSOutputSummer.CHKSUM_AS_FRACTION * size;
051      }
052      
053      public ChecksumFileSystem(FileSystem fs) {
054        super(fs);
055      }
056    
057      public void setConf(Configuration conf) {
058        super.setConf(conf);
059        if (conf != null) {
060          bytesPerChecksum = conf.getInt(LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_KEY,
061                                         LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_DEFAULT);
062        }
063      }
064      
065      /**
066       * Set whether to verify checksum.
067       */
068      public void setVerifyChecksum(boolean verifyChecksum) {
069        this.verifyChecksum = verifyChecksum;
070      }
071    
072      @Override
073      public void setWriteChecksum(boolean writeChecksum) {
074        this.writeChecksum = writeChecksum;
075      }
076      
077      /** get the raw file system */
078      public FileSystem getRawFileSystem() {
079        return fs;
080      }
081    
082      /** Return the name of the checksum file associated with a file.*/
083      public Path getChecksumFile(Path file) {
084        return new Path(file.getParent(), "." + file.getName() + ".crc");
085      }
086    
087      /** Return true iff file is a checksum file name.*/
088      public static boolean isChecksumFile(Path file) {
089        String name = file.getName();
090        return name.startsWith(".") && name.endsWith(".crc");
091      }
092    
093      /** Return the length of the checksum file given the size of the 
094       * actual file.
095       **/
096      public long getChecksumFileLength(Path file, long fileSize) {
097        return getChecksumLength(fileSize, getBytesPerSum());
098      }
099    
100      /** Return the bytes Per Checksum */
101      public int getBytesPerSum() {
102        return bytesPerChecksum;
103      }
104    
105      private int getSumBufferSize(int bytesPerSum, int bufferSize) {
106        int defaultBufferSize = getConf().getInt(
107                           LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY,
108                           LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT);
109        int proportionalBufferSize = bufferSize / bytesPerSum;
110        return Math.max(bytesPerSum,
111                        Math.max(proportionalBufferSize, defaultBufferSize));
112      }
113    
114      /*******************************************************
115       * For open()'s FSInputStream
116       * It verifies that data matches checksums.
117       *******************************************************/
118      private static class ChecksumFSInputChecker extends FSInputChecker {
119        public static final Log LOG 
120          = LogFactory.getLog(FSInputChecker.class);
121        
122        private ChecksumFileSystem fs;
123        private FSDataInputStream datas;
124        private FSDataInputStream sums;
125        
126        private static final int HEADER_LENGTH = 8;
127        
128        private int bytesPerSum = 1;
129        
130        public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file)
131          throws IOException {
132          this(fs, file, fs.getConf().getInt(
133                           LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY, 
134                           LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT));
135        }
136        
137        public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize)
138          throws IOException {
139          super( file, fs.getFileStatus(file).getReplication() );
140          this.datas = fs.getRawFileSystem().open(file, bufferSize);
141          this.fs = fs;
142          Path sumFile = fs.getChecksumFile(file);
143          try {
144            int sumBufferSize = fs.getSumBufferSize(fs.getBytesPerSum(), bufferSize);
145            sums = fs.getRawFileSystem().open(sumFile, sumBufferSize);
146    
147            byte[] version = new byte[CHECKSUM_VERSION.length];
148            sums.readFully(version);
149            if (!Arrays.equals(version, CHECKSUM_VERSION))
150              throw new IOException("Not a checksum file: "+sumFile);
151            this.bytesPerSum = sums.readInt();
152            set(fs.verifyChecksum, new PureJavaCrc32(), bytesPerSum, 4);
153          } catch (FileNotFoundException e) {         // quietly ignore
154            set(fs.verifyChecksum, null, 1, 0);
155          } catch (IOException e) {                   // loudly ignore
156            LOG.warn("Problem opening checksum file: "+ file + 
157                     ".  Ignoring exception: " , e); 
158            set(fs.verifyChecksum, null, 1, 0);
159          }
160        }
161        
162        private long getChecksumFilePos( long dataPos ) {
163          return HEADER_LENGTH + 4*(dataPos/bytesPerSum);
164        }
165        
166        protected long getChunkPosition( long dataPos ) {
167          return dataPos/bytesPerSum*bytesPerSum;
168        }
169        
170        public int available() throws IOException {
171          return datas.available() + super.available();
172        }
173        
174        public int read(long position, byte[] b, int off, int len)
175          throws IOException {
176          // parameter check
177          if ((off | len | (off + len) | (b.length - (off + len))) < 0) {
178            throw new IndexOutOfBoundsException();
179          } else if (len == 0) {
180            return 0;
181          }
182          if( position<0 ) {
183            throw new IllegalArgumentException(
184                "Parameter position can not to be negative");
185          }
186    
187          ChecksumFSInputChecker checker = new ChecksumFSInputChecker(fs, file);
188          checker.seek(position);
189          int nread = checker.read(b, off, len);
190          checker.close();
191          return nread;
192        }
193        
194        public void close() throws IOException {
195          datas.close();
196          if( sums != null ) {
197            sums.close();
198          }
199          set(fs.verifyChecksum, null, 1, 0);
200        }
201        
202    
203        @Override
204        public boolean seekToNewSource(long targetPos) throws IOException {
205          long sumsPos = getChecksumFilePos(targetPos);
206          fs.reportChecksumFailure(file, datas, targetPos, sums, sumsPos);
207          boolean newDataSource = datas.seekToNewSource(targetPos);
208          return sums.seekToNewSource(sumsPos) || newDataSource;
209        }
210    
211        @Override
212        protected int readChunk(long pos, byte[] buf, int offset, int len,
213            byte[] checksum) throws IOException {
214    
215          boolean eof = false;
216          if (needChecksum()) {
217            assert checksum != null; // we have a checksum buffer
218            assert checksum.length % CHECKSUM_SIZE == 0; // it is sane length
219            assert len >= bytesPerSum; // we must read at least one chunk
220    
221            final int checksumsToRead = Math.min(
222              len/bytesPerSum, // number of checksums based on len to read
223              checksum.length / CHECKSUM_SIZE); // size of checksum buffer
224            long checksumPos = getChecksumFilePos(pos); 
225            if(checksumPos != sums.getPos()) {
226              sums.seek(checksumPos);
227            }
228    
229            int sumLenRead = sums.read(checksum, 0, CHECKSUM_SIZE * checksumsToRead);
230            if (sumLenRead >= 0 && sumLenRead % CHECKSUM_SIZE != 0) {
231              throw new ChecksumException(
232                "Checksum file not a length multiple of checksum size " +
233                "in " + file + " at " + pos + " checksumpos: " + checksumPos +
234                " sumLenread: " + sumLenRead,
235                pos);
236            }
237            if (sumLenRead <= 0) { // we're at the end of the file
238              eof = true;
239            } else {
240              // Adjust amount of data to read based on how many checksum chunks we read
241              len = Math.min(len, bytesPerSum * (sumLenRead / CHECKSUM_SIZE));
242            }
243          }
244          if(pos != datas.getPos()) {
245            datas.seek(pos);
246          }
247          int nread = readFully(datas, buf, offset, len);
248          if (eof && nread > 0) {
249            throw new ChecksumException("Checksum error: "+file+" at "+pos, pos);
250          }
251          return nread;
252        }
253      }
254      
255      private static class FSDataBoundedInputStream extends FSDataInputStream {
256        private FileSystem fs;
257        private Path file;
258        private long fileLen = -1L;
259    
260        FSDataBoundedInputStream(FileSystem fs, Path file, InputStream in)
261            throws IOException {
262          super(in);
263          this.fs = fs;
264          this.file = file;
265        }
266        
267        @Override
268        public boolean markSupported() {
269          return false;
270        }
271        
272        /* Return the file length */
273        private long getFileLength() throws IOException {
274          if( fileLen==-1L ) {
275            fileLen = fs.getContentSummary(file).getLength();
276          }
277          return fileLen;
278        }
279        
280        /**
281         * Skips over and discards <code>n</code> bytes of data from the
282         * input stream.
283         *
284         *The <code>skip</code> method skips over some smaller number of bytes
285         * when reaching end of file before <code>n</code> bytes have been skipped.
286         * The actual number of bytes skipped is returned.  If <code>n</code> is
287         * negative, no bytes are skipped.
288         *
289         * @param      n   the number of bytes to be skipped.
290         * @return     the actual number of bytes skipped.
291         * @exception  IOException  if an I/O error occurs.
292         *             ChecksumException if the chunk to skip to is corrupted
293         */
294        public synchronized long skip(long n) throws IOException {
295          long curPos = getPos();
296          long fileLength = getFileLength();
297          if( n+curPos > fileLength ) {
298            n = fileLength - curPos;
299          }
300          return super.skip(n);
301        }
302        
303        /**
304         * Seek to the given position in the stream.
305         * The next read() will be from that position.
306         * 
307         * <p>This method does not allow seek past the end of the file.
308         * This produces IOException.
309         *
310         * @param      pos   the postion to seek to.
311         * @exception  IOException  if an I/O error occurs or seeks after EOF
312         *             ChecksumException if the chunk to seek to is corrupted
313         */
314    
315        public synchronized void seek(long pos) throws IOException {
316          if(pos>getFileLength()) {
317            throw new IOException("Cannot seek after EOF");
318          }
319          super.seek(pos);
320        }
321    
322      }
323    
324      /**
325       * Opens an FSDataInputStream at the indicated Path.
326       * @param f the file name to open
327       * @param bufferSize the size of the buffer to be used.
328       */
329      @Override
330      public FSDataInputStream open(Path f, int bufferSize) throws IOException {
331        FileSystem fs;
332        InputStream in;
333        if (verifyChecksum) {
334          fs = this;
335          in = new ChecksumFSInputChecker(this, f, bufferSize);
336        } else {
337          fs = getRawFileSystem();
338          in = fs.open(f, bufferSize);
339        }
340        return new FSDataBoundedInputStream(fs, f, in);
341      }
342    
343      /** {@inheritDoc} */
344      public FSDataOutputStream append(Path f, int bufferSize,
345          Progressable progress) throws IOException {
346        throw new IOException("Not supported");
347      }
348    
349      /**
350       * Calculated the length of the checksum file in bytes.
351       * @param size the length of the data file in bytes
352       * @param bytesPerSum the number of bytes in a checksum block
353       * @return the number of bytes in the checksum file
354       */
355      public static long getChecksumLength(long size, int bytesPerSum) {
356        //the checksum length is equal to size passed divided by bytesPerSum +
357        //bytes written in the beginning of the checksum file.  
358        return ((size + bytesPerSum - 1) / bytesPerSum) * 4 +
359                 CHECKSUM_VERSION.length + 4;  
360      }
361    
362      /** This class provides an output stream for a checksummed file.
363       * It generates checksums for data. */
364      private static class ChecksumFSOutputSummer extends FSOutputSummer {
365        private FSDataOutputStream datas;    
366        private FSDataOutputStream sums;
367        private static final float CHKSUM_AS_FRACTION = 0.01f;
368        private boolean isClosed = false;
369        
370        public ChecksumFSOutputSummer(ChecksumFileSystem fs, 
371                              Path file, 
372                              boolean overwrite, 
373                              short replication,
374                              long blockSize,
375                              Configuration conf)
376          throws IOException {
377          this(fs, file, overwrite, 
378               conf.getInt(LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY,
379                           LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT),
380               replication, blockSize, null);
381        }
382        
383        public ChecksumFSOutputSummer(ChecksumFileSystem fs, 
384                              Path file, 
385                              boolean overwrite,
386                              int bufferSize,
387                              short replication,
388                              long blockSize,
389                              Progressable progress)
390          throws IOException {
391          super(new PureJavaCrc32(), fs.getBytesPerSum(), 4);
392          int bytesPerSum = fs.getBytesPerSum();
393          this.datas = fs.getRawFileSystem().create(file, overwrite, bufferSize, 
394                                             replication, blockSize, progress);
395          int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize);
396          this.sums = fs.getRawFileSystem().create(fs.getChecksumFile(file), true, 
397                                                   sumBufferSize, replication,
398                                                   blockSize);
399          sums.write(CHECKSUM_VERSION, 0, CHECKSUM_VERSION.length);
400          sums.writeInt(bytesPerSum);
401        }
402        
403        public void close() throws IOException {
404          try {
405            flushBuffer();
406            sums.close();
407            datas.close();
408          } finally {
409            isClosed = true;
410          }
411        }
412        
413        @Override
414        protected void writeChunk(byte[] b, int offset, int len, byte[] checksum)
415        throws IOException {
416          datas.write(b, offset, len);
417          sums.write(checksum);
418        }
419    
420        @Override
421        protected void checkClosed() throws IOException {
422          if (isClosed) {
423            throw new ClosedChannelException();
424          }
425        }
426      }
427    
428      /** {@inheritDoc} */
429      @Override
430      public FSDataOutputStream create(Path f, FsPermission permission,
431          boolean overwrite, int bufferSize, short replication, long blockSize,
432          Progressable progress) throws IOException {
433        return create(f, permission, overwrite, true, bufferSize,
434            replication, blockSize, progress);
435      }
436    
437      private FSDataOutputStream create(Path f, FsPermission permission,
438          boolean overwrite, boolean createParent, int bufferSize,
439          short replication, long blockSize,
440          Progressable progress) throws IOException {
441        Path parent = f.getParent();
442        if (parent != null) {
443          if (!createParent && !exists(parent)) {
444            throw new FileNotFoundException("Parent directory doesn't exist: "
445                + parent);
446          } else if (!mkdirs(parent)) {
447            throw new IOException("Mkdirs failed to create " + parent);
448          }
449        }
450        final FSDataOutputStream out;
451        if (writeChecksum) {
452          out = new FSDataOutputStream(
453              new ChecksumFSOutputSummer(this, f, overwrite, bufferSize, replication,
454                  blockSize, progress), null);
455        } else {
456          out = fs.create(f, permission, overwrite, bufferSize, replication,
457              blockSize, progress);
458          // remove the checksum file since we aren't writing one
459          Path checkFile = getChecksumFile(f);
460          if (fs.exists(checkFile)) {
461            fs.delete(checkFile, true);
462          }
463        }
464        if (permission != null) {
465          setPermission(f, permission);
466        }
467        return out;
468      }
469    
470      /** {@inheritDoc} */
471      @Override
472      public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
473          boolean overwrite, int bufferSize, short replication, long blockSize,
474          Progressable progress) throws IOException {
475        return create(f, permission, overwrite, false, bufferSize, replication,
476            blockSize, progress);
477      }
478    
479      /**
480       * Set replication for an existing file.
481       * Implement the abstract <tt>setReplication</tt> of <tt>FileSystem</tt>
482       * @param src file name
483       * @param replication new replication
484       * @throws IOException
485       * @return true if successful;
486       *         false if file does not exist or is a directory
487       */
488      public boolean setReplication(Path src, short replication) throws IOException {
489        boolean value = fs.setReplication(src, replication);
490        if (!value)
491          return false;
492    
493        Path checkFile = getChecksumFile(src);
494        if (exists(checkFile))
495          fs.setReplication(checkFile, replication);
496    
497        return true;
498      }
499    
500      /**
501       * Rename files/dirs
502       */
503      public boolean rename(Path src, Path dst) throws IOException {
504        if (fs.isDirectory(src)) {
505          return fs.rename(src, dst);
506        } else {
507          if (fs.isDirectory(dst)) {
508            dst = new Path(dst, src.getName());
509          }
510    
511          boolean value = fs.rename(src, dst);
512          if (!value)
513            return false;
514    
515          Path srcCheckFile = getChecksumFile(src);
516          Path dstCheckFile = getChecksumFile(dst);
517          if (fs.exists(srcCheckFile)) { //try to rename checksum
518            value = fs.rename(srcCheckFile, dstCheckFile);
519          } else if (fs.exists(dstCheckFile)) {
520            // no src checksum, so remove dst checksum
521            value = fs.delete(dstCheckFile, true); 
522          }
523    
524          return value;
525        }
526      }
527    
528      /**
529       * Implement the delete(Path, boolean) in checksum
530       * file system.
531       */
532      public boolean delete(Path f, boolean recursive) throws IOException{
533        FileStatus fstatus = null;
534        try {
535          fstatus = fs.getFileStatus(f);
536        } catch(FileNotFoundException e) {
537          return false;
538        }
539        if (fstatus.isDirectory()) {
540          //this works since the crcs are in the same
541          //directories and the files. so we just delete
542          //everything in the underlying filesystem
543          return fs.delete(f, recursive);
544        } else {
545          Path checkFile = getChecksumFile(f);
546          if (fs.exists(checkFile)) {
547            fs.delete(checkFile, true);
548          }
549          return fs.delete(f, true);
550        }
551      }
552        
553      final private static PathFilter DEFAULT_FILTER = new PathFilter() {
554        public boolean accept(Path file) {
555          return !isChecksumFile(file);
556        }
557      };
558    
559      /**
560       * List the statuses of the files/directories in the given path if the path is
561       * a directory.
562       * 
563       * @param f
564       *          given path
565       * @return the statuses of the files/directories in the given patch
566       * @throws IOException
567       */
568      @Override
569      public FileStatus[] listStatus(Path f) throws IOException {
570        return fs.listStatus(f, DEFAULT_FILTER);
571      }
572      
573      /**
574       * List the statuses of the files/directories in the given path if the path is
575       * a directory.
576       * 
577       * @param f
578       *          given path
579       * @return the statuses of the files/directories in the given patch
580       * @throws IOException
581       */
582      @Override
583      public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f)
584      throws IOException {
585        return fs.listLocatedStatus(f, DEFAULT_FILTER);
586      }
587      
588      @Override
589      public boolean mkdirs(Path f) throws IOException {
590        return fs.mkdirs(f);
591      }
592    
593      @Override
594      public void copyFromLocalFile(boolean delSrc, Path src, Path dst)
595        throws IOException {
596        Configuration conf = getConf();
597        FileUtil.copy(getLocal(conf), src, this, dst, delSrc, conf);
598      }
599    
600      /**
601       * The src file is under FS, and the dst is on the local disk.
602       * Copy it from FS control to the local dst name.
603       */
604      @Override
605      public void copyToLocalFile(boolean delSrc, Path src, Path dst)
606        throws IOException {
607        Configuration conf = getConf();
608        FileUtil.copy(this, src, getLocal(conf), dst, delSrc, conf);
609      }
610    
611      /**
612       * The src file is under FS, and the dst is on the local disk.
613       * Copy it from FS control to the local dst name.
614       * If src and dst are directories, the copyCrc parameter
615       * determines whether to copy CRC files.
616       */
617      public void copyToLocalFile(Path src, Path dst, boolean copyCrc)
618        throws IOException {
619        if (!fs.isDirectory(src)) { // source is a file
620          fs.copyToLocalFile(src, dst);
621          FileSystem localFs = getLocal(getConf()).getRawFileSystem();
622          if (localFs.isDirectory(dst)) {
623            dst = new Path(dst, src.getName());
624          }
625          dst = getChecksumFile(dst);
626          if (localFs.exists(dst)) { //remove old local checksum file
627            localFs.delete(dst, true);
628          }
629          Path checksumFile = getChecksumFile(src);
630          if (copyCrc && fs.exists(checksumFile)) { //copy checksum file
631            fs.copyToLocalFile(checksumFile, dst);
632          }
633        } else {
634          FileStatus[] srcs = listStatus(src);
635          for (FileStatus srcFile : srcs) {
636            copyToLocalFile(srcFile.getPath(), 
637                            new Path(dst, srcFile.getPath().getName()), copyCrc);
638          }
639        }
640      }
641    
642      @Override
643      public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
644        throws IOException {
645        return tmpLocalFile;
646      }
647    
648      @Override
649      public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile)
650        throws IOException {
651        moveFromLocalFile(tmpLocalFile, fsOutputFile);
652      }
653    
654      /**
655       * Report a checksum error to the file system.
656       * @param f the file name containing the error
657       * @param in the stream open on the file
658       * @param inPos the position of the beginning of the bad data in the file
659       * @param sums the stream open on the checksum file
660       * @param sumsPos the position of the beginning of the bad data in the checksum file
661       * @return if retry is neccessary
662       */
663      public boolean reportChecksumFailure(Path f, FSDataInputStream in,
664                                           long inPos, FSDataInputStream sums, long sumsPos) {
665        return false;
666      }
667    }